Refactor BYAF parsing (#4303) * Refactor BYAF parsing * Refactor ByafParser to use private field for data and improve type annotations

38a5e4a4f2a0068510124423f9e4c9913116e1f2

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

Parents: 7edd81f183014568c47831d2c02dc926a80d0eda

src/byaf.js+124 -40 Viewed

 import { DEFAULT_AVATAR_PATH } from './constants.js';
 import { extractFileFromZipBuffer, humanizedISO8601DateTime } from './util.js';
-export const replaceByafMacros = (s) =>
+/**
-    String(s || '')
+ * A parser for BYAF (Backyard Archive Format) files.
+ */
+export class ByafParser {
+    /**
+     * @param {ArrayBufferLike} data BYAF ZIP buffer
+     */
+    #data;
+    /**
+     * Creates an instance of ByafParser.
+     * @param {ArrayBufferLike} data BYAF ZIP buffer
+     */
+    constructor(data) {
+        this.#data = data;
+    }
+    /**
+     * Replaces known macros in a string.
+     * @param {string} [str] String to process
+     * @returns {string} String with macros replaced
+     * @private
+     */
+    replaceMacros(str) {
+        return String(str || '')
             .replace(/#{user}:/gi, '{{user}}:')
             .replace(/#{character}:/gi, '{{char}}:')
             .replace(/{character}(?!})/gi, '{{char}}')
             .replace(/{user}(?!})/gi, '{{user}}');
+    }
-export const formatByafExampleMessages = (examples) => {
+    /**
+     * Formats example messages for a character.
+     * @param {ByafExampleMessage[]} [examples] Array of example objects
+     * @returns {string} Formatted example messages
+     * @private
+     */
+    formatExampleMessages(examples) {
         if (!Array.isArray(examples)) {
             return '';
+        }

             if (!example?.text) {
                 return;
+            }
-        formattedExamples += `<START>\n${replaceByafMacros(example.text)}\n`;
+            formattedExamples += `<START>\n${this.replaceMacros(example.text)}\n`;
         });
         return formattedExamples.trimEnd();
-};
+    }
-export const formatByafAlternateGreetings = (greetings) => {
+    /**
+     * Formats alternate greetings for a character.
+     * @param {ByafExampleMessage[]} [greetings] Array of greeting objects
+     * @returns {string[]} Formatted alternate greetings
+     * @private
+     */
+    formatAlternateGreetings(greetings) {
         if (!Array.isArray(greetings)) {
             return [];
+        }

+        }
         // Skip one because it goes into 'first_mes'
-    return greetings.slice(1).map(g => replaceByafMacros(g?.text));
+        return greetings.slice(1).map(g => this.replaceMacros(g?.text));
-};
+    }
-export const convertByafCharacterBook = (items) => {
+    /**
+     * Converts character book items to a structured format.
+     * @param {ByafLoreItem[]} items Array of key-value pairs
+     * @returns {CharacterBook|undefined} Converted character book or undefined if invalid
+     * @private
+     */
+    convertCharacterBook(items) {
         if (!Array.isArray(items) || items.length === 0) {
-        return null;
+            return undefined;
+        }
+        /** @type {CharacterBook} */
         const book = {
-        /** @type {any[]} */
             entries: [],
+            extensions: {},
         };
         items.forEach((item, index) => {

                 return;
+            }
             book.entries.push({
-            keys: replaceByafMacros(item?.key).split(',').map(key => key.trim()).filter(Boolean),
+                keys: this.replaceMacros(item?.key).split(',').map(key => key.trim()).filter(Boolean),
-            content: replaceByafMacros(item?.value),
+                content: this.replaceMacros(item?.value),
                 extensions: {},
                 enabled: true,
                 insertion_order: index,

         });
         return book;
-};
+    }
     /**
      * Extracts a character object from BYAF buffer.
- * @param {ArrayBufferLike} data ZIP buffer
+     * @param {ByafManifest} manifest BYAF manifest
- * @param {object} manifest BYAF manifest
+     * @returns {Promise<{character:ByafCharacter,characterPath:string}>} Character object
- * @returns {Promise<{character:object,characterPath:string}>} Character object
+     * @private
      */
-export async function getCharacterFromByafManifest(data, manifest) {
+    async getCharacterFromManifest(manifest) {
         const charactersArray = manifest?.characters;
         if (!Array.isArray(charactersArray)) {

@@ -94,7 +137,7 @@ export async function getCharacterFromByafManifest(data, manifest) {
94	throw new Error('Invalid BYAF file: missing character path');	137	throw new Error('Invalid BYAF file: missing character path');
95	}	138	}
96		139
97	const characterBuffer = await extractFileFromZipBuffer(data, characterPath);	140	const characterBuffer = await extractFileFromZipBuffer(this.#data, characterPath);
98	if (!characterBuffer) {	141	if (!characterBuffer) {
99	throw new Error('Invalid BYAF file: failed to extract character JSON');	142	throw new Error('Invalid BYAF file: failed to extract character JSON');
100	}	143	}

     /**
      * Extracts a scenario object from BYAF buffer.
- * @param {ArrayBufferLike} data ZIP buffer
+     * @param {ByafManifest} manifest BYAF manifest
- * @param {object} manifest BYAF manifest
+     * @returns {Promise<Partial<ByafScenario>>} Scenario object
- * @returns {Promise<object>} Scenario object
+     * @private
      */
-export async function getScenarioFromByafManifest(data, manifest) {
+    async getScenarioFromManifest(manifest) {
         const scenariosArray = manifest?.scenarios;
         if (!Array.isArray(scenariosArray) || scenariosArray.length === 0) {

             return {};
+        }
-    const scenarioBuffer = await extractFileFromZipBuffer(data, scenarioPath);
+        const scenarioBuffer = await extractFileFromZipBuffer(this.#data, scenarioPath);
         if (!scenarioBuffer) {
             console.warn('Warning: failed to extract BYAF scenario JSON');
             return {};

     /**
      * Extracts an image from BYAF buffer.
- * @param {ArrayBufferLike} data ZIP buffer
+     * @param {ByafCharacter} character Character object
- * @param {object} character Character object
      * @param {string} characterPath Path to the character in the BYAF manifest
+     * @return {Promise<Buffer>} Image buffer
+     * @private
      */
-export async function getImageBufferFromByafCharacter(data, character, characterPath) {
+    async getCharacterImage(character, characterPath) {
         const defaultAvatarBuffer = await fsPromises.readFile(DEFAULT_AVATAR_PATH);
         const characterImages = character?.images;

+        }
         const fullImagePath = urlJoin(path.dirname(characterPath), imagePath);
-    const imageBuffer = await extractFileFromZipBuffer(data, fullImagePath);
+        const imageBuffer = await extractFileFromZipBuffer(this.#data, fullImagePath);
         if (!imageBuffer) {
             console.warn('Warning: failed to extract BYAF character image');
             return defaultAvatarBuffer;

     /**
      * Formats BYAF data as a character card.
- * @param {object} character
+     * @param {ByafManifest} manifest BYAF manifest
- * @param {object} scenario
+     * @param {ByafCharacter} character Character object
+     * @param {Partial<ByafScenario>} scenario Scenario object
+     * @return {TavernCardV2} Character card object
+     * @private
      */
-export function formatByafAsCharacterCard(character, scenario) {
+    getCharacterCard(manifest, character, scenario) {
         return {
             spec: 'chara_card_v2',
             spec_version: '2.0',
-        create_date: humanizedISO8601DateTime(),
             data: {
                 name: sanitize(character?.name || character?.displayName || ''),
-            description: replaceByafMacros(character?.persona),
+                description: this.replaceMacros(character?.persona),
                 personality: '',
-            scenario: replaceByafMacros(scenario?.narrative),
+                scenario: this.replaceMacros(scenario?.narrative),
-            first_mes: replaceByafMacros(scenario?.firstMessages?.[0]?.text),
+                first_mes: this.replaceMacros(scenario?.firstMessages?.[0]?.text),
-            mes_example: formatByafExampleMessages(scenario?.exampleMessages),
+                mes_example: this.formatExampleMessages(scenario?.exampleMessages),
                 creator_notes: '',
-            system_prompt: replaceByafMacros(scenario?.formattingInstructions),
+                system_prompt: this.replaceMacros(scenario?.formattingInstructions),
                 post_history_instructions: '',
-            alternate_greetings: formatByafAlternateGreetings(scenario?.firstMessages),
+                alternate_greetings: this.formatAlternateGreetings(scenario?.firstMessages),
-            character_book: convertByafCharacterBook(character?.loreItems),
+                character_book: this.convertCharacterBook(character?.loreItems),
                 tags: [],
-            creator: '',
+                creator: manifest?.author?.name || '',
                 character_version: '',
                 extensions: {},
             },
+            // @ts-ignore Non-standard spec extension
+            create_date: humanizedISO8601DateTime(),
         };
+    }
+    /**
+     * Gets the manifest from the BYAF data.
+     * @returns {Promise<ByafManifest>} Parsed manifest
+     * @private
+     */
+    async getManifest() {
+        const manifestBuffer = await extractFileFromZipBuffer(this.#data, 'manifest.json');
+        if (!manifestBuffer) {
+            throw new Error('Failed to extract manifest.json from BYAF file');
+        }
+        const manifest = JSON.parse(manifestBuffer.toString());
+        if (!manifest || typeof manifest !== 'object') {
+            throw new Error('Invalid BYAF manifest');
+        }
+        return manifest;
+    }
+    /**
+     * Parses the BYAF data.
+     * @return {Promise<{card: TavernCardV2, image: Buffer}>} Parsed character card and image buffer
+     */
+    async parse() {
+        const manifest = await this.getManifest();
+        const { character, characterPath } = await this.getCharacterFromManifest(manifest);
+        const scenario = await this.getScenarioFromManifest(manifest);
+        const image = await this.getCharacterImage(character, characterPath);
+        const card = this.getCharacterCard(manifest, character, scenario);
+        return { card, image };
+    }
+}
+export default ByafParser;

src/endpoints/characters.js+4 -13 Viewed

 import { importRisuSprites } from './sprites.js';
 import { getUserDirectories } from '../users.js';
 import { getChatInfo } from './chats.js';
-import { formatByafAsCharacterCard, getCharacterFromByafManifest, getImageBufferFromByafCharacter, getScenarioFromByafManifest } from '../byaf.js';
+import { ByafParser } from '../byaf.js';
 import cacheBuster from '../middleware/cacheBuster.js';
 // With 100 MB limit it would take roughly 3000 characters to reach this limit

     await fsPromises.unlink(uploadPath);
     console.info('Importing from BYAF');
-    const manifestBuffer = await extractFileFromZipBuffer(data, 'manifest.json');
+    const byafData = await new ByafParser(data).parse();
-    if (!manifestBuffer) {
+    const card = readFromV2(byafData.card);
-        throw new Error('Failed to extract manifest.json from BYAF file');
-    }
-    const manifest = JSON.parse(manifestBuffer.toString());
-    const { character, characterPath } = await getCharacterFromByafManifest(data, manifest);
-    const scenario = await getScenarioFromByafManifest(data, manifest);
-    const image = await getImageBufferFromByafCharacter(data, character, characterPath);
-    const card = readFromV2(formatByafAsCharacterCard(character, scenario));
     const fileName = preservedFileName || getPngName(card.name, request.user.directories);
-    const result = await writeCharacterData(image, JSON.stringify(card), fileName, request);
+    const result = await writeCharacterData(byafData.image, JSON.stringify(card), fileName, request);
     return result ? fileName : '';
+}

src/types/byaf.d.ts+77 -0 Viewed

+type ByafLoreItem = {
+    key: string;
+    value: string;
+};
+type ByafCharacterImage = {
+    path: string;
+    label: string;
+};
+type ByafExampleMessage = {
+    characterID: string;
+    text: string;
+};
+type ByafCharacter = {
+    schemaVersion: 1;
+    id: string;
+    name: string;
+    displayName: string;
+    isNSFW: boolean;
+    persona: string;
+    createdAt: string;
+    updatedAt: string;
+    loreItems: Array<ByafLoreItem>;
+    images: Array<ByafCharacterImage>;
+};
+type ByafManifest = {
+    schemaVersion: 1;
+    createdAt: string;
+    characters: string[];
+    scenarios: string[];
+    author?: {
+        name: string;
+        backyardURL: string;
+    };
+};
+type ByafAiMessage = {
+    type: "ai";
+    outputs: Array<{
+        createdAt: string;
+        updatedAt: string;
+        text: string;
+        activeTimestamp: string;
+    }>;
+};
+type ByafHumanMessage = {
+    type: "human";
+    createdAt: string;
+    updatedAt: string;
+    text: string;
+};
+type ByafScenario = {
+    schemaVersion: 1;
+    title?: string;
+    model?: string;
+    formattingInstructions: string;
+    minP: number;
+    minPEnabled: boolean;
+    temperature: number;
+    repeatPenalty: number;
+    repeatLastN: number;
+    topK: number;
+    topP: number;
+    exampleMessages: Array<ByafExampleMessage>;
+    canDeleteExampleMessages: boolean;
+    firstMessages: Array<ByafExampleMessage>;
+    narrative: string;
+    promptTemplate: "general" | "ChatML" | "Llama3" | "Gemma2" | "CommandR" | "MistralInstruct" | null;
+    grammar: string | null;
+    messages: Array<ByafAiMessage | ByafHumanMessage>;
+    backgroundImage?: string;
+};

src/types/spec-v2.d.ts+52 -0 Viewed

+type TavernCardV2 = {
+    spec: 'chara_card_v2';
+    spec_version: '2.0';
+    data: {
+        name: string;
+        description: string;
+        personality: string;
+        scenario: string;
+        first_mes: string;
+        mes_example: string;
+        creator_notes: string;
+        system_prompt: string;
+        post_history_instructions: string;
+        alternate_greetings: Array<string>;
+        character_book?: CharacterBook;
+        tags: Array<string>;
+        creator: string;
+        character_version: string;
+        extensions: Record<string, any>;
+    }
+}
+type CharacterBook = {
+    name?: string;
+    description?: string;
+    scan_depth?: number;
+    token_budget?: number;
+    recursive_scanning?: boolean;
+    extensions: Record<string, any>;
+    entries: Array<CharacterBookEntry>;
+}
+type CharacterBookEntry = {
+    keys: Array<string>;
+    content: string;
+    extensions: Record<string, any>;
+    enabled: boolean;
+    insertion_order: number;
+    case_sensitive?: boolean;
+    name?: string;
+    priority?: number;
+    id?: number;
+    comment?: string;
+    selective?: boolean;
+    secondary_keys?: Array<string>;
+    constant?: boolean;
+    position?: 'before_char' | 'after_char';
+};