Refactor BYAF parsing (#4303) * Refactor BYAF parsing * Refactor ByafParser to use private field for data and improve type annotations

38a5e4a4f2a0068510124423f9e4c9913116e1f2

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

Parents: 7edd81f183014568c47831d2c02dc926a80d0eda

src/byaf.js+124 -40 Viewed

 import { DEFAULT_AVATAR_PATH } from './constants.js';
 import { extractFileFromZipBuffer, humanizedISO8601DateTime } from './util.js';
 -export const replaceByafMacros = (s) =>
 +/**
 -    String(s || '')
 + * A parser for BYAF (Backyard Archive Format) files.
 + */
 +export class ByafParser {
 +    /**
 +     * @param {ArrayBufferLike} data BYAF ZIP buffer
 +     */
 +    #data;
++
 +    /**
 +     * Creates an instance of ByafParser.
 +     * @param {ArrayBufferLike} data BYAF ZIP buffer
 +     */
 +    constructor(data) {
 +        this.#data = data;
 +    }
++
 +    /**
 +     * Replaces known macros in a string.
 +     * @param {string} [str] String to process
 +     * @returns {string} String with macros replaced
 +     * @private
 +     */
 +    replaceMacros(str) {
 +        return String(str || '')
             .replace(/#{user}:/gi, '{{user}}:')
             .replace(/#{character}:/gi, '{{char}}:')
             .replace(/{character}(?!})/gi, '{{char}}')
             .replace(/{user}(?!})/gi, '{{user}}');
 +    }
 -export const formatByafExampleMessages = (examples) => {
 +    /**
 +     * Formats example messages for a character.
 +     * @param {ByafExampleMessage[]} [examples] Array of example objects
 +     * @returns {string} Formatted example messages
 +     * @private
 +     */
 +    formatExampleMessages(examples) {
         if (!Array.isArray(examples)) {
             return '';
+        }

             if (!example?.text) {
                 return;
+            }
                     formattedExamples += `<START>\n${replaceByafMacrosthis.replaceMacros(example.text)}\n`;
         });
         return formattedExamples.trimEnd();
     };
 -export const formatByafAlternateGreetings = (greetings) => {
 +    /**
 +     * Formats alternate greetings for a character.
 +     * @param {ByafExampleMessage[]} [greetings] Array of greeting objects
 +     * @returns {string[]} Formatted alternate greetings
 +     * @private
 +     */
 +    formatAlternateGreetings(greetings) {
         if (!Array.isArray(greetings)) {
             return [];
+        }

+        }
         // Skip one because it goes into 'first_mes'
             return greetings.slice(1).map(g => replaceByafMacrosthis.replaceMacros(g?.text));
     };
 -export const convertByafCharacterBook = (items) => {
 +    /**
 +     * Converts character book items to a structured format.
 +     * @param {ByafLoreItem[]} items Array of key-value pairs
 +     * @returns {CharacterBook|undefined} Converted character book or undefined if invalid
 +     * @private
 +     */
 +    convertCharacterBook(items) {
         if (!Array.isArray(items) || items.length === 0) {
                     return nullundefined;
+        }
 +        /** @type {CharacterBook} */
         const book = {
 -        /** @type {any[]} */
             entries: [],
 +            extensions: {},
         };
         items.forEach((item, index) => {

                 return;
+            }
             book.entries.push({
                             keys: replaceByafMacrosthis.replaceMacros(item?.key).split(',').map(key => key.trim()).filter(Boolean),
                             content: replaceByafMacrosthis.replaceMacros(item?.value),
                 extensions: {},
                 enabled: true,
                 insertion_order: index,

         });
         return book;
     };
     /**
      * Extracts a character object from BYAF buffer.
       * @param {ArrayBufferLikeByafManifest} datamanifest ZIPBYAF buffermanifest
 - * @param {object} manifest BYAF manifest
 +     * @returns {Promise<{character:ByafCharacter,characterPath:string}>} Character object
 - * @returns {Promise<{character:object,characterPath:string}>} Character object
 +     * @private
      */
 export     async function getCharacterFromByafManifestgetCharacterFromManifest(data, manifest) {
         const charactersArray = manifest?.characters;
         if (!Array.isArray(charactersArray)) {

@@ -94,7 +137,7 @@ export async function getCharacterFromByafManifest(data, manifest) {
94	137	throw new Error('Invalid BYAF file: missing character path');
95	138	}
96	139
97	140	const characterBuffer = await extractFileFromZipBuffer(this.#data, characterPath);
98	141	if (!characterBuffer) {
99	142	throw new Error('Invalid BYAF file: failed to extract character JSON');
100	143	}

     /**
      * Extracts a scenario object from BYAF buffer.
       * @param {ArrayBufferLikeByafManifest} datamanifest ZIPBYAF buffermanifest
 - * @param {object} manifest BYAF manifest
 +     * @returns {Promise<Partial<ByafScenario>>} Scenario object
 - * @returns {Promise<object>} Scenario object
 +     * @private
      */
 export     async function getScenarioFromByafManifestgetScenarioFromManifest(data, manifest) {
         const scenariosArray = manifest?.scenarios;
         if (!Array.isArray(scenariosArray) || scenariosArray.length === 0) {

             return {};
+        }
             const scenarioBuffer = await extractFileFromZipBuffer(this.#data, scenarioPath);
         if (!scenarioBuffer) {
             console.warn('Warning: failed to extract BYAF scenario JSON');
             return {};

     /**
      * Extracts an image from BYAF buffer.
       * @param {ArrayBufferLikeByafCharacter} datacharacter ZIPCharacter bufferobject
 - * @param {object} character Character object
      * @param {string} characterPath Path to the character in the BYAF manifest
 +     * @return {Promise<Buffer>} Image buffer
 +     * @private
      */
 export     async function getImageBufferFromByafCharactergetCharacterImage(data, character, characterPath) {
         const defaultAvatarBuffer = await fsPromises.readFile(DEFAULT_AVATAR_PATH);
         const characterImages = character?.images;

+        }
         const fullImagePath = urlJoin(path.dirname(characterPath), imagePath);
             const imageBuffer = await extractFileFromZipBuffer(this.#data, fullImagePath);
         if (!imageBuffer) {
             console.warn('Warning: failed to extract BYAF character image');
             return defaultAvatarBuffer;

     /**
      * Formats BYAF data as a character card.
       * @param {objectByafManifest} charactermanifest BYAF manifest
       * @param {objectByafCharacter} scenariocharacter Character object
 +     * @param {Partial<ByafScenario>} scenario Scenario object
 +     * @return {TavernCardV2} Character card object
 +     * @private
      */
 export function formatByafAsCharacterCard    getCharacterCard(manifest, character, scenario) {
         return {
             spec: 'chara_card_v2',
             spec_version: '2.0',
 -        create_date: humanizedISO8601DateTime(),
             data: {
                 name: sanitize(character?.name || character?.displayName || ''),
                             description: replaceByafMacrosthis.replaceMacros(character?.persona),
                 personality: '',
                             scenario: replaceByafMacrosthis.replaceMacros(scenario?.narrative),
                             first_mes: replaceByafMacrosthis.replaceMacros(scenario?.firstMessages?.[0]?.text),
                             mes_example: formatByafExampleMessagesthis.formatExampleMessages(scenario?.exampleMessages),
                 creator_notes: '',
                             system_prompt: replaceByafMacrosthis.replaceMacros(scenario?.formattingInstructions),
                 post_history_instructions: '',
                             alternate_greetings: formatByafAlternateGreetingsthis.formatAlternateGreetings(scenario?.firstMessages),
                             character_book: convertByafCharacterBookthis.convertCharacterBook(character?.loreItems),
                 tags: [],
 -            creator: '',
 +                creator: manifest?.author?.name || '',
                 character_version: '',
                 extensions: {},
             },
 +            // @ts-ignore Non-standard spec extension
 +            create_date: humanizedISO8601DateTime(),
         };
+    }
++
 +    /**
 +     * Gets the manifest from the BYAF data.
 +     * @returns {Promise<ByafManifest>} Parsed manifest
 +     * @private
 +     */
 +    async getManifest() {
 +        const manifestBuffer = await extractFileFromZipBuffer(this.#data, 'manifest.json');
 +        if (!manifestBuffer) {
 +            throw new Error('Failed to extract manifest.json from BYAF file');
 +        }
++
 +        const manifest = JSON.parse(manifestBuffer.toString());
 +        if (!manifest || typeof manifest !== 'object') {
 +            throw new Error('Invalid BYAF manifest');
 +        }
++
 +        return manifest;
 +    }
++
 +    /**
 +     * Parses the BYAF data.
 +     * @return {Promise<{card: TavernCardV2, image: Buffer}>} Parsed character card and image buffer
 +     */
 +    async parse() {
 +        const manifest = await this.getManifest();
 +        const { character, characterPath } = await this.getCharacterFromManifest(manifest);
 +        const scenario = await this.getScenarioFromManifest(manifest);
 +        const image = await this.getCharacterImage(character, characterPath);
 +        const card = this.getCharacterCard(manifest, character, scenario);
++
 +        return { card, image };
 +    }
 +}
++
 +export default ByafParser;

src/endpoints/characters.js+4 -13 Viewed

 import { importRisuSprites } from './sprites.js';
 import { getUserDirectories } from '../users.js';
 import { getChatInfo } from './chats.js';
 import { formatByafAsCharacterCard, getCharacterFromByafManifest, getImageBufferFromByafCharacter, getScenarioFromByafManifestByafParser } from '../byaf.js';
 import cacheBuster from '../middleware/cacheBuster.js';
 // With 100 MB limit it would take roughly 3000 characters to reach this limit

     await fsPromises.unlink(uploadPath);
     console.info('Importing from BYAF');
     const manifestBufferbyafData = await extractFileFromZipBuffernew ByafParser(data, 'manifest).json'parse();
 -    if (!manifestBuffer) {
 +    const card = readFromV2(byafData.card);
 -        throw new Error('Failed to extract manifest.json from BYAF file');
 -    }
+-
 -    const manifest = JSON.parse(manifestBuffer.toString());
 -    const { character, characterPath } = await getCharacterFromByafManifest(data, manifest);
 -    const scenario = await getScenarioFromByafManifest(data, manifest);
 -    const image = await getImageBufferFromByafCharacter(data, character, characterPath);
+-
 -    const card = readFromV2(formatByafAsCharacterCard(character, scenario));
     const fileName = preservedFileName || getPngName(card.name, request.user.directories);
     const result = await writeCharacterData(byafData.image, JSON.stringify(card), fileName, request);
     return result ? fileName : '';
+}

src/types/byaf.d.ts+77 -0 Viewed

 +type ByafLoreItem = {
 +    key: string;
 +    value: string;
 +};
++
 +type ByafCharacterImage = {
 +    path: string;
 +    label: string;
 +};
++
 +type ByafExampleMessage = {
 +    characterID: string;
 +    text: string;
 +};
++
 +type ByafCharacter = {
 +    schemaVersion: 1;
 +    id: string;
 +    name: string;
 +    displayName: string;
 +    isNSFW: boolean;
 +    persona: string;
 +    createdAt: string;
 +    updatedAt: string;
 +    loreItems: Array<ByafLoreItem>;
 +    images: Array<ByafCharacterImage>;
 +};
++
 +type ByafManifest = {
 +    schemaVersion: 1;
 +    createdAt: string;
 +    characters: string[];
 +    scenarios: string[];
 +    author?: {
 +        name: string;
 +        backyardURL: string;
 +    };
 +};
++
 +type ByafAiMessage = {
 +    type: "ai";
 +    outputs: Array<{
 +        createdAt: string;
 +        updatedAt: string;
 +        text: string;
 +        activeTimestamp: string;
 +    }>;
 +};
++
 +type ByafHumanMessage = {
 +    type: "human";
 +    createdAt: string;
 +    updatedAt: string;
 +    text: string;
 +};
++
 +type ByafScenario = {
 +    schemaVersion: 1;
 +    title?: string;
 +    model?: string;
 +    formattingInstructions: string;
 +    minP: number;
 +    minPEnabled: boolean;
 +    temperature: number;
 +    repeatPenalty: number;
 +    repeatLastN: number;
 +    topK: number;
 +    topP: number;
 +    exampleMessages: Array<ByafExampleMessage>;
 +    canDeleteExampleMessages: boolean;
 +    firstMessages: Array<ByafExampleMessage>;
 +    narrative: string;
 +    promptTemplate: "general" | "ChatML" | "Llama3" | "Gemma2" | "CommandR" | "MistralInstruct" | null;
 +    grammar: string | null;
 +    messages: Array<ByafAiMessage | ByafHumanMessage>;
 +    backgroundImage?: string;
 +};

src/types/spec-v2.d.ts+52 -0 Viewed

 +type TavernCardV2 = {
 +    spec: 'chara_card_v2';
 +    spec_version: '2.0';
 +    data: {
 +        name: string;
 +        description: string;
 +        personality: string;
 +        scenario: string;
 +        first_mes: string;
 +        mes_example: string;
++
 +        creator_notes: string;
 +        system_prompt: string;
 +        post_history_instructions: string;
 +        alternate_greetings: Array<string>;
 +        character_book?: CharacterBook;
++
 +        tags: Array<string>;
 +        creator: string;
 +        character_version: string;
 +        extensions: Record<string, any>;
 +    }
 +}
++
 +type CharacterBook = {
 +    name?: string;
 +    description?: string;
 +    scan_depth?: number;
 +    token_budget?: number;
 +    recursive_scanning?: boolean;
 +    extensions: Record<string, any>;
 +    entries: Array<CharacterBookEntry>;
 +}
++
 +type CharacterBookEntry = {
 +    keys: Array<string>;
 +    content: string;
 +    extensions: Record<string, any>;
 +    enabled: boolean;
 +    insertion_order: number;
 +    case_sensitive?: boolean;
++
 +    name?: string;
 +    priority?: number;
++
 +    id?: number;
 +    comment?: string;
 +    selective?: boolean;
 +    secondary_keys?: Array<string>;
 +    constant?: boolean;
 +    position?: 'before_char' | 'after_char';
 +};