1
import { promises as fsPromises } from 'node:fs';
2
import path from 'node:path';
3
import urlJoin from 'url-join';
4
import { DEFAULT_AVATAR_PATH } from './constants.js';
5
import { extractFileFromZipBuffer } from './util.js';
6

7
/**
8
 * A parser for BYAF (Backyard Archive Format) files.
9
 */
10
export class ByafParser {
11
    /**
12
     * @param {ArrayBufferLike} data BYAF ZIP buffer
13
     */
14
    #data;
15

16
    /**
17
     * Creates an instance of ByafParser.
18
     * @param {ArrayBufferLike} data BYAF ZIP buffer
19
     */
20
    constructor(data) {
21
        this.#data = data;
22
    }
23

24
    /**
25
     * Replaces known macros in a string.
26
     * @param {string} [str] String to process
27
     * @returns {string} String with macros replaced
28
     * @private
29
     */
30
    static replaceMacros(str) {
31
        return String(str || '')
32
            .replace(/#{user}:/gi, '{{user}}:')
33
            .replace(/#{character}:/gi, '{{char}}:')
34
            .replace(/{character}(?!})/gi, '{{char}}')
35
            .replace(/{user}(?!})/gi, '{{user}}');
36
    }
37

38
    /**
39
     * Formats example messages for a character.
40
     * @param {ByafExampleMessage[]} [examples] Array of example objects
41
     * @returns {string} Formatted example messages
42
     * @private
43
     */
44
    static formatExampleMessages(examples) {
45
        if (!Array.isArray(examples)) {
46
            return '';
47
        }
48

49
        let formattedExamples = '';
50

51
        examples.forEach((example) => {
52
            if (!example?.text) {
53
                return;
54
            }
55
            formattedExamples += `<START>\n${ByafParser.replaceMacros(example.text)}\n`;
56
        });
57

58
        return formattedExamples.trimEnd();
59
    }
60

61
    /**
62
     * Formats alternate greetings for a character.
63
     * @param {Partial<ByafScenario>[]} [scenarios] Array of scenario objects
64
     * @returns {string[]} Formatted alternate greetings
65
     * @private
66
     */
67
    formatAlternateGreetings(scenarios) {
68
        if (!Array.isArray(scenarios)) {
69
            return [];
70
        }
71

72
        // Skip one because it goes into 'first_mes'
73
        if (scenarios.length <= 1) {
74
            return [];
75
        }
76
        const greetings = new Set();
77
        const firstScenarioFirstMessage = scenarios?.[0]?.firstMessages?.[0]?.text;
78
        for (const scenario of scenarios.slice(1).filter(s => Array.isArray(s.firstMessages) && s.firstMessages.length > 0)) {
79
            // As per the BYAF spec, "firstMessages" array MUST contain AT MOST one message.
80
            // So we only consider the first one if it exists.
81
            const firstMessage = scenario?.firstMessages?.[0];
82
            if (firstMessage?.text && firstMessage.text !== firstScenarioFirstMessage) {
83
                greetings.add(ByafParser.replaceMacros(firstMessage.text));
84
            }
85
        }
86
        return Array.from(greetings);
87
    }
88

89
    /**
90
     * Converts character book items to a structured format.
91
     * @param {ByafLoreItem[]} items Array of key-value pairs
92
     * @returns {CharacterBook|undefined} Converted character book or undefined if invalid
93
     * @private
94
     */
95
    convertCharacterBook(items) {
96
        if (!Array.isArray(items) || items.length === 0) {
97
            return undefined;
98
        }
99

100
        /** @type {CharacterBook} */
101
        const book = {
102
            entries: [],
103
            extensions: {},
104
        };
105

106
        items.forEach((item, index) => {
107
            if (!item) {
108
                return;
109
            }
110
            book.entries.push({
111
                keys: ByafParser.replaceMacros(item?.key).split(',').map(key => key.trim()).filter(Boolean),
112
                content: ByafParser.replaceMacros(item?.value),
113
                extensions: {},
114
                enabled: true,
115
                insertion_order: index,
116
            });
117
        });
118

119
        return book;
120
    }
121

122
    /**
123
     * Extracts a character object from BYAF buffer.
124
     * @param {ByafManifest} manifest BYAF manifest
125
     * @returns {Promise<{character:ByafCharacter,characterPath:string}>} Character object
126
     * @private
127
     */
128
    async getCharacterFromManifest(manifest) {
129
        const charactersArray = manifest?.characters;
130

131
        if (!Array.isArray(charactersArray)) {
132
            throw new Error('Invalid BYAF file: missing characters array');
133
        }
134

135
        if (charactersArray.length === 0) {
136
            throw new Error('Invalid BYAF file: characters array is empty');
137
        }
138

139
        if (charactersArray.length > 1) {
140
            console.warn('Warning: BYAF manifest contains more than one character, only the first one will be imported');
141
        }
142

143
        const characterPath = charactersArray[0];
144
        if (!characterPath) {
145
            throw new Error('Invalid BYAF file: missing character path');
146
        }
147

148
        const characterBuffer = await extractFileFromZipBuffer(this.#data, characterPath);
149
        if (!characterBuffer) {
150
            throw new Error('Invalid BYAF file: failed to extract character JSON');
151
        }
152

153
        try {
154
            const character = JSON.parse(characterBuffer.toString());
155
            return { character, characterPath };
156
        } catch (error) {
157
            console.error('Failed to parse character JSON from BYAF:', error);
158
            throw new Error('Invalid BYAF file: character is not a valid JSON');
159
        }
160
    }
161

162
    /**
163
     * Extracts all scenario objects from BYAF buffer.
164
     * @param {ByafManifest} manifest BYAF manifest
165
     * @returns {Promise<Partial<ByafScenario>[]>} Scenarios array
166
     * @private
167
     */
168
    async getScenariosFromManifest(manifest) {
169
        const scenariosArray = manifest?.scenarios;
170

171
        if (!Array.isArray(scenariosArray) || scenariosArray.length === 0) {
172
            console.warn('Warning: BYAF manifest contains no scenarios');
173
            return [{}];
174
        }
175

176
        const scenarios = [];
177

178
        for (const scenarioPath of scenariosArray) {
179
            const scenarioBuffer = await extractFileFromZipBuffer(this.#data, scenarioPath);
180
            if (!scenarioBuffer) {
181
                console.warn('Warning: failed to extract BYAF scenario JSON');
182
            }
183
            if (scenarioBuffer) {
184
                try {
185
                    scenarios.push(JSON.parse(scenarioBuffer.toString()));
186
                } catch (error) {
187
                    console.warn('Warning: BYAF scenario is not a valid JSON', error);
188
                }
189
            }
190
        }
191

192
        if (scenarios.length === 0) {
193
            console.warn('Warning: BYAF manifest contains no valid scenarios');
194
            return [{}];
195
        }
196

197
        return scenarios;
198
    }
199

200
    /**
201
     * Extracts all character icon images from BYAF buffer.
202
     * @param {ByafCharacter} character Character object
203
     * @param {string} characterPath Path to the character in the BYAF manifest
204
     * @return {Promise<{filename: string, image: Buffer, label: string}[]>} Image buffer
205
     * @private
206
     */
207
    async getCharacterImages(character, characterPath) {
208
        const defaultAvatarBuffer = await fsPromises.readFile(DEFAULT_AVATAR_PATH);
209
        const characterImages = character?.images;
210

211
        if (!Array.isArray(characterImages) || characterImages.length === 0) {
212
            console.warn('Warning: BYAF character has no images');
213
            return [{ filename: '', image: defaultAvatarBuffer, label: '' }];
214
        }
215

216
        const imageBuffers = [];
217
        for (const image of characterImages) {
218
            const imagePath = image?.path;
219
            if (!imagePath) {
220
                console.warn('Warning: BYAF character image path is empty');
221
                continue;
222
            }
223

224
            const fullImagePath = urlJoin(path.dirname(characterPath), imagePath);
225
            const imageBuffer = await extractFileFromZipBuffer(this.#data, fullImagePath);
226
            if (!imageBuffer) {
227
                console.warn('Warning: failed to extract BYAF character image');
228
                continue;
229
            }
230

231
            imageBuffers.push({ filename: path.basename(imagePath), image: imageBuffer, label: image?.label || '' });
232
        }
233
        if (imageBuffers.length === 0) {
234
            console.warn('Warning: BYAF character has no valid images');
235
            return [{ filename: '', image: defaultAvatarBuffer, label: '' }];
236
        }
237
        return imageBuffers;
238
    }
239

240
    /**
241
     * Formats BYAF data as a character card.
242
     * @param {ByafManifest} manifest BYAF manifest
243
     * @param {ByafCharacter} character Character object
244
     * @param {Partial<ByafScenario>[]} scenarios Scenarios array
245
     * @return {TavernCardV2} Character card object
246
     * @private
247
     */
248
    getCharacterCard(manifest, character, scenarios) {
249
        return {
250
            spec: 'chara_card_v2',
251
            spec_version: '2.0',
252
            data: {
253
                name: character?.name || character?.displayName || '',
254
                description: ByafParser.replaceMacros(character?.persona),
255
                personality: '',
256
                scenario: ByafParser.replaceMacros(scenarios[0]?.narrative),
257
                first_mes: ByafParser.replaceMacros(scenarios[0]?.firstMessages?.[0]?.text),
258
                mes_example: ByafParser.formatExampleMessages(scenarios[0]?.exampleMessages),
259
                creator_notes: manifest?.author?.backyardURL || '', // To preserve the link to the author from BYAF manifest, this is a good place.
260
                system_prompt: ByafParser.replaceMacros(scenarios[0]?.formattingInstructions),
261
                post_history_instructions: '',
262
                alternate_greetings: this.formatAlternateGreetings(scenarios),
263
                character_book: this.convertCharacterBook(character?.loreItems),
264
                tags: character?.isNSFW ? ['nsfw'] : [], // Since there are no tags in BYAF spec, we can use this to preserve the isNSFW flag.
265
                creator: manifest?.author?.name || '',
266
                character_version: '',
267
                extensions: { ...(character?.displayName && { 'display_name': character?.displayName }) }, // Preserve display name unmodified using extensions. "display_name" is not used by SillyTavern currently.
268
            },
269
            // @ts-ignore Non-standard spec extension
270
            create_date: new Date().toISOString(),
271
        };
272
    }
273
    /**
274
     * Gets chat backgrounds from BYAF data mapped to their respective scenarios.
275
     * @param {ByafCharacter} character Character object
276
     * @param {Partial<ByafScenario>[]} scenarios Scenarios array
277
     * @returns {Promise<Array<ByafChatBackground>>} Chat backgrounds
278
     * @private
279
     */
280
    async getChatBackgrounds(character, scenarios) {
281
        // Implementation for extracting chat backgrounds from BYAF data
282
        const backgrounds = [];
283
        let i = 1;
284
        for (const scenario of scenarios) {
285
            const bgImagePath = scenario?.backgroundImage;
286
            if (bgImagePath) {
287
                const data = await extractFileFromZipBuffer(this.#data, bgImagePath);
288
                if (data) {
289
                    const existingIndex = backgrounds.findIndex(bg => bg.data.compare(data) === 0);
290
                    if (existingIndex !== -1) {
291
                        backgrounds[existingIndex].paths.push(bgImagePath);
292
                        continue; // Skip adding a new background since it already exists
293
                    }
294
                    backgrounds.push({
295
                        name: `${character?.name} bg ${i++}` || '',
296
                        data: data,
297
                        paths: [bgImagePath],
298
                    });
299
                }
300
            }
301
        }
302
        return backgrounds;
303
    }
304

305
    /**
306
     * Gets the manifest from the BYAF data.
307
     * @returns {Promise<ByafManifest>} Parsed manifest
308
     * @private
309
     */
310
    async getManifest() {
311
        const manifestBuffer = await extractFileFromZipBuffer(this.#data, 'manifest.json');
312
        if (!manifestBuffer) {
313
            throw new Error('Failed to extract manifest.json from BYAF file');
314
        }
315

316
        const manifest = JSON.parse(manifestBuffer.toString());
317
        if (!manifest || typeof manifest !== 'object') {
318
            throw new Error('Invalid BYAF manifest');
319
        }
320

321
        return manifest;
322
    }
323

324
    /**
325
     * Imports a chat from BYAF format.
326
     * @param {Partial<ByafScenario>} scenario Scenario object
327
     * @param {string} userName User name
328
     * @param {string} characterName Character name
329
     * @param {Array<ByafChatBackground>} chatBackgrounds Chat backgrounds
330
     * @returns {string} Chat data
331
     */
332
    static getChatFromScenario(scenario, userName, characterName, chatBackgrounds) {
333
        const chatStartDate = scenario?.messages?.length == 0 ? new Date().toISOString() : scenario?.messages?.filter(m => 'createdAt' in m)[0].createdAt;
334
        const chatBackground = chatBackgrounds.find(bg => bg.paths.includes(scenario?.backgroundImage || ''))?.name || '';
335
        /** @type {object[]} */
336
        const chat = [{
337
            user_name: 'unused',
338
            character_name: 'unused',
339
            chat_metadata: {
340
                scenario: scenario?.narrative ?? '',
341
                mes_example: ByafParser.formatExampleMessages(scenario?.exampleMessages),
342
                system_prompt: ByafParser.replaceMacros(scenario?.formattingInstructions),
343
                mes_examples_optional: scenario?.canDeleteExampleMessages ?? false,
344
                byaf_model_settings: {
345
                    model: scenario?.model ?? '',
346
                    temperature: scenario?.temperature ?? 1.2,
347
                    top_k: scenario?.topK ?? 40,
348
                    top_p: scenario?.topP ?? 0.9,
349
                    min_p: scenario?.minP ?? 0.1,
350
                    min_p_enabled: scenario?.minPEnabled ?? true,
351
                    repeat_penalty: scenario?.repeatPenalty ?? 1.05,
352
                    repeat_penalty_tokens: scenario?.repeatLastN ?? 256,
353
                    by_prompt_template: scenario?.promptTemplate ?? 'general',
354
                    grammar: scenario?.grammar ?? null,
355
                },
356
                chat_backgrounds: chatBackground ? [chatBackground] : [],
357
                custom_background: chatBackground ? `url("${encodeURI(chatBackground)}")` : '',
358
            },
359
        }];
360
        // Add the first message IF it exists.
361
        if (scenario?.firstMessages?.length && scenario?.firstMessages?.length > 0 && scenario?.firstMessages?.[0]?.text) {
362
            chat.push({
363
                name: characterName,
364
                is_user: false,
365
                send_date: chatStartDate,
366
                mes: scenario?.firstMessages?.[0]?.text || '',
367
            });
368
        }
369

370
        const sortByTimestamp = (newest, curr) => {
371
            const aTime = new Date(newest.activeTimestamp);
372
            const bTime = new Date(curr.activeTimestamp);
373
            return aTime >= bTime ? newest : curr;
374
        };
375

376
        const getNewestAiMessage = (message) => {
377
            return message.outputs.reduce(sortByTimestamp);
378
        };
379
        const getSwipesForAiMessage = (aiMessage) => {
380
            return aiMessage.outputs.map(output => output.text);
381
        };
382

383
        const userMessages = scenario?.messages?.filter(msg => msg.type === 'human');
384
        const characterMessages = scenario?.messages?.filter(msg => msg.type === 'ai');
385
        /**
386
         * Reorders messages by interleaving user and character messages so that they are in correct chronological order.
387
         * This is only needed to import old chats from Backyard AI that were incorrectly imported by an earlier version
388
         * that completely messed up the order of messages. Backyard AI Windows frontend never supported creation of chats
389
         * with which were ordered like this in the first place, so for most users this is desired functionality.
390
         */
391
        if (userMessages && characterMessages && userMessages.length === characterMessages.length) { // Only do the reordering if there are equal numbers of user and character messages, otherwise just import in existing order, because it's probably correct already.
392
            for (let i = 0; i < userMessages.length; i++) {
393
                chat.push({
394
                    name: userName,
395
                    is_user: true,
396
                    send_date: Number(userMessages[i]?.createdAt),
397
                    mes: userMessages[i]?.text,
398
                });
399
                const aiMessage = getNewestAiMessage(characterMessages[i]);
400
                const aiSwipes = getSwipesForAiMessage(characterMessages[i]);
401
                chat.push({
402
                    name: characterName,
403
                    is_user: false,
404
                    send_date: Number(aiMessage.createdAt),
405
                    mes: aiMessage.text,
406
                    swipes: aiSwipes,
407
                    swipe_id: aiSwipes.findIndex(s => s === aiMessage.text),
408
                });
409
            }
410
        } else if (scenario?.messages) {
411
            for (const message of scenario.messages) {
412
                const isUser = message.type === 'human';
413
                const aiMessage = !isUser ? getNewestAiMessage(message) : null;
414
                const chatMessage = {
415
                    name: isUser ? userName : characterName,
416
                    is_user: isUser,
417
                    send_date: Number(isUser ? message.createdAt : aiMessage.createdAt),
418
                    mes: isUser ? message.text : aiMessage.text,
419
                };
420
                if (!isUser) {
421
                    const aiSwipes = getSwipesForAiMessage(message);
422
                    chatMessage.swipes = aiSwipes;
423
                    chatMessage.swipe_id = aiSwipes.findIndex(s => s === aiMessage.text);
424
                }
425
                chat.push(chatMessage);
426
            }
427
        } else {
428
            console.warn('Warning: BYAF scenario contained no messages property.');
429
        }
430

431
        return chat.map(obj => JSON.stringify(obj)).join('\n');
432
    }
433

434
    /**
435
     * Parses the BYAF data.
436
     * @return {Promise<ByafParseResult>} Parsed character card and image buffer
437
     */
438
    async parse() {
439
        const manifest = await this.getManifest();
440
        const { character, characterPath } = await this.getCharacterFromManifest(manifest);
441
        const scenarios = await this.getScenariosFromManifest(manifest);
442
        const images = await this.getCharacterImages(character, characterPath);
443
        const card = this.getCharacterCard(manifest, character, scenarios);
444
        const chatBackgrounds = await this.getChatBackgrounds(character, scenarios);
445
        return { card, images, scenarios, chatBackgrounds, character };
446
    }
447
}
448

449
export default ByafParser;