Refactor BYAF parsing (#4303) * Refactor BYAF parsing * Refactor ByafParser to use private field for data and improve type annotations

38a5e4a4f2a0068510124423f9e4c9913116e1f2

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: 7edd81f183014568c47831d2c02dc926a80d0eda
4 files changed, +377 -173Ignore whitespace
src/byaf.js+244 -160
@@ -5,204 +5,288 @@ import urlJoin from 'url-join';
5import { DEFAULT_AVATAR_PATH } from './constants.js';5import { DEFAULT_AVATAR_PATH } from './constants.js';
6import { extractFileFromZipBuffer, humanizedISO8601DateTime } from './util.js';6import { extractFileFromZipBuffer, humanizedISO8601DateTime } from './util.js';
77
8export const replaceByafMacros = (s) =>8/**
9 String(s || '')9 * A parser for BYAF (Backyard Archive Format) files.
10 .replace(/#{user}:/gi, '{{user}}:')10 */
11 .replace(/#{character}:/gi, '{{char}}:')11export class ByafParser {
12 .replace(/{character}(?!})/gi, '{{char}}')12 /**
13 .replace(/{user}(?!})/gi, '{{user}}');13 * @param {ArrayBufferLike} data BYAF ZIP buffer
1414 */
15export const formatByafExampleMessages = (examples) => {15 #data;
16 if (!Array.isArray(examples)) {16
17 return '';17 /**
18 * Creates an instance of ByafParser.
19 * @param {ArrayBufferLike} data BYAF ZIP buffer
20 */
21 constructor(data) {
22 this.#data = data;
18 }23 }
1924
20 let formattedExamples = '';25 /**
26 * Replaces known macros in a string.
27 * @param {string} [str] String to process
28 * @returns {string} String with macros replaced
29 * @private
30 */
31 replaceMacros(str) {
32 return String(str || '')
33 .replace(/#{user}:/gi, '{{user}}:')
34 .replace(/#{character}:/gi, '{{char}}:')
35 .replace(/{character}(?!})/gi, '{{char}}')
36 .replace(/{user}(?!})/gi, '{{user}}');
37 }
2138
22 examples.forEach((example) => {39 /**
23 if (!example?.text) {40 * Formats example messages for a character.
24 return;41 * @param {ByafExampleMessage[]} [examples] Array of example objects
42 * @returns {string} Formatted example messages
43 * @private
44 */
45 formatExampleMessages(examples) {
46 if (!Array.isArray(examples)) {
47 return '';
25 }48 }
26 formattedExamples += `<START>\n${replaceByafMacros(example.text)}\n`;
27 });
2849
29 return formattedExamples.trimEnd();50 let formattedExamples = '';
30};
3151
32export const formatByafAlternateGreetings = (greetings) => {52 examples.forEach((example) => {
33 if (!Array.isArray(greetings)) {53 if (!example?.text) {
34 return [];54 return;
35 }55 }
56 formattedExamples += `<START>\n${this.replaceMacros(example.text)}\n`;
57 });
3658
37 if (greetings.length <= 1) {59 return formattedExamples.trimEnd();
38 return [];
39 }60 }
4061
41 // Skip one because it goes into 'first_mes'62 /**
42 return greetings.slice(1).map(g => replaceByafMacros(g?.text));63 * Formats alternate greetings for a character.
43};64 * @param {ByafExampleMessage[]} [greetings] Array of greeting objects
65 * @returns {string[]} Formatted alternate greetings
66 * @private
67 */
68 formatAlternateGreetings(greetings) {
69 if (!Array.isArray(greetings)) {
70 return [];
71 }
4472
45export const convertByafCharacterBook = (items) => {73 if (greetings.length <= 1) {
46 if (!Array.isArray(items) || items.length === 0) {74 return [];
47 return null;75 }
48 }
4976
50 const book = {77 // Skip one because it goes into 'first_mes'
51 /** @type {any[]} */78 return greetings.slice(1).map(g => this.replaceMacros(g?.text));
52 entries: [],79 }
53 };
5480
55 items.forEach((item, index) => {81 /**
56 if (!item) {82 * Converts character book items to a structured format.
57 return;83 * @param {ByafLoreItem[]} items Array of key-value pairs
84 * @returns {CharacterBook|undefined} Converted character book or undefined if invalid
85 * @private
86 */
87 convertCharacterBook(items) {
88 if (!Array.isArray(items) || items.length === 0) {
89 return undefined;
58 }90 }
59 book.entries.push({91
60 keys: replaceByafMacros(item?.key).split(',').map(key => key.trim()).filter(Boolean),92 /** @type {CharacterBook} */
61 content: replaceByafMacros(item?.value),93 const book = {
94 entries: [],
62 extensions: {},95 extensions: {},
63 enabled: true,96 };
64 insertion_order: index,97
98 items.forEach((item, index) => {
99 if (!item) {
100 return;
101 }
102 book.entries.push({
103 keys: this.replaceMacros(item?.key).split(',').map(key => key.trim()).filter(Boolean),
104 content: this.replaceMacros(item?.value),
105 extensions: {},
106 enabled: true,
107 insertion_order: index,
108 });
65 });109 });
66 });
67110
68 return book;111 return book;
69};112 }
70113
71/**114 /**
72 * Extracts a character object from BYAF buffer.115 * Extracts a character object from BYAF buffer.
73 * @param {ArrayBufferLike} data ZIP buffer116 * @param {ByafManifest} manifest BYAF manifest
74 * @param {object} manifest BYAF manifest117 * @returns {Promise<{character:ByafCharacter,characterPath:string}>} Character object
75 * @returns {Promise<{character:object,characterPath:string}>} Character object118 * @private
76 */119 */
77export async function getCharacterFromByafManifest(data, manifest) {120 async getCharacterFromManifest(manifest) {
78 const charactersArray = manifest?.characters;121 const charactersArray = manifest?.characters;
79122
80 if (!Array.isArray(charactersArray)) {123 if (!Array.isArray(charactersArray)) {
81 throw new Error('Invalid BYAF file: missing characters array');124 throw new Error('Invalid BYAF file: missing characters array');
82 }125 }
83126
84 if (charactersArray.length === 0) {127 if (charactersArray.length === 0) {
85 throw new Error('Invalid BYAF file: characters array is empty');128 throw new Error('Invalid BYAF file: characters array is empty');
86 }129 }
87130
88 if (charactersArray.length > 1) {131 if (charactersArray.length > 1) {
89 console.warn('Warning: BYAF manifest contains more than one character, only the first one will be imported');132 console.warn('Warning: BYAF manifest contains more than one character, only the first one will be imported');
90 }133 }
91134
92 const characterPath = charactersArray[0];135 const characterPath = charactersArray[0];
93 if (!characterPath) {136 if (!characterPath) {
94 throw new Error('Invalid BYAF file: missing character path');137 throw new Error('Invalid BYAF file: missing character path');
95 }138 }
96139
97 const characterBuffer = await extractFileFromZipBuffer(data, characterPath);140 const characterBuffer = await extractFileFromZipBuffer(this.#data, characterPath);
98 if (!characterBuffer) {141 if (!characterBuffer) {
99 throw new Error('Invalid BYAF file: failed to extract character JSON');142 throw new Error('Invalid BYAF file: failed to extract character JSON');
100 }143 }
101144
102 try {145 try {
103 const character = JSON.parse(characterBuffer.toString());146 const character = JSON.parse(characterBuffer.toString());
104 return { character, characterPath };147 return { character, characterPath };
105 } catch (error) {148 } catch (error) {
106 console.error('Failed to parse character JSON from BYAF:', error);149 console.error('Failed to parse character JSON from BYAF:', error);
107 throw new Error('Invalid BYAF file: character is not a valid JSON');150 throw new Error('Invalid BYAF file: character is not a valid JSON');
151 }
108 }152 }
109}
110153
111/**154 /**
112 * Extracts a scenario object from BYAF buffer.155 * Extracts a scenario object from BYAF buffer.
113 * @param {ArrayBufferLike} data ZIP buffer156 * @param {ByafManifest} manifest BYAF manifest
114 * @param {object} manifest BYAF manifest157 * @returns {Promise<Partial<ByafScenario>>} Scenario object
115 * @returns {Promise<object>} Scenario object158 * @private
116 */159 */
117export async function getScenarioFromByafManifest(data, manifest) {160 async getScenarioFromManifest(manifest) {
118 const scenariosArray = manifest?.scenarios;161 const scenariosArray = manifest?.scenarios;
119162
120 if (!Array.isArray(scenariosArray) || scenariosArray.length === 0) {163 if (!Array.isArray(scenariosArray) || scenariosArray.length === 0) {
121 console.warn('Warning: BYAF manifest contains no scenarios');164 console.warn('Warning: BYAF manifest contains no scenarios');
122 return {};165 return {};
123 }166 }
124167
125 if (scenariosArray.length > 1) {168 if (scenariosArray.length > 1) {
126 console.warn('Warning: BYAF manifest contains more than one scenario, only the first one will be imported');169 console.warn('Warning: BYAF manifest contains more than one scenario, only the first one will be imported');
127 }170 }
128171
129 const scenarioPath = scenariosArray[0];172 const scenarioPath = scenariosArray[0];
130 if (!scenarioPath) {173 if (!scenarioPath) {
131 console.warn('Warning: missing BYAF scenario path');174 console.warn('Warning: missing BYAF scenario path');
132 return {};175 return {};
133 }176 }
134177
135 const scenarioBuffer = await extractFileFromZipBuffer(data, scenarioPath);178 const scenarioBuffer = await extractFileFromZipBuffer(this.#data, scenarioPath);
136 if (!scenarioBuffer) {179 if (!scenarioBuffer) {
137 console.warn('Warning: failed to extract BYAF scenario JSON');180 console.warn('Warning: failed to extract BYAF scenario JSON');
138 return {};181 return {};
139 }182 }
140183
141 try {184 try {
142 return JSON.parse(scenarioBuffer.toString());185 return JSON.parse(scenarioBuffer.toString());
143 } catch (error) {186 } catch (error) {
144 console.warn('Warning: BYAF scenario is not a valid JSON', error);187 console.warn('Warning: BYAF scenario is not a valid JSON', error);
145 return {};188 return {};
189 }
146 }190 }
147}
148191
149/**192 /**
150 * Extracts an image from BYAF buffer.193 * Extracts an image from BYAF buffer.
151 * @param {ArrayBufferLike} data ZIP buffer194 * @param {ByafCharacter} character Character object
152 * @param {object} character Character object195 * @param {string} characterPath Path to the character in the BYAF manifest
153 * @param {string} characterPath Path to the character in the BYAF manifest196 * @return {Promise<Buffer>} Image buffer
154 */197 * @private
155export async function getImageBufferFromByafCharacter(data, character, characterPath) {198 */
156 const defaultAvatarBuffer = await fsPromises.readFile(DEFAULT_AVATAR_PATH);199 async getCharacterImage(character, characterPath) {
157 const characterImages = character?.images;200 const defaultAvatarBuffer = await fsPromises.readFile(DEFAULT_AVATAR_PATH);
201 const characterImages = character?.images;
202
203 if (!Array.isArray(characterImages) || characterImages.length === 0) {
204 console.warn('Warning: BYAF character has no images');
205 return defaultAvatarBuffer;
206 }
158207
159 if (!Array.isArray(characterImages) || characterImages.length === 0) {208 const imagePath = characterImages[0]?.path;
160 console.warn('Warning: BYAF character has no images');209 if (!imagePath) {
161 return defaultAvatarBuffer;210 console.warn('Warning: BYAF character image path is empty');
211 return defaultAvatarBuffer;
212 }
213
214 const fullImagePath = urlJoin(path.dirname(characterPath), imagePath);
215 const imageBuffer = await extractFileFromZipBuffer(this.#data, fullImagePath);
216 if (!imageBuffer) {
217 console.warn('Warning: failed to extract BYAF character image');
218 return defaultAvatarBuffer;
219 }
220
221 return imageBuffer;
162 }222 }
163223
164 const imagePath = characterImages[0]?.path;224 /**
165 if (!imagePath) {225 * Formats BYAF data as a character card.
166 console.warn('Warning: BYAF character image path is empty');226 * @param {ByafManifest} manifest BYAF manifest
167 return defaultAvatarBuffer;227 * @param {ByafCharacter} character Character object
228 * @param {Partial<ByafScenario>} scenario Scenario object
229 * @return {TavernCardV2} Character card object
230 * @private
231 */
232 getCharacterCard(manifest, character, scenario) {
233 return {
234 spec: 'chara_card_v2',
235 spec_version: '2.0',
236 data: {
237 name: sanitize(character?.name || character?.displayName || ''),
238 description: this.replaceMacros(character?.persona),
239 personality: '',
240 scenario: this.replaceMacros(scenario?.narrative),
241 first_mes: this.replaceMacros(scenario?.firstMessages?.[0]?.text),
242 mes_example: this.formatExampleMessages(scenario?.exampleMessages),
243 creator_notes: '',
244 system_prompt: this.replaceMacros(scenario?.formattingInstructions),
245 post_history_instructions: '',
246 alternate_greetings: this.formatAlternateGreetings(scenario?.firstMessages),
247 character_book: this.convertCharacterBook(character?.loreItems),
248 tags: [],
249 creator: manifest?.author?.name || '',
250 character_version: '',
251 extensions: {},
252 },
253 // @ts-ignore Non-standard spec extension
254 create_date: humanizedISO8601DateTime(),
255 };
168 }256 }
169257
170 const fullImagePath = urlJoin(path.dirname(characterPath), imagePath);258 /**
171 const imageBuffer = await extractFileFromZipBuffer(data, fullImagePath);259 * Gets the manifest from the BYAF data.
172 if (!imageBuffer) {260 * @returns {Promise<ByafManifest>} Parsed manifest
173 console.warn('Warning: failed to extract BYAF character image');261 * @private
174 return defaultAvatarBuffer;262 */
263 async getManifest() {
264 const manifestBuffer = await extractFileFromZipBuffer(this.#data, 'manifest.json');
265 if (!manifestBuffer) {
266 throw new Error('Failed to extract manifest.json from BYAF file');
267 }
268
269 const manifest = JSON.parse(manifestBuffer.toString());
270 if (!manifest || typeof manifest !== 'object') {
271 throw new Error('Invalid BYAF manifest');
272 }
273
274 return manifest;
175 }275 }
176276
177 return imageBuffer;277 /**
178}278 * Parses the BYAF data.
279 * @return {Promise<{card: TavernCardV2, image: Buffer}>} Parsed character card and image buffer
280 */
281 async parse() {
282 const manifest = await this.getManifest();
283 const { character, characterPath } = await this.getCharacterFromManifest(manifest);
284 const scenario = await this.getScenarioFromManifest(manifest);
285 const image = await this.getCharacterImage(character, characterPath);
286 const card = this.getCharacterCard(manifest, character, scenario);
179287
180/**288 return { card, image };
181 * Formats BYAF data as a character card.289 }
182 * @param {object} character
183 * @param {object} scenario
184 */
185export function formatByafAsCharacterCard(character, scenario) {
186 return {
187 spec: 'chara_card_v2',
188 spec_version: '2.0',
189 create_date: humanizedISO8601DateTime(),
190 data: {
191 name: sanitize(character?.name || character?.displayName || ''),
192 description: replaceByafMacros(character?.persona),
193 personality: '',
194 scenario: replaceByafMacros(scenario?.narrative),
195 first_mes: replaceByafMacros(scenario?.firstMessages?.[0]?.text),
196 mes_example: formatByafExampleMessages(scenario?.exampleMessages),
197 creator_notes: '',
198 system_prompt: replaceByafMacros(scenario?.formattingInstructions),
199 post_history_instructions: '',
200 alternate_greetings: formatByafAlternateGreetings(scenario?.firstMessages),
201 character_book: convertByafCharacterBook(character?.loreItems),
202 tags: [],
203 creator: '',
204 character_version: '',
205 extensions: {},
206 },
207 };
208}290}
291
292export default ByafParser;
src/endpoints/characters.js+4 -13
@@ -22,7 +22,7 @@ import { invalidateThumbnail } from './thumbnails.js';
22import { importRisuSprites } from './sprites.js';22import { importRisuSprites } from './sprites.js';
23import { getUserDirectories } from '../users.js';23import { getUserDirectories } from '../users.js';
24import { getChatInfo } from './chats.js';24import { getChatInfo } from './chats.js';
25import { formatByafAsCharacterCard, getCharacterFromByafManifest, getImageBufferFromByafCharacter, getScenarioFromByafManifest } from '../byaf.js';25import { ByafParser } from '../byaf.js';
26import cacheBuster from '../middleware/cacheBuster.js';26import cacheBuster from '../middleware/cacheBuster.js';
2727
28// With 100 MB limit it would take roughly 3000 characters to reach this limit28// With 100 MB limit it would take roughly 3000 characters to reach this limit
@@ -803,19 +803,10 @@ async function importFromByaf(uploadPath, { request }, preservedFileName) {
803 await fsPromises.unlink(uploadPath);803 await fsPromises.unlink(uploadPath);
804 console.info('Importing from BYAF');804 console.info('Importing from BYAF');
805805
806 const manifestBuffer = await extractFileFromZipBuffer(data, 'manifest.json');806 const byafData = await new ByafParser(data).parse();
807 if (!manifestBuffer) {807 const card = readFromV2(byafData.card);
808 throw new Error('Failed to extract manifest.json from BYAF file');
809 }
810
811 const manifest = JSON.parse(manifestBuffer.toString());
812 const { character, characterPath } = await getCharacterFromByafManifest(data, manifest);
813 const scenario = await getScenarioFromByafManifest(data, manifest);
814 const image = await getImageBufferFromByafCharacter(data, character, characterPath);
815
816 const card = readFromV2(formatByafAsCharacterCard(character, scenario));
817 const fileName = preservedFileName || getPngName(card.name, request.user.directories);808 const fileName = preservedFileName || getPngName(card.name, request.user.directories);
818 const result = await writeCharacterData(image, JSON.stringify(card), fileName, request);809 const result = await writeCharacterData(byafData.image, JSON.stringify(card), fileName, request);
819 return result ? fileName : '';810 return result ? fileName : '';
820}811}
821812
src/types/byaf.d.ts+77 -0
@@ -0,0 +1,77 @@
1type ByafLoreItem = {
2 key: string;
3 value: string;
4};
5
6type ByafCharacterImage = {
7 path: string;
8 label: string;
9};
10
11type ByafExampleMessage = {
12 characterID: string;
13 text: string;
14};
15
16type ByafCharacter = {
17 schemaVersion: 1;
18 id: string;
19 name: string;
20 displayName: string;
21 isNSFW: boolean;
22 persona: string;
23 createdAt: string;
24 updatedAt: string;
25 loreItems: Array<ByafLoreItem>;
26 images: Array<ByafCharacterImage>;
27};
28
29type ByafManifest = {
30 schemaVersion: 1;
31 createdAt: string;
32 characters: string[];
33 scenarios: string[];
34 author?: {
35 name: string;
36 backyardURL: string;
37 };
38};
39
40type ByafAiMessage = {
41 type: "ai";
42 outputs: Array<{
43 createdAt: string;
44 updatedAt: string;
45 text: string;
46 activeTimestamp: string;
47 }>;
48};
49
50type ByafHumanMessage = {
51 type: "human";
52 createdAt: string;
53 updatedAt: string;
54 text: string;
55};
56
57type ByafScenario = {
58 schemaVersion: 1;
59 title?: string;
60 model?: string;
61 formattingInstructions: string;
62 minP: number;
63 minPEnabled: boolean;
64 temperature: number;
65 repeatPenalty: number;
66 repeatLastN: number;
67 topK: number;
68 topP: number;
69 exampleMessages: Array<ByafExampleMessage>;
70 canDeleteExampleMessages: boolean;
71 firstMessages: Array<ByafExampleMessage>;
72 narrative: string;
73 promptTemplate: "general" | "ChatML" | "Llama3" | "Gemma2" | "CommandR" | "MistralInstruct" | null;
74 grammar: string | null;
75 messages: Array<ByafAiMessage | ByafHumanMessage>;
76 backgroundImage?: string;
77};
src/types/spec-v2.d.ts+52 -0
@@ -0,0 +1,52 @@
1type TavernCardV2 = {
2 spec: 'chara_card_v2';
3 spec_version: '2.0';
4 data: {
5 name: string;
6 description: string;
7 personality: string;
8 scenario: string;
9 first_mes: string;
10 mes_example: string;
11
12 creator_notes: string;
13 system_prompt: string;
14 post_history_instructions: string;
15 alternate_greetings: Array<string>;
16 character_book?: CharacterBook;
17
18 tags: Array<string>;
19 creator: string;
20 character_version: string;
21 extensions: Record<string, any>;
22 }
23}
24
25type CharacterBook = {
26 name?: string;
27 description?: string;
28 scan_depth?: number;
29 token_budget?: number;
30 recursive_scanning?: boolean;
31 extensions: Record<string, any>;
32 entries: Array<CharacterBookEntry>;
33}
34
35type CharacterBookEntry = {
36 keys: Array<string>;
37 content: string;
38 extensions: Record<string, any>;
39 enabled: boolean;
40 insertion_order: number;
41 case_sensitive?: boolean;
42
43 name?: string;
44 priority?: number;
45
46 id?: number;
47 comment?: string;
48 selective?: boolean;
49 secondary_keys?: Array<string>;
50 constant?: boolean;
51 position?: 'before_char' | 'after_char';
52};