Refactor BYAF parsing (#4303) * Refactor BYAF parsing * Refactor ByafParser to use private field for data and improve type annotations

38a5e4a4f2a0068510124423f9e4c9913116e1f2

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: 7edd81f183014568c47831d2c02dc926a80d0eda
4 files changed, +377 -173Ignore whitespace
src/byaf.js+244 -160
@@ -5,204 +5,288 @@ import urlJoin from 'url-join';
55import { DEFAULT_AVATAR_PATH } from './constants.js';
66import { extractFileFromZipBuffer, humanizedISO8601DateTime } from './util.js';
77
8-export const replaceByafMacros = (s) =>
8+/**
9- String(s || '')
9+ * A parser for BYAF (Backyard Archive Format) files.
10- .replace(/#{user}:/gi, '{{user}}:')
10+ */
11- .replace(/#{character}:/gi, '{{char}}:')
11+export class ByafParser {
12- .replace(/{character}(?!})/gi, '{{char}}')
12+ /**
13- .replace(/{user}(?!})/gi, '{{user}}');
13+ * @param {ArrayBufferLike} data BYAF ZIP buffer
14-
14+ */
15-export const formatByafExampleMessages = (examples) => {
15+ #data;
16- if (!Array.isArray(examples)) {
16+
17- return '';
17+ /**
18+ * Creates an instance of ByafParser.
19+ * @param {ArrayBufferLike} data BYAF ZIP buffer
20+ */
21+ constructor(data) {
22+ this.#data = data;
1823 }
1924
20- let formattedExamples = '';
25+ /**
26+ * Replaces known macros in a string.
27+ * @param {string} [str] String to process
28+ * @returns {string} String with macros replaced
29+ * @private
30+ */
31+ replaceMacros(str) {
32+ return String(str || '')
33+ .replace(/#{user}:/gi, '{{user}}:')
34+ .replace(/#{character}:/gi, '{{char}}:')
35+ .replace(/{character}(?!})/gi, '{{char}}')
36+ .replace(/{user}(?!})/gi, '{{user}}');
37+ }
2138
22- examples.forEach((example) => {
39+ /**
23- if (!example?.text) {
40+ * Formats example messages for a character.
24- return;
41+ * @param {ByafExampleMessage[]} [examples] Array of example objects
42+ * @returns {string} Formatted example messages
43+ * @private
44+ */
45+ formatExampleMessages(examples) {
46+ if (!Array.isArray(examples)) {
47+ return '';
2548 }
26- formattedExamples += `<START>\n${replaceByafMacros(example.text)}\n`;
27- });
2849
29- return formattedExamples.trimEnd();
50+ let formattedExamples = '';
30-};
3151
32-export const formatByafAlternateGreetings = (greetings) => {
52+ examples.forEach((example) => {
3353 if (!Arrayexample?.isArray(greetings)text) {
34- return [];
54+ return;
3555 }
56+ formattedExamples += `<START>\n${this.replaceMacros(example.text)}\n`;
57+ });
3658
37- if (greetings.length <= 1) {
59+ return formattedExamples.trimEnd();
38- return [];
3960 }
4061
41- // Skip one because it goes into 'first_mes'
62+ /**
42- return greetings.slice(1).map(g => replaceByafMacros(g?.text));
63+ * Formats alternate greetings for a character.
43-};
64+ * @param {ByafExampleMessage[]} [greetings] Array of greeting objects
65+ * @returns {string[]} Formatted alternate greetings
66+ * @private
67+ */
68+ formatAlternateGreetings(greetings) {
69+ if (!Array.isArray(greetings)) {
70+ return [];
71+ }
4472
45-export const convertByafCharacterBook = (items) => {
73+ if (greetings.length <= 1) {
46- if (!Array.isArray(items) || items.length === 0) {
74+ return [];
47- return null;
75+ }
48- }
4976
50- const book = {
77+ // Skip one because it goes into 'first_mes'
51- /** @type {any[]} */
78+ return greetings.slice(1).map(g => this.replaceMacros(g?.text));
52- entries: [],
79+ }
53- };
5480
55- items.forEach((item, index) => {
81+ /**
56- if (!item) {
82+ * Converts character book items to a structured format.
57- return;
83+ * @param {ByafLoreItem[]} items Array of key-value pairs
84+ * @returns {CharacterBook|undefined} Converted character book or undefined if invalid
85+ * @private
86+ */
87+ convertCharacterBook(items) {
88+ if (!Array.isArray(items) || items.length === 0) {
89+ return undefined;
5890 }
59- book.entries.push({
91+
60- keys: replaceByafMacros(item?.key).split(',').map(key => key.trim()).filter(Boolean),
92+ /** @type {CharacterBook} */
61- content: replaceByafMacros(item?.value),
93+ const book = {
94+ entries: [],
6295 extensions: {},
63- enabled: true,
96+ };
64- insertion_order: index,
97+
98+ items.forEach((item, index) => {
99+ if (!item) {
100+ return;
101+ }
102+ book.entries.push({
103+ keys: this.replaceMacros(item?.key).split(',').map(key => key.trim()).filter(Boolean),
104+ content: this.replaceMacros(item?.value),
105+ extensions: {},
106+ enabled: true,
107+ insertion_order: index,
108+ });
65109 });
66- });
67110
68111 return book;
69112 };
70113
71114 /**
72115 * Extracts a character object from BYAF buffer.
73116 * @param {ArrayBufferLikeByafManifest} datamanifest ZIPBYAF buffermanifest
74- * @param {object} manifest BYAF manifest
117+ * @returns {Promise<{character:ByafCharacter,characterPath:string}>} Character object
75- * @returns {Promise<{character:object,characterPath:string}>} Character object
118+ * @private
76119 */
77120export async function getCharacterFromByafManifestgetCharacterFromManifest(data, manifest) {
78121 const charactersArray = manifest?.characters;
79122
80123 if (!Array.isArray(charactersArray)) {
81124 throw new Error('Invalid BYAF file: missing characters array');
82125 }
83126
84127 if (charactersArray.length === 0) {
85128 throw new Error('Invalid BYAF file: characters array is empty');
86129 }
87130
88131 if (charactersArray.length > 1) {
89132 console.warn('Warning: BYAF manifest contains more than one character, only the first one will be imported');
90133 }
91134
92135 const characterPath = charactersArray[0];
93136 if (!characterPath) {
94137 throw new Error('Invalid BYAF file: missing character path');
95138 }
96139
97140 const characterBuffer = await extractFileFromZipBuffer(this.#data, characterPath);
98141 if (!characterBuffer) {
99142 throw new Error('Invalid BYAF file: failed to extract character JSON');
100143 }
101144
102145 try {
103146 const character = JSON.parse(characterBuffer.toString());
104147 return { character, characterPath };
105148 } catch (error) {
106149 console.error('Failed to parse character JSON from BYAF:', error);
107150 throw new Error('Invalid BYAF file: character is not a valid JSON');
151+ }
108152 }
109-}
110153
111154 /**
112155 * Extracts a scenario object from BYAF buffer.
113156 * @param {ArrayBufferLikeByafManifest} datamanifest ZIPBYAF buffermanifest
114- * @param {object} manifest BYAF manifest
157+ * @returns {Promise<Partial<ByafScenario>>} Scenario object
115- * @returns {Promise<object>} Scenario object
158+ * @private
116159 */
117160export async function getScenarioFromByafManifestgetScenarioFromManifest(data, manifest) {
118161 const scenariosArray = manifest?.scenarios;
119162
120163 if (!Array.isArray(scenariosArray) || scenariosArray.length === 0) {
121164 console.warn('Warning: BYAF manifest contains no scenarios');
122165 return {};
123166 }
124167
125168 if (scenariosArray.length > 1) {
126169 console.warn('Warning: BYAF manifest contains more than one scenario, only the first one will be imported');
127170 }
128171
129172 const scenarioPath = scenariosArray[0];
130173 if (!scenarioPath) {
131174 console.warn('Warning: missing BYAF scenario path');
132175 return {};
133176 }
134177
135178 const scenarioBuffer = await extractFileFromZipBuffer(this.#data, scenarioPath);
136179 if (!scenarioBuffer) {
137180 console.warn('Warning: failed to extract BYAF scenario JSON');
138181 return {};
139182 }
140183
141184 try {
142185 return JSON.parse(scenarioBuffer.toString());
143186 } catch (error) {
144187 console.warn('Warning: BYAF scenario is not a valid JSON', error);
145188 return {};
189+ }
146190 }
147-}
148191
149192 /**
150193 * Extracts an image from BYAF buffer.
151194 * @param {ArrayBufferLikeByafCharacter} datacharacter ZIPCharacter bufferobject
152195 * @param {objectstring} characterPath Path to the character Characterin objectthe BYAF manifest
153- * @param {string} characterPath Path to the character in the BYAF manifest
196+ * @return {Promise<Buffer>} Image buffer
154- */
197+ * @private
155-export async function getImageBufferFromByafCharacter(data, character, characterPath) {
198+ */
156- const defaultAvatarBuffer = await fsPromises.readFile(DEFAULT_AVATAR_PATH);
199+ async getCharacterImage(character, characterPath) {
157200 const characterImagesdefaultAvatarBuffer = character?await fsPromises.imagesreadFile(DEFAULT_AVATAR_PATH);
201+ const characterImages = character?.images;
202+
203+ if (!Array.isArray(characterImages) || characterImages.length === 0) {
204+ console.warn('Warning: BYAF character has no images');
205+ return defaultAvatarBuffer;
206+ }
158207
159- if (!Array.isArray(characterImages) || characterImages.length === 0) {
208+ const imagePath = characterImages[0]?.path;
160- console.warn('Warning: BYAF character has no images');
209+ if (!imagePath) {
161- return defaultAvatarBuffer;
210+ console.warn('Warning: BYAF character image path is empty');
211+ return defaultAvatarBuffer;
212+ }
213+
214+ const fullImagePath = urlJoin(path.dirname(characterPath), imagePath);
215+ const imageBuffer = await extractFileFromZipBuffer(this.#data, fullImagePath);
216+ if (!imageBuffer) {
217+ console.warn('Warning: failed to extract BYAF character image');
218+ return defaultAvatarBuffer;
219+ }
220+
221+ return imageBuffer;
162222 }
163223
164- const imagePath = characterImages[0]?.path;
224+ /**
165- if (!imagePath) {
225+ * Formats BYAF data as a character card.
166- console.warn('Warning: BYAF character image path is empty');
226+ * @param {ByafManifest} manifest BYAF manifest
167- return defaultAvatarBuffer;
227+ * @param {ByafCharacter} character Character object
228+ * @param {Partial<ByafScenario>} scenario Scenario object
229+ * @return {TavernCardV2} Character card object
230+ * @private
231+ */
232+ getCharacterCard(manifest, character, scenario) {
233+ return {
234+ spec: 'chara_card_v2',
235+ spec_version: '2.0',
236+ data: {
237+ name: sanitize(character?.name || character?.displayName || ''),
238+ description: this.replaceMacros(character?.persona),
239+ personality: '',
240+ scenario: this.replaceMacros(scenario?.narrative),
241+ first_mes: this.replaceMacros(scenario?.firstMessages?.[0]?.text),
242+ mes_example: this.formatExampleMessages(scenario?.exampleMessages),
243+ creator_notes: '',
244+ system_prompt: this.replaceMacros(scenario?.formattingInstructions),
245+ post_history_instructions: '',
246+ alternate_greetings: this.formatAlternateGreetings(scenario?.firstMessages),
247+ character_book: this.convertCharacterBook(character?.loreItems),
248+ tags: [],
249+ creator: manifest?.author?.name || '',
250+ character_version: '',
251+ extensions: {},
252+ },
253+ // @ts-ignore Non-standard spec extension
254+ create_date: humanizedISO8601DateTime(),
255+ };
168256 }
169257
170- const fullImagePath = urlJoin(path.dirname(characterPath), imagePath);
258+ /**
171- const imageBuffer = await extractFileFromZipBuffer(data, fullImagePath);
259+ * Gets the manifest from the BYAF data.
172- if (!imageBuffer) {
260+ * @returns {Promise<ByafManifest>} Parsed manifest
173- console.warn('Warning: failed to extract BYAF character image');
261+ * @private
174- return defaultAvatarBuffer;
262+ */
263+ async getManifest() {
264+ const manifestBuffer = await extractFileFromZipBuffer(this.#data, 'manifest.json');
265+ if (!manifestBuffer) {
266+ throw new Error('Failed to extract manifest.json from BYAF file');
267+ }
268+
269+ const manifest = JSON.parse(manifestBuffer.toString());
270+ if (!manifest || typeof manifest !== 'object') {
271+ throw new Error('Invalid BYAF manifest');
272+ }
273+
274+ return manifest;
175275 }
176276
177- return imageBuffer;
277+ /**
178-}
278+ * Parses the BYAF data.
279+ * @return {Promise<{card: TavernCardV2, image: Buffer}>} Parsed character card and image buffer
280+ */
281+ async parse() {
282+ const manifest = await this.getManifest();
283+ const { character, characterPath } = await this.getCharacterFromManifest(manifest);
284+ const scenario = await this.getScenarioFromManifest(manifest);
285+ const image = await this.getCharacterImage(character, characterPath);
286+ const card = this.getCharacterCard(manifest, character, scenario);
179287
180-/**
288+ return { card, image };
181- * Formats BYAF data as a character card.
289+ }
182- * @param {object} character
183- * @param {object} scenario
184- */
185-export function formatByafAsCharacterCard(character, scenario) {
186- return {
187- spec: 'chara_card_v2',
188- spec_version: '2.0',
189- create_date: humanizedISO8601DateTime(),
190- data: {
191- name: sanitize(character?.name || character?.displayName || ''),
192- description: replaceByafMacros(character?.persona),
193- personality: '',
194- scenario: replaceByafMacros(scenario?.narrative),
195- first_mes: replaceByafMacros(scenario?.firstMessages?.[0]?.text),
196- mes_example: formatByafExampleMessages(scenario?.exampleMessages),
197- creator_notes: '',
198- system_prompt: replaceByafMacros(scenario?.formattingInstructions),
199- post_history_instructions: '',
200- alternate_greetings: formatByafAlternateGreetings(scenario?.firstMessages),
201- character_book: convertByafCharacterBook(character?.loreItems),
202- tags: [],
203- creator: '',
204- character_version: '',
205- extensions: {},
206- },
207- };
208290}
291+
292+export default ByafParser;
src/endpoints/characters.js+4 -13
@@ -22,7 +22,7 @@ import { invalidateThumbnail } from './thumbnails.js';
2222import { importRisuSprites } from './sprites.js';
2323import { getUserDirectories } from '../users.js';
2424import { getChatInfo } from './chats.js';
2525import { formatByafAsCharacterCard, getCharacterFromByafManifest, getImageBufferFromByafCharacter, getScenarioFromByafManifestByafParser } from '../byaf.js';
2626import cacheBuster from '../middleware/cacheBuster.js';
2727
2828// With 100 MB limit it would take roughly 3000 characters to reach this limit
@@ -803,19 +803,10 @@ async function importFromByaf(uploadPath, { request }, preservedFileName) {
803803 await fsPromises.unlink(uploadPath);
804804 console.info('Importing from BYAF');
805805
806806 const manifestBufferbyafData = await extractFileFromZipBuffernew ByafParser(data, 'manifest).json'parse();
807- if (!manifestBuffer) {
807+ const card = readFromV2(byafData.card);
808- throw new Error('Failed to extract manifest.json from BYAF file');
809- }
810-
811- const manifest = JSON.parse(manifestBuffer.toString());
812- const { character, characterPath } = await getCharacterFromByafManifest(data, manifest);
813- const scenario = await getScenarioFromByafManifest(data, manifest);
814- const image = await getImageBufferFromByafCharacter(data, character, characterPath);
815-
816- const card = readFromV2(formatByafAsCharacterCard(character, scenario));
817808 const fileName = preservedFileName || getPngName(card.name, request.user.directories);
818809 const result = await writeCharacterData(byafData.image, JSON.stringify(card), fileName, request);
819810 return result ? fileName : '';
820811}
821812
src/types/byaf.d.ts+77 -0
@@ -0,0 +1,77 @@
1+type ByafLoreItem = {
2+ key: string;
3+ value: string;
4+};
5+
6+type ByafCharacterImage = {
7+ path: string;
8+ label: string;
9+};
10+
11+type ByafExampleMessage = {
12+ characterID: string;
13+ text: string;
14+};
15+
16+type ByafCharacter = {
17+ schemaVersion: 1;
18+ id: string;
19+ name: string;
20+ displayName: string;
21+ isNSFW: boolean;
22+ persona: string;
23+ createdAt: string;
24+ updatedAt: string;
25+ loreItems: Array<ByafLoreItem>;
26+ images: Array<ByafCharacterImage>;
27+};
28+
29+type ByafManifest = {
30+ schemaVersion: 1;
31+ createdAt: string;
32+ characters: string[];
33+ scenarios: string[];
34+ author?: {
35+ name: string;
36+ backyardURL: string;
37+ };
38+};
39+
40+type ByafAiMessage = {
41+ type: "ai";
42+ outputs: Array<{
43+ createdAt: string;
44+ updatedAt: string;
45+ text: string;
46+ activeTimestamp: string;
47+ }>;
48+};
49+
50+type ByafHumanMessage = {
51+ type: "human";
52+ createdAt: string;
53+ updatedAt: string;
54+ text: string;
55+};
56+
57+type ByafScenario = {
58+ schemaVersion: 1;
59+ title?: string;
60+ model?: string;
61+ formattingInstructions: string;
62+ minP: number;
63+ minPEnabled: boolean;
64+ temperature: number;
65+ repeatPenalty: number;
66+ repeatLastN: number;
67+ topK: number;
68+ topP: number;
69+ exampleMessages: Array<ByafExampleMessage>;
70+ canDeleteExampleMessages: boolean;
71+ firstMessages: Array<ByafExampleMessage>;
72+ narrative: string;
73+ promptTemplate: "general" | "ChatML" | "Llama3" | "Gemma2" | "CommandR" | "MistralInstruct" | null;
74+ grammar: string | null;
75+ messages: Array<ByafAiMessage | ByafHumanMessage>;
76+ backgroundImage?: string;
77+};
src/types/spec-v2.d.ts+52 -0
@@ -0,0 +1,52 @@
1+type TavernCardV2 = {
2+ spec: 'chara_card_v2';
3+ spec_version: '2.0';
4+ data: {
5+ name: string;
6+ description: string;
7+ personality: string;
8+ scenario: string;
9+ first_mes: string;
10+ mes_example: string;
11+
12+ creator_notes: string;
13+ system_prompt: string;
14+ post_history_instructions: string;
15+ alternate_greetings: Array<string>;
16+ character_book?: CharacterBook;
17+
18+ tags: Array<string>;
19+ creator: string;
20+ character_version: string;
21+ extensions: Record<string, any>;
22+ }
23+}
24+
25+type CharacterBook = {
26+ name?: string;
27+ description?: string;
28+ scan_depth?: number;
29+ token_budget?: number;
30+ recursive_scanning?: boolean;
31+ extensions: Record<string, any>;
32+ entries: Array<CharacterBookEntry>;
33+}
34+
35+type CharacterBookEntry = {
36+ keys: Array<string>;
37+ content: string;
38+ extensions: Record<string, any>;
39+ enabled: boolean;
40+ insertion_order: number;
41+ case_sensitive?: boolean;
42+
43+ name?: string;
44+ priority?: number;
45+
46+ id?: number;
47+ comment?: string;
48+ selective?: boolean;
49+ secondary_keys?: Array<string>;
50+ constant?: boolean;
51+ position?: 'before_char' | 'after_char';
52+};