Switch branch
feat/multi-feature-pack maindefault
View all branches
Blame Raw
Cohee · 51ad27fb · · 1685 lines (66.9 KB)
3 contributors 
1import path from 'node:path';


2import fs from 'node:fs';


3import { promises as fsPromises } from 'node:fs';


4import { Buffer } from 'node:buffer';


5


6import express from 'express';


7import sanitize from 'sanitize-filename';


8import { sync as writeFileAtomicSync } from 'write-file-atomic';


9import yaml from 'yaml';


10import _ from 'lodash';


11import mime from 'mime-types';


12import { Jimp, JimpMime } from '../jimp.js';


13import storage from 'node-persist';


14


15import { AVATAR_WIDTH, AVATAR_HEIGHT, DEFAULT_AVATAR_PATH } from '../constants.js';


16import { default as validateAvatarUrlMiddleware, getFileNameValidationFunction, forbiddenRegExp } from '../middleware/validateFileName.js';


17import { deepMerge, humanizedDateTime, tryParse, MemoryLimitedMap, getConfigValue, mutateJsonString, clientRelativePath, getUniqueName, sanitizeSafeCharacterReplacements } from '../util.js';


18import { TavernCardValidator } from '../validator/TavernCardValidator.js';


19import { parse, read, write } from '../character-card-parser.js';


20import { readWorldInfoFile } from './worldinfo.js';


21import { invalidateThumbnail } from './thumbnails.js';


22import { importRisuSprites } from './sprites.js';


23import { getUserDirectories } from '../users.js';


24import { getChatInfo } from './chats.js';


25import { ByafParser } from '../byaf.js';


26import { CharXParser, persistCharXAssets } from '../charx.js';


27import cacheBuster from '../middleware/cacheBuster.js';


28


29// With 100 MB limit it would take roughly 3000 characters to reach this limit


30const memoryCacheCapacity = getConfigValue('performance.memoryCacheCapacity', '100mb');


31const memoryCache = new MemoryLimitedMap(memoryCacheCapacity);


32// Some Android devices require tighter memory management


33const isAndroid = process.platform === 'android';


34// Use shallow character data for the character list


35const useShallowCharacters = !!getConfigValue('performance.lazyLoadCharacters', false, 'boolean');


36const useDiskCache = !!getConfigValue('performance.useDiskCache', true, 'boolean');


37


38class DiskCache {


39    /**


40     * @type {string}


41     * @readonly


42     */


43    static DIRECTORY = 'characters';


44


45    /**


46     * @type {number}


47     * @readonly


48     */


49    static SYNC_INTERVAL = 5 * 60 * 1000;


50


51    /** @type {import('node-persist').LocalStorage} */


52    #instance;


53


54    /** @type {NodeJS.Timeout} */


55    #syncInterval;


56


57    /**


58     * Queue of user handles to sync.


59     * @type {Set<string>}


60     * @readonly


61     */


62    syncQueue = new Set();


63


64    /**


65     * Path to the cache directory.


66     * @returns {string}


67     */


68    get cachePath() {


69        return path.join(globalThis.DATA_ROOT, '_cache', DiskCache.DIRECTORY);


70    }


71


72    /**


73     * Returns the list of hashed keys in the cache.


74     * @returns {string[]}


75     */


76    get hashedKeys() {


77        return fs.readdirSync(this.cachePath);


78    }


79


80    /**


81     * Processes the synchronization queue.


82     * @returns {Promise<void>}


83     */


84    async #syncCacheEntries() {


85        try {


86            if (!useDiskCache || this.syncQueue.size === 0) {


87                return;


88            }


89


90            const directories = [...this.syncQueue].map(entry => getUserDirectories(entry));


91            this.syncQueue.clear();


92


93            await this.verify(directories);


94        } catch (error) {


95            console.error('Error while synchronizing cache entries:', error);


96        }


97    }


98


99    /**


100     * Gets the disk cache instance.


101     * @returns {Promise<import('node-persist').LocalStorage>}


102     */


103    async instance() {


104        if (this.#instance) {


105            return this.#instance;


106        }


107


108        this.#instance = storage.create({


109            dir: this.cachePath,


110            ttl: false,


111            forgiveParseErrors: true,


112            expiredInterval: 0,


113            // @ts-ignore


114            maxFileDescriptors: 100,


115        });


116        await this.#instance.init();


117        this.#syncInterval = setInterval(this.#syncCacheEntries.bind(this), DiskCache.SYNC_INTERVAL);


118        return this.#instance;


119    }


120


121    /**


122     * Verifies disk cache size and prunes it if necessary.


123     * @param {import('../users.js').UserDirectoryList[]} directoriesList List of user directories


124     * @returns {Promise<void>}


125     */


126    async verify(directoriesList) {


127        try {


128            if (!useDiskCache) {


129                return;


130            }


131


132            const cache = await this.instance();


133            const validKeys = new Set();


134            for (const dir of directoriesList) {


135                const files = fs.readdirSync(dir.characters, { withFileTypes: true });


136                for (const file of files.filter(f => f.isFile() && path.extname(f.name) === '.png')) {


137                    const filePath = path.join(dir.characters, file.name);


138                    const cacheKey = getCacheKey(filePath);


139                    validKeys.add(path.parse(cache.getDatumPath(cacheKey)).base);


140                }


141            }


142            for (const key of this.hashedKeys) {


143                if (!validKeys.has(key)) {


144                    await cache.removeItem(key);


145                }


146            }


147        } catch (error) {


148            console.error('Error while verifying disk cache:', error);


149        }


150    }


151


152    dispose() {


153        if (this.#syncInterval) {


154            clearInterval(this.#syncInterval);


155        }


156    }


157}


158


159export const diskCache = new DiskCache();


160


161/**


162 * Gets the cache key for the specified image file.


163 * @param {string} inputFile - Path to the image file


164 * @returns {string} - Cache key


165 */


166function getCacheKey(inputFile) {


167    if (fs.existsSync(inputFile)) {


168        const stat = fs.statSync(inputFile);


169        return `${inputFile}-${stat.mtimeMs}`;


170    }


171


172    return inputFile;


173}


174


175/**


176 * Reads the character card from the specified image file.


177 * @param {string} inputFile - Path to the image file


178 * @param {string} inputFormat - 'png'


179 * @returns {Promise<string | undefined>} - Character card data


180 */


181async function readCharacterData(inputFile, inputFormat = 'png') {


182    const cacheKey = getCacheKey(inputFile);


183    if (memoryCache.has(cacheKey)) {


184        return memoryCache.get(cacheKey);


185    }


186    if (useDiskCache) {


187        try {


188            const cache = await diskCache.instance();


189            const cachedData = await cache.getItem(cacheKey);


190            if (cachedData) {


191                return cachedData;


192            }


193        } catch (error) {


194            console.warn('Error while reading from disk cache:', error);


195        }


196    }


197


198    const result = await parse(inputFile, inputFormat);


199    !isAndroid && memoryCache.set(cacheKey, result);


200    if (useDiskCache) {


201        try {


202            const cache = await diskCache.instance();


203            await cache.setItem(cacheKey, result);


204        } catch (error) {


205            console.warn('Error while writing to disk cache:', error);


206        }


207    }


208    return result;


209}


210


211/**


212 * Writes the character card to the specified image file.


213 * @param {string|Buffer} inputFile - Path to the image file or image buffer


214 * @param {string} data - Character card data


215 * @param {string} outputFile - Target image file name


216 * @param {import('express').Request} request - Express request obejct


217 * @param {Crop|undefined} crop - Crop parameters


218 * @returns {Promise<boolean>} - True if the operation was successful


219 */


220async function writeCharacterData(inputFile, data, outputFile, request, crop = undefined) {


221    try {


222        // Reset the cache


223        for (const key of memoryCache.keys()) {


224            if (Buffer.isBuffer(inputFile)) {


225                break;


226            }


227            if (key.startsWith(inputFile)) {


228                memoryCache.delete(key);


229                break;


230            }


231        }


232        if (useDiskCache && !Buffer.isBuffer(inputFile)) {


233            diskCache.syncQueue.add(request.user.profile.handle);


234        }


235        /**


236         * Read the image, resize, and save it as a PNG into the buffer.


237         * @returns {Promise<Buffer>} Image buffer


238         */


239        async function getInputImage() {


240            try {


241                if (Buffer.isBuffer(inputFile)) {


242                    return await parseImageBuffer(inputFile, crop);


243                }


244


245                return await tryReadImage(inputFile, crop);


246            } catch (error) {


247                const message = Buffer.isBuffer(inputFile) ? 'Failed to read image buffer.' : `Failed to read image: ${inputFile}.`;


248                console.warn(message, 'Using a fallback image.', error);


249                return await fs.promises.readFile(DEFAULT_AVATAR_PATH);


250            }


251        }


252


253        const inputImage = await getInputImage();


254


255        // Get the chunks


256        const outputImage = write(inputImage, data);


257        const outputImagePath = path.join(request.user.directories.characters, `${outputFile}.png`);


258


259        writeFileAtomicSync(outputImagePath, outputImage);


260        return true;


261    } catch (err) {


262        console.error(err);


263        return false;


264    }


265}


266


267/**


268 * @typedef {Object} Crop


269 * @property {number} x X-coordinate


270 * @property {number} y Y-coordinate


271 * @property {number} width Width


272 * @property {number} height Height


273 * @property {boolean} want_resize Resize the image to the standard avatar size


274 */


275


276/**


277 * Applies avatar crop and resize operations to an image.


278 * I couldn't fix the type issue, so the first argument has {any} type.


279 * @param {object} jimp Jimp image instance


280 * @param {Crop|undefined} [crop] Crop parameters


281 * @returns {Promise<Buffer>} Processed image buffer


282 */


283export async function applyAvatarCropResize(jimp, crop) {


284    if (!(jimp instanceof Jimp)) {


285        throw new TypeError('Expected a Jimp instance');


286    }


287


288    const image = /** @type {InstanceType<typeof Jimp>} */ (jimp);


289    let finalWidth = image.bitmap.width, finalHeight = image.bitmap.height;


290


291    // Apply crop if defined


292    if (typeof crop == 'object' && [crop.x, crop.y, crop.width, crop.height].every(x => typeof x === 'number')) {


293        image.crop({ x: crop.x, y: crop.y, w: crop.width, h: crop.height });


294        // Apply standard resize if requested


295        if (crop.want_resize) {


296            finalWidth = AVATAR_WIDTH;


297            finalHeight = AVATAR_HEIGHT;


298        } else {


299            finalWidth = crop.width;


300            finalHeight = crop.height;


301        }


302    }


303


304    image.cover({ w: finalWidth, h: finalHeight });


305    return await image.getBuffer(JimpMime.png);


306}


307


308/**


309 * Parses an image buffer and applies crop if defined.


310 * @param {Buffer} buffer Buffer of the image


311 * @param {Crop|undefined} [crop] Crop parameters


312 * @returns {Promise<Buffer>} Image buffer


313 */


314async function parseImageBuffer(buffer, crop) {


315    const image = await Jimp.fromBuffer(buffer);


316    return await applyAvatarCropResize(image, crop);


317}


318


319/**


320 * Reads an image file and applies crop if defined.


321 * @param {string} imgPath Path to the image file


322 * @param {Crop|undefined} crop Crop parameters


323 * @returns {Promise<Buffer>} Image buffer


324 */


325async function tryReadImage(imgPath, crop) {


326    try {


327        const rawImg = await Jimp.read(imgPath);


328        return await applyAvatarCropResize(rawImg, crop);


329    } catch (error) {


330        // If it's an unsupported type of image (APNG) - just read the file as buffer


331        console.error(`Failed to read image: ${imgPath}`, error);


332        return fs.readFileSync(imgPath);


333    }


334}


335


336/**


337 * calculateChatSize - Calculates the total chat size for a given character.


338 *


339 * @param  {string} charDir The directory where the chats are stored.


340 * @return { {chatSize: number, dateLastChat: number} }         The total chat size.


341 */


342const calculateChatSize = (charDir) => {


343    let chatSize = 0;


344    let dateLastChat = 0;


345


346    if (fs.existsSync(charDir)) {


347        const chats = fs.readdirSync(charDir);


348        if (Array.isArray(chats) && chats.length) {


349            for (const chat of chats) {


350                const chatStat = fs.statSync(path.join(charDir, chat));


351                chatSize += chatStat.size;


352                dateLastChat = Math.max(dateLastChat, chatStat.mtimeMs);


353            }


354        }


355    }


356


357    return { chatSize, dateLastChat };


358};


359


360// Calculate the total string length of the data object


361const calculateDataSize = (data) => {


362    return typeof data === 'object' ? Object.values(data).reduce((acc, val) => acc + String(val).length, 0) : 0;


363};


364


365/**


366 * Only get fields that are used to display the character list.


367 * @param {object} character Character object


368 * @returns {{shallow: true, [key: string]: any}} Shallow character


369 */


370const toShallow = (character) => {


371    return {


372        shallow: true,


373        name: character.name,


374        avatar: character.avatar,


375        chat: character.chat,


376        fav: character.fav,


377        date_added: character.date_added,


378        create_date: character.create_date,


379        date_last_chat: character.date_last_chat,


380        chat_size: character.chat_size,


381        data_size: character.data_size,


382        tags: character.tags,


383        data: {


384            name: _.get(character, 'data.name', ''),


385            character_version: _.get(character, 'data.character_version', ''),


386            creator: _.get(character, 'data.creator', ''),


387            creator_notes: _.get(character, 'data.creator_notes', ''),


388            tags: _.get(character, 'data.tags', []),


389            extensions: {


390                fav: _.get(character, 'data.extensions.fav', false),


391                world: _.get(character, 'data.extensions.world', ''),


392            },


393        },


394    };


395};


396


397/**


398 * processCharacter - Process a given character, read its data and calculate its statistics.


399 *


400 * @param  {string} item The name of the character.


401 * @param  {import('../users.js').UserDirectoryList} directories User directories


402 * @param  {object} options Options for the character processing


403 * @param  {boolean} options.shallow If true, only return the core character's metadata


404 * @return {Promise<object>}     A Promise that resolves when the character processing is done.


405 */


406const processCharacter = async (item, directories, { shallow }) => {


407    try {


408        const imgFile = path.join(directories.characters, item);


409        const imgData = await readCharacterData(imgFile);


410        if (imgData === undefined) throw new Error('Failed to read character file');


411


412        let jsonObject = getCharaCardV2(JSON.parse(imgData), directories, false);


413        jsonObject.avatar = item;


414        const character = jsonObject;


415        character.json_data = imgData;


416        const charStat = fs.statSync(path.join(directories.characters, item));


417        character.date_added = charStat.ctimeMs;


418        character.create_date = jsonObject.create_date || new Date(Math.round(charStat.ctimeMs)).toISOString();


419        const chatsDirectory = path.join(directories.chats, item.replace('.png', ''));


420


421        const { chatSize, dateLastChat } = calculateChatSize(chatsDirectory);


422        character.chat_size = chatSize;


423        character.date_last_chat = dateLastChat;


424        character.data_size = calculateDataSize(jsonObject?.data);


425        return shallow ? toShallow(character) : character;


426    } catch (err) {


427        console.error(`Could not process character: ${item}`);


428


429        if (err instanceof SyntaxError) {


430            console.error(`${item} does not contain a valid JSON object.`);


431        } else {


432            console.error('An unexpected error occurred: ', err);


433        }


434


435        return {


436            date_added: 0,


437            date_last_chat: 0,


438            chat_size: 0,


439        };


440    }


441};


442


443/**


444 * Convert a character object to Spec V2 format.


445 * @param {object} jsonObject Character object


446 * @param {import('../users.js').UserDirectoryList} directories User directories


447 * @param {boolean} hoistDate Will set the chat and create_date fields to the current date if they are missing


448 * @returns {object} Character object in Spec V2 format


449 */


450function getCharaCardV2(jsonObject, directories, hoistDate = true) {


451    if (jsonObject.spec === undefined) {


452        jsonObject = convertToV2(jsonObject, directories);


453


454        if (hoistDate && !jsonObject.create_date) {


455            jsonObject.create_date = new Date().toISOString();


456        }


457    } else {


458        jsonObject = readFromV2(jsonObject);


459    }


460    return jsonObject;


461}


462


463/**


464 * Convert a character object to Spec V2 format.


465 * @param {object} char Character object


466 * @param {import('../users.js').UserDirectoryList} directories User directories


467 * @returns {object} Character object in Spec V2 format


468 */


469function convertToV2(char, directories) {


470    // Simulate incoming data from frontend form


471    const result = charaFormatData({


472        json_data: JSON.stringify(char),


473        ch_name: char.name,


474        description: char.description,


475        personality: char.personality,


476        scenario: char.scenario,


477        first_mes: char.first_mes,


478        mes_example: char.mes_example,


479        creator_notes: char.creatorcomment,


480        talkativeness: char.talkativeness,


481        fav: char.fav,


482        creator: char.creator,


483        tags: char.tags,


484        depth_prompt_prompt: char.depth_prompt_prompt,


485        depth_prompt_depth: char.depth_prompt_depth,


486        depth_prompt_role: char.depth_prompt_role,


487    }, directories);


488


489    result.chat = char.chat ?? `${char.name} - ${humanizedDateTime()}`;


490    result.create_date = char.create_date;


491


492    return result;


493}


494


495/**


496 * Removes fields that are not meant to be shared.


497 */


498function unsetPrivateFields(char) {


499    _.set(char, 'fav', false);


500    _.set(char, 'data.extensions.fav', false);


501    _.unset(char, 'chat');


502}


503


504function readFromV2(char) {


505    if (_.isUndefined(char.data)) {


506        console.warn(`Char ${char.name} has Spec v2 data missing`);


507        return char;


508    }


509


510    // If 'json_data' was already saved, don't let it propagate


511    _.unset(char, 'json_data');


512


513    const fieldMappings = {


514        name: 'name',


515        description: 'description',


516        personality: 'personality',


517        scenario: 'scenario',


518        first_mes: 'first_mes',


519        mes_example: 'mes_example',


520        talkativeness: 'extensions.talkativeness',


521        fav: 'extensions.fav',


522        tags: 'tags',


523    };


524


525    _.forEach(fieldMappings, (v2Path, charField) => {


526        //console.info(`Migrating field: ${charField} from ${v2Path}`);


527        const v2Value = _.get(char.data, v2Path);


528        if (_.isUndefined(v2Value)) {


529            let defaultValue = undefined;


530


531            // Backfill default values for missing ST extension fields


532            if (v2Path === 'extensions.talkativeness') {


533                defaultValue = 0.5;


534            }


535


536            if (v2Path === 'extensions.fav') {


537                defaultValue = false;


538            }


539


540            if (!_.isUndefined(defaultValue)) {


541                //console.warn(`Spec v2 extension data missing for field: ${charField}, using default value: ${defaultValue}`);


542                char[charField] = defaultValue;


543            } else {


544                console.warn(`Char ${char.name} has Spec v2 data missing for unknown field: ${charField}`);


545                return;


546            }


547        }


548        if (!_.isUndefined(char[charField]) && !_.isUndefined(v2Value) && String(char[charField]) !== String(v2Value)) {


549            console.warn(`Char ${char.name} has Spec v2 data mismatch with Spec v1 for field: ${charField}`, char[charField], v2Value);


550        }


551        char[charField] = v2Value;


552    });


553


554    char.chat = char.chat ?? `${char.name} - ${humanizedDateTime()}`;


555


556    return char;


557}


558


559/**


560 * Format character data to Spec V2 format.


561 * @param {object} data Character data


562 * @param {import('../users.js').UserDirectoryList} directories User directories


563 * @returns


564 */


565function charaFormatData(data, directories) {


566    // This is supposed to save all the foreign keys that ST doesn't care about


567    const char = tryParse(data.json_data) || {};


568


569    // Prevent erroneous 'json_data' recursive saving


570    _.unset(char, 'json_data');


571


572    // Checks if data.alternate_greetings is an array, a string, or neither, and acts accordingly. (expected to be an array of strings)


573    const getAlternateGreetings = data => {


574        if (Array.isArray(data.alternate_greetings)) return data.alternate_greetings;


575        if (typeof data.alternate_greetings === 'string') return [data.alternate_greetings];


576        return [];


577    };


578


579    // Spec V1 fields


580    _.set(char, 'name', data.ch_name);


581    _.set(char, 'description', data.description || '');


582    _.set(char, 'personality', data.personality || '');


583    _.set(char, 'scenario', data.scenario || '');


584    _.set(char, 'first_mes', data.first_mes || '');


585    _.set(char, 'mes_example', data.mes_example || '');


586


587    // Old ST extension fields (for backward compatibility, will be deprecated)


588    _.set(char, 'creatorcomment', data.creator_notes || '');


589    _.set(char, 'avatar', 'none');


590    _.set(char, 'chat', data.ch_name + ' - ' + humanizedDateTime());


591    _.set(char, 'talkativeness', data.talkativeness || 0.5);


592    _.set(char, 'fav', data.fav == 'true');


593    _.set(char, 'tags', typeof data.tags == 'string' ? (data.tags.split(',').map(x => x.trim()).filter(x => x)) : data.tags || []);


594


595    // Spec V2 fields


596    _.set(char, 'spec', 'chara_card_v2');


597    _.set(char, 'spec_version', '2.0');


598    _.set(char, 'data.name', data.ch_name);


599    _.set(char, 'data.description', data.description || '');


600    _.set(char, 'data.personality', data.personality || '');


601    _.set(char, 'data.scenario', data.scenario || '');


602    _.set(char, 'data.first_mes', data.first_mes || '');


603    _.set(char, 'data.mes_example', data.mes_example || '');


604


605    // New V2 fields


606    _.set(char, 'data.creator_notes', data.creator_notes || '');


607    _.set(char, 'data.system_prompt', data.system_prompt || '');


608    _.set(char, 'data.post_history_instructions', data.post_history_instructions || '');


609    _.set(char, 'data.tags', typeof data.tags == 'string' ? (data.tags.split(',').map(x => x.trim()).filter(x => x)) : data.tags || []);


610    _.set(char, 'data.creator', data.creator || '');


611    _.set(char, 'data.character_version', data.character_version || '');


612    _.set(char, 'data.alternate_greetings', getAlternateGreetings(data));


613


614    // ST extension fields to V2 object


615    _.set(char, 'data.extensions.talkativeness', data.talkativeness || 0.5);


616    _.set(char, 'data.extensions.fav', data.fav == 'true');


617    _.set(char, 'data.extensions.world', data.world || '');


618


619    // Spec extension: depth prompt


620    const depth_default = 4;


621    const role_default = 'system';


622    const depth_value = !isNaN(Number(data.depth_prompt_depth)) ? Number(data.depth_prompt_depth) : depth_default;


623    const role_value = data.depth_prompt_role ?? role_default;


624    _.set(char, 'data.extensions.depth_prompt.prompt', data.depth_prompt_prompt ?? '');


625    _.set(char, 'data.extensions.depth_prompt.depth', depth_value);


626    _.set(char, 'data.extensions.depth_prompt.role', role_value);


627


628    if (data.world) {


629        try {


630            const file = readWorldInfoFile(directories, data.world, false);


631


632            // File was imported - save it to the character book


633            if (file && file.originalData) {


634                _.set(char, 'data.character_book', file.originalData);


635            }


636


637            // File was not imported - convert the world info to the character book


638            if (file && file.entries) {


639                _.set(char, 'data.character_book', convertWorldInfoToCharacterBook(data.world, file.entries));


640            }


641        } catch {


642            console.warn(`Failed to read world info file: ${data.world}. Character book will not be available.`);


643        }


644    }


645


646    if (data.extensions) {


647        try {


648            const extensions = JSON.parse(data.extensions);


649            // Deep merge the extensions object


650            _.set(char, 'data.extensions', deepMerge(char.data.extensions, extensions));


651        } catch {


652            console.warn(`Failed to parse extensions JSON: ${data.extensions}`);


653        }


654    }


655


656    return char;


657}


658


659/**


660 * @param {string} name Name of World Info file


661 * @param {object} entries Entries object


662 */


663function convertWorldInfoToCharacterBook(name, entries) {


664    /** @type {{ entries: object[]; name: string }} */


665    const result = { entries: [], name };


666


667    for (const index in entries) {


668        const entry = entries[index];


669


670        const originalEntry = {


671            id: entry.uid,


672            keys: entry.key,


673            secondary_keys: entry.keysecondary,


674            comment: entry.comment,


675            content: entry.content,


676            constant: entry.constant,


677            selective: entry.selective,


678            insertion_order: entry.order,


679            enabled: !entry.disable,


680            position: entry.position == 0 ? 'before_char' : 'after_char',


681            use_regex: true, // ST keys are always regex


682            extensions: {


683                ...entry.extensions,


684                position: entry.position,


685                exclude_recursion: entry.excludeRecursion,


686                display_index: entry.displayIndex,


687                probability: entry.probability ?? null,


688                useProbability: entry.useProbability ?? false,


689                depth: entry.depth ?? 4,


690                selectiveLogic: entry.selectiveLogic ?? 0,


691                outlet_name: entry.outletName ?? '',


692                group: entry.group ?? '',


693                group_override: entry.groupOverride ?? false,


694                group_weight: entry.groupWeight ?? null,


695                prevent_recursion: entry.preventRecursion ?? false,


696                delay_until_recursion: entry.delayUntilRecursion ?? false,


697                scan_depth: entry.scanDepth ?? null,


698                match_whole_words: entry.matchWholeWords ?? null,


699                use_group_scoring: entry.useGroupScoring ?? false,


700                case_sensitive: entry.caseSensitive ?? null,


701                automation_id: entry.automationId ?? '',


702                role: entry.role ?? 0,


703                vectorized: entry.vectorized ?? false,


704                sticky: entry.sticky ?? null,


705                cooldown: entry.cooldown ?? null,


706                delay: entry.delay ?? null,


707                match_persona_description: entry.matchPersonaDescription ?? false,


708                match_character_description: entry.matchCharacterDescription ?? false,


709                match_character_personality: entry.matchCharacterPersonality ?? false,


710                match_character_depth_prompt: entry.matchCharacterDepthPrompt ?? false,


711                match_scenario: entry.matchScenario ?? false,


712                match_creator_notes: entry.matchCreatorNotes ?? false,


713                triggers: entry.triggers ?? [],


714                ignore_budget: entry.ignoreBudget ?? false,


715            },


716        };


717


718        result.entries.push(originalEntry);


719    }


720


721    return result;


722}


723


724/**


725 * Import a character from a YAML file.


726 * @param {string} uploadPath Path to the uploaded file


727 * @param {{ request: import('express').Request, response: import('express').Response }} context Express request and response objects


728 * @param {string|undefined} preservedFileName Preserved file name


729 * @returns {Promise<string>} Internal name of the character


730 */


731async function importFromYaml(uploadPath, context, preservedFileName) {


732    const fileText = fs.readFileSync(uploadPath, 'utf8');


733    fs.unlinkSync(uploadPath);


734    const yamlData = yaml.parse(fileText);


735    console.info('Importing from YAML');


736    yamlData.name = sanitize(yamlData.name);


737    const fileName = preservedFileName || getPngName(yamlData.name, context.request.user.directories);


738    let char = convertToV2({


739        'name': yamlData.name,


740        'description': yamlData.context ?? '',


741        'first_mes': yamlData.greeting ?? '',


742        'create_date': new Date().toISOString(),


743        'chat': `${yamlData.name} - ${humanizedDateTime()}`,


744        'personality': '',


745        'creatorcomment': '',


746        'avatar': 'none',


747        'mes_example': '',


748        'scenario': '',


749        'talkativeness': 0.5,


750        'creator': '',


751        'tags': '',


752    }, context.request.user.directories);


753    const result = await writeCharacterData(DEFAULT_AVATAR_PATH, JSON.stringify(char), fileName, context.request);


754    return result ? fileName : '';


755}


756


757/**


758 * Imports a character card from CharX (ZIP) file.


759 * @param {string} uploadPath


760 * @param {object} params


761 * @param {import('express').Request} params.request


762 * @param {string|undefined} preservedFileName Preserved file name


763 * @returns {Promise<string>} Internal name of the character


764 */


765async function importFromCharX(uploadPath, { request }, preservedFileName) {


766    const fileBuffer = fs.readFileSync(uploadPath);


767    // Create a properly-sized ArrayBuffer (Node's buffer pool can cause oversized .buffer)


768    const data = fileBuffer.buffer.slice(fileBuffer.byteOffset, fileBuffer.byteOffset + fileBuffer.byteLength);


769    fs.unlinkSync(uploadPath);


770


771    const parser = new CharXParser(data);


772    const { card, avatar, auxiliaryAssets, extractedBuffers } = await parser.parse();


773


774    // Apply standard character transformations


775    if (card.data?.name) {


776        card.data.name = sanitize(card.data.name);


777    }


778    card.name = sanitize(card.data?.name || card.name);


779    let processedCard = readFromV2(card);


780    unsetPrivateFields(processedCard);


781    processedCard.create_date = new Date().toISOString();


782


783    const fileName = preservedFileName || getPngName(processedCard.name, request.user.directories);


784    // Use the actual character name for asset folders, not the unique filename


785    // ST's sprite system looks up by character name, not PNG filename


786    const characterFolder = processedCard.name;


787


788    if (auxiliaryAssets.length > 0) {


789        try {


790            const summary = persistCharXAssets(auxiliaryAssets, extractedBuffers, request.user.directories, characterFolder);


791            if (summary.sprites || summary.backgrounds || summary.misc) {


792                console.log(`CharX: Imported ${summary.sprites} sprite(s), ${summary.backgrounds} background(s), ${summary.misc} misc asset(s) for ${characterFolder}`);


793            }


794        } catch (error) {


795            console.warn(`CharX: Failed to persist auxiliary assets for ${characterFolder}`, error);


796        }


797    }


798


799    const result = await writeCharacterData(avatar, JSON.stringify(processedCard), fileName, request);


800    return result ? fileName : '';


801}


802


803async function importFromByaf(uploadPath, { request }, preservedFileName) {


804    const data = (await fsPromises.readFile(uploadPath)).buffer;


805    await fsPromises.unlink(uploadPath);


806    console.info('Importing from BYAF');


807


808    const byafData = await new ByafParser(data).parse();


809    const card = readFromV2(byafData.card);


810    const fileName = preservedFileName || getPngName(sanitize(byafData.character.displayName || card.name, { replacement: sanitizeSafeCharacterReplacements }), request.user.directories);


811


812    // Don't import chats and images if the character is being replaced or updated, instead of newly imported.


813    if (!preservedFileName) {


814        /**


815         * @param {Partial<ByafScenario>} scenario


816        */


817        const createChatAsCurrentPersona = (scenario) => {


818            const chatName = sanitize(`${scenario.title || card.name} - ${humanizedDateTime()} imported.jsonl`, { replacement: sanitizeSafeCharacterReplacements });


819            const filePath = path.join(request.user.directories.chats, path.basename(fileName), chatName);


820            const dir = path.dirname(filePath);


821            if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });


822            writeFileAtomicSync(filePath, ByafParser.getChatFromScenario(scenario, request.body.user_name, card.name, byafData.chatBackgrounds), 'utf8');


823            console.log(`Created ${chatName} chat from BYAF import`);


824            return chatName;


825        };


826


827        // Upload backgrounds


828        for (const bg of byafData.chatBackgrounds) {


829            const extension = path.extname(bg.paths?.[0]) || '.png';


830            const baseName = `${path.basename(fileName)}_bg`;


831            const filePath = path.join(request.user.directories.userImages, fileName);


832            if (!fs.existsSync(filePath)) fs.mkdirSync(filePath, { recursive: true });


833            const file = getUniqueName(baseName, (name) => fs.existsSync(path.join(filePath, `${name}${extension}`)));


834            if (Buffer.isBuffer(bg.data)) {


835                const newFile = `${file}${extension}`;


836                writeFileAtomicSync(path.join(filePath, newFile), bg.data);


837                bg.name = clientRelativePath(request.user.directories.root, path.join(filePath, newFile)); // Update background name to the new file


838                console.log(`Created ${newFile} background from BYAF import`);


839            }


840        }


841


842        const chats = [];


843        // Create chats for each scenario


844        if (Array.isArray(byafData.scenarios)) {


845            for (const scenario of byafData.scenarios) {


846                chats.push(createChatAsCurrentPersona(scenario));


847            }


848        }


849


850        // Update the default chat if there are any so we open to an existing chat instead of creating a new one and opening that.


851        if (chats.length > 0) {


852            card.chat = path.basename(chats[0], path.extname(chats[0]));


853        }


854


855        // Save alternate icons for the character.


856        for (const icon of byafData.images.slice(1)) {


857            // BYAF does not support character expressions, so using the same structure will not result in conflicts,


858            // even if the expression system did not tolerate additional icons that are not mapped to expressions.


859            // This will not yet allow changing icons within the UI but at least the icons will be available for manual selection, rather than being lost.


860            const altImagesFolder = path.join(request.user.directories.characters, sanitize(card.name));


861            if (!fs.existsSync(altImagesFolder)) fs.mkdirSync(altImagesFolder, { recursive: true });


862            const extension = path.extname(icon.filename) || '.png';


863            const file = getUniqueName(`${sanitize(icon.label, { replacement: sanitizeSafeCharacterReplacements }) || 'alt'}`, (name) => fs.existsSync(path.join(altImagesFolder, `${name}${extension}`)));


864            if (Buffer.isBuffer(icon.image)) {


865                writeFileAtomicSync(path.join(altImagesFolder, `${file}${extension}`), icon.image);


866                console.log(`Created ${file}${extension} alternate icon from BYAF import`);


867            }


868        }


869    }


870


871    const result = await writeCharacterData(byafData.images[0].image, JSON.stringify(card), fileName, request);


872


873    return result ? fileName : '';


874}


875


876/**


877 * Import a character from a JSON file.


878 * @param {string} uploadPath Path to the uploaded file


879 * @param {{ request: import('express').Request, response: import('express').Response }} context Express request and response objects


880 * @param {string|undefined} preservedFileName Preserved file name


881 * @returns {Promise<string>} Internal name of the character


882 */


883async function importFromJson(uploadPath, { request }, preservedFileName) {


884    const data = fs.readFileSync(uploadPath, 'utf8');


885    fs.unlinkSync(uploadPath);


886


887    let jsonData = JSON.parse(data);


888


889    if (jsonData.spec !== undefined) {


890        console.info(`Importing from ${jsonData.spec} json`);


891        importRisuSprites(request.user.directories, jsonData);


892        unsetPrivateFields(jsonData);


893        if (jsonData.data?.name) {


894            jsonData.data.name = sanitize(jsonData.data.name);


895        }


896        jsonData.name = sanitize(jsonData.data?.name || jsonData.name);


897        jsonData = readFromV2(jsonData);


898        jsonData.create_date = new Date().toISOString();


899        const pngName = preservedFileName || getPngName(jsonData.name, request.user.directories);


900        const char = JSON.stringify(jsonData);


901        const result = await writeCharacterData(DEFAULT_AVATAR_PATH, char, pngName, request);


902        return result ? pngName : '';


903    } else if (jsonData.name !== undefined) {


904        console.info('Importing from v1 json');


905        jsonData.name = sanitize(jsonData.name);


906        if (jsonData.creator_notes) {


907            jsonData.creator_notes = jsonData.creator_notes.replace('Creator\'s notes go here.', '');


908        }


909        const pngName = preservedFileName || getPngName(jsonData.name, request.user.directories);


910        let char = {


911            'name': jsonData.name,


912            'description': jsonData.description ?? '',


913            'creatorcomment': jsonData.creatorcomment ?? jsonData.creator_notes ?? '',


914            'personality': jsonData.personality ?? '',


915            'first_mes': jsonData.first_mes ?? '',


916            'avatar': 'none',


917            'chat': jsonData.name + ' - ' + humanizedDateTime(),


918            'mes_example': jsonData.mes_example ?? '',


919            'scenario': jsonData.scenario ?? '',


920            'create_date': new Date().toISOString(),


921            'talkativeness': jsonData.talkativeness ?? 0.5,


922            'creator': jsonData.creator ?? '',


923            'tags': jsonData.tags ?? '',


924        };


925        char = convertToV2(char, request.user.directories);


926        let charJSON = JSON.stringify(char);


927        const result = await writeCharacterData(DEFAULT_AVATAR_PATH, charJSON, pngName, request);


928        return result ? pngName : '';


929    } else if (jsonData.char_name !== undefined) {


930        //json Pygmalion notepad


931        console.info('Importing from gradio json');


932        jsonData.char_name = sanitize(jsonData.char_name);


933        if (jsonData.creator_notes) {


934            jsonData.creator_notes = jsonData.creator_notes.replace('Creator\'s notes go here.', '');


935        }


936        const pngName = preservedFileName || getPngName(jsonData.char_name, request.user.directories);


937        let char = {


938            'name': jsonData.char_name,


939            'description': jsonData.char_persona ?? '',


940            'creatorcomment': jsonData.creatorcomment ?? jsonData.creator_notes ?? '',


941            'personality': '',


942            'first_mes': jsonData.char_greeting ?? '',


943            'avatar': 'none',


944            'chat': jsonData.name + ' - ' + humanizedDateTime(),


945            'mes_example': jsonData.example_dialogue ?? '',


946            'scenario': jsonData.world_scenario ?? '',


947            'create_date': new Date().toISOString(),


948            'talkativeness': jsonData.talkativeness ?? 0.5,


949            'creator': jsonData.creator ?? '',


950            'tags': jsonData.tags ?? '',


951        };


952        char = convertToV2(char, request.user.directories);


953        const charJSON = JSON.stringify(char);


954        const result = await writeCharacterData(DEFAULT_AVATAR_PATH, charJSON, pngName, request);


955        return result ? pngName : '';


956    }


957


958    return '';


959}


960


961/**


962 * Import a character from a PNG file.


963 * @param {string} uploadPath Path to the uploaded file


964 * @param {{ request: import('express').Request, response: import('express').Response }} context Express request and response objects


965 * @param {string|undefined} preservedFileName Preserved file name


966 * @returns {Promise<string>} Internal name of the character


967 */


968async function importFromPng(uploadPath, { request }, preservedFileName) {


969    const imgData = await readCharacterData(uploadPath);


970    if (imgData === undefined) throw new Error('Failed to read character data');


971


972    let jsonData = JSON.parse(imgData);


973


974    if (jsonData.data?.name) {


975        jsonData.data.name = sanitize(jsonData.data.name);


976    }


977    jsonData.name = sanitize(jsonData.data?.name || jsonData.name);


978    const pngName = preservedFileName || getPngName(jsonData.name, request.user.directories);


979


980    if (jsonData.spec !== undefined) {


981        console.info(`Found a ${jsonData.spec} character file.`);


982        importRisuSprites(request.user.directories, jsonData);


983        unsetPrivateFields(jsonData);


984        jsonData = readFromV2(jsonData);


985        jsonData.create_date = new Date().toISOString();


986        const char = JSON.stringify(jsonData);


987        const result = await writeCharacterData(uploadPath, char, pngName, request);


988        fs.unlinkSync(uploadPath);


989        return result ? pngName : '';


990    } else if (jsonData.name !== undefined) {


991        console.info('Found a v1 character file.');


992


993        if (jsonData.creator_notes) {


994            jsonData.creator_notes = jsonData.creator_notes.replace('Creator\'s notes go here.', '');


995        }


996


997        let char = {


998            'name': jsonData.name,


999            'description': jsonData.description ?? '',


1000            'creatorcomment': jsonData.creatorcomment ?? jsonData.creator_notes ?? '',


1001            'personality': jsonData.personality ?? '',


1002            'first_mes': jsonData.first_mes ?? '',


1003            'avatar': 'none',


1004            'chat': jsonData.name + ' - ' + humanizedDateTime(),


1005            'mes_example': jsonData.mes_example ?? '',


1006            'scenario': jsonData.scenario ?? '',


1007            'create_date': new Date().toISOString(),


1008            'talkativeness': jsonData.talkativeness ?? 0.5,


1009            'creator': jsonData.creator ?? '',


1010            'tags': jsonData.tags ?? '',


1011        };


1012        char = convertToV2(char, request.user.directories);


1013        const charJSON = JSON.stringify(char);


1014        const result = await writeCharacterData(uploadPath, charJSON, pngName, request);


1015        fs.unlinkSync(uploadPath);


1016        return result ? pngName : '';


1017    }


1018


1019    return '';


1020}


1021


1022export const router = express.Router();


1023


1024router.post('/create', getFileNameValidationFunction('file_name'), async function (request, response) {


1025    try {


1026        if (!request.body) return response.sendStatus(400);


1027


1028        request.body.ch_name = sanitize(request.body.ch_name);


1029


1030        const char = JSON.stringify(charaFormatData(request.body, request.user.directories));


1031        const internalName = request.body.file_name || getPngName(request.body.ch_name, request.user.directories);


1032        const avatarName = `${internalName}.png`;


1033        const chatsPath = path.join(request.user.directories.chats, internalName);


1034


1035        if (!fs.existsSync(chatsPath)) fs.mkdirSync(chatsPath);


1036


1037        if (!request.file) {


1038            await writeCharacterData(DEFAULT_AVATAR_PATH, char, internalName, request);


1039            return response.send(avatarName);


1040        } else {


1041            const crop = tryParse(request.query.crop);


1042            const uploadPath = path.join(request.file.destination, request.file.filename);


1043            await writeCharacterData(uploadPath, char, internalName, request, crop);


1044            fs.unlinkSync(uploadPath);


1045            return response.send(avatarName);


1046        }


1047    } catch (err) {


1048        console.error(err);


1049        response.sendStatus(500);


1050    }


1051});


1052


1053router.post('/rename', validateAvatarUrlMiddleware, async function (request, response) {


1054    if (!request.body.avatar_url || !request.body.new_name) {


1055        return response.sendStatus(400);


1056    }


1057


1058    const oldAvatarName = request.body.avatar_url;


1059    const newName = sanitize(request.body.new_name);


1060    const oldInternalName = path.parse(request.body.avatar_url).name;


1061    const newInternalName = getPngName(newName, request.user.directories);


1062    const newAvatarName = `${newInternalName}.png`;


1063


1064    const oldAvatarPath = path.join(request.user.directories.characters, oldAvatarName);


1065


1066    const oldChatsPath = path.join(request.user.directories.chats, oldInternalName);


1067    const newChatsPath = path.join(request.user.directories.chats, newInternalName);


1068


1069    try {


1070        // Read old file, replace name int it


1071        const rawOldData = await readCharacterData(oldAvatarPath);


1072        if (rawOldData === undefined) throw new Error('Failed to read character file');


1073


1074        const oldData = getCharaCardV2(JSON.parse(rawOldData), request.user.directories);


1075        _.set(oldData, 'data.name', newName);


1076        _.set(oldData, 'name', newName);


1077        const newData = JSON.stringify(oldData);


1078


1079        // Write data to new location


1080        await writeCharacterData(oldAvatarPath, newData, newInternalName, request);


1081


1082        // Rename chats folder


1083        if (fs.existsSync(oldChatsPath) && !fs.existsSync(newChatsPath)) {


1084            fs.cpSync(oldChatsPath, newChatsPath, { recursive: true });


1085            fs.rmSync(oldChatsPath, { recursive: true, force: true });


1086        }


1087


1088        // Remove the old character file


1089        fs.unlinkSync(oldAvatarPath);


1090


1091        // Return new avatar name to ST


1092        return response.send({ avatar: newAvatarName });


1093    } catch (err) {


1094        console.error(err);


1095        return response.sendStatus(500);


1096    }


1097});


1098


1099router.post('/edit', validateAvatarUrlMiddleware, async function (request, response) {


1100    if (!request.body) {


1101        console.warn('Error: no response body detected');


1102        response.status(400).send('Error: no response body detected');


1103        return;


1104    }


1105


1106    if (request.body.ch_name === '' || request.body.ch_name === undefined || request.body.ch_name === '.') {


1107        console.warn('Error: invalid name.');


1108        response.status(400).send('Error: invalid name.');


1109        return;


1110    }


1111


1112    let char = charaFormatData(request.body, request.user.directories);


1113    char.chat = request.body.chat;


1114    char.create_date = request.body.create_date;


1115    char = JSON.stringify(char);


1116    let targetFile = (request.body.avatar_url).replace('.png', '');


1117


1118    try {


1119        if (!request.file) {


1120            const avatarPath = path.join(request.user.directories.characters, request.body.avatar_url);


1121            await writeCharacterData(avatarPath, char, targetFile, request);


1122        } else {


1123            const crop = tryParse(request.query.crop);


1124            const newAvatarPath = path.join(request.file.destination, request.file.filename);


1125            invalidateThumbnail(request.user.directories, 'avatar', request.body.avatar_url);


1126            await writeCharacterData(newAvatarPath, char, targetFile, request, crop);


1127            fs.unlinkSync(newAvatarPath);


1128


1129            // Bust cache to reload the new avatar


1130            cacheBuster.bust(request, response);


1131        }


1132


1133        return response.sendStatus(200);


1134    } catch (err) {


1135        console.error('An error occurred, character edit invalidated.', err);


1136        return response.sendStatus(500);


1137    }


1138});


1139


1140router.post('/edit-avatar', validateAvatarUrlMiddleware, async function (request, response) {


1141    try {


1142        if (!request.file) {


1143            return response.status(400).send('Error: no file uploaded');


1144        }


1145


1146        if (!request.body || !request.body.avatar_url) {


1147            return response.status(400).send('Error: no avatar_url in request body');


1148        }


1149


1150        const uploadPath = path.join(request.file.destination, request.file.filename);


1151        if (!fs.existsSync(uploadPath)) {


1152            return response.status(400).send('Error: uploaded file does not exist');


1153        }


1154        const characterPath = path.join(request.user.directories.characters, request.body.avatar_url);


1155        if (!fs.existsSync(characterPath)) {


1156            return response.status(400).send('Error: character file does not exist');


1157        }


1158        const data = await readCharacterData(characterPath);


1159        if (!data) {


1160            return response.status(400).send('Error: failed to read character data');


1161        }


1162


1163        const crop = tryParse(request.query.crop);


1164        const fileName = request.body.avatar_url.replace('.png', '');


1165        await writeCharacterData(uploadPath, data, fileName, request, crop);


1166


1167        // Remove uploaded temp file


1168        fs.unlinkSync(uploadPath);


1169


1170        // Reset images caches


1171        cacheBuster.bust(request, response);


1172        invalidateThumbnail(request.user.directories, 'avatar', request.body.avatar_url);


1173


1174        return response.sendStatus(200);


1175    } catch (err) {


1176        console.error('An error occurred while editing avatar', err);


1177        return response.sendStatus(500);


1178    }


1179});


1180


1181/**


1182 * Handle a POST request to edit a character attribute.


1183 *


1184 * This function reads the character data from a file, updates the specified attribute,


1185 * and writes the updated data back to the file.


1186 *


1187 * @param {Object} request - The HTTP request object.


1188 * @param {Object} response - The HTTP response object.


1189 * @returns {void}


1190 */


1191router.post('/edit-attribute', validateAvatarUrlMiddleware, async function (request, response) {


1192    console.debug(request.body);


1193    if (!request.body) {


1194        console.warn('Error: no response body detected');


1195        return response.status(400).send('Error: no response body detected');


1196    }


1197


1198    if (request.body.ch_name === '' || request.body.ch_name === undefined || request.body.ch_name === '.') {


1199        console.warn('Error: invalid name.');


1200        return response.status(400).send('Error: invalid name.');


1201    }


1202


1203    if (request.body.field === 'json_data') {


1204        console.warn('Error: cannot edit json_data field.');


1205        return response.status(400).send('Error: cannot edit json_data field.');


1206    }


1207


1208    try {


1209        const avatarPath = path.join(request.user.directories.characters, request.body.avatar_url);


1210        const charJSON = await readCharacterData(avatarPath);


1211        if (typeof charJSON !== 'string') throw new Error('Failed to read character file');


1212


1213        const char = JSON.parse(charJSON);


1214        //check if the field exists


1215        if (char[request.body.field] === undefined && char.data[request.body.field] === undefined) {


1216            console.warn('Error: invalid field.');


1217            response.status(400).send('Error: invalid field.');


1218            return;


1219        }


1220        char[request.body.field] = request.body.value;


1221        char.data[request.body.field] = request.body.value;


1222        let newCharJSON = JSON.stringify(char);


1223        const targetFile = (request.body.avatar_url).replace('.png', '');


1224        await writeCharacterData(avatarPath, newCharJSON, targetFile, request);


1225        return response.sendStatus(200);


1226    } catch (err) {


1227        console.error('An error occurred, character edit invalidated.', err);


1228        return response.sendStatus(500);


1229    }


1230});


1231


1232/**


1233 * Sentinel value that signals a field should be completely removed (unset)


1234 * from the character card rather than being set to any value. Use this in


1235 * the merge payload wherever a key should be deleted.


1236 *


1237 * Both the server and the frontend share this constant so that callers can


1238 * explicitly opt into deletion without overloading `null`.


1239 * @type {string}


1240 */


1241const UNSET_SENTINEL = '__@@UNSET@@__';


1242


1243/** Maximum number of characters processed in parallel during bulk merge */


1244const BULK_MERGE_CONCURRENCY = 10;


1245


1246/**


1247 * Recursively walks `source` and removes any key from `target` whose


1248 * corresponding value in `source` equals the {@link UNSET_SENTINEL}.


1249 * Called after {@link deepMerge} so that the sentinel gets replaced by


1250 * an actual key deletion.


1251 * @param {object} target The merged character object to clean up


1252 * @param {object} source The original update payload (pre-merge clone)


1253 */


1254function processUnsetSentinels(target, source) {


1255    for (const key of Object.keys(source)) {


1256        if (source[key] === UNSET_SENTINEL) {


1257            _.unset(target, key);


1258        } else if (_.isPlainObject(source[key]) && _.isPlainObject(target[key])) {


1259            processUnsetSentinels(target[key], source[key]);


1260        }


1261    }


1262}


1263


1264/**


1265 * Reads a character card, applies a merge update (with sentinel-based


1266 * unsetting), validates the result, and writes it back.


1267 * @param {string} avatarPath Full path to the character PNG


1268 * @param {string} avatar     Avatar filename (e.g. "char.png")


1269 * @param {object} updateData The merge payload to apply


1270 * @param {import("express").Request} request Express request object


1271 * @param {((data: any) => boolean) | null} [shouldSkip] Optional function to determine if a character should be skipped based on its original data (used for bulk merge filtering)


1272 * @returns {Promise<{ok: boolean, error?: string, skipped?: boolean}>} Result of the merge operation, including any validation error


1273 */


1274async function mergeCharacterUpdate(avatarPath, avatar, updateData, request, shouldSkip = null) {


1275    const pngStringData = await readCharacterData(avatarPath);


1276    if (!pngStringData) {


1277        return { ok: false, error: 'Invalid character file' };


1278    }


1279


1280    let character = JSON.parse(pngStringData);


1281


1282    if (typeof shouldSkip === 'function' && shouldSkip(character)) {


1283        return { ok: false, skipped: true };


1284    }


1285


1286    const update = _.cloneDeep(updateData);


1287    _.unset(update, 'json_data');


1288    _.unset(character, 'json_data');


1289


1290    character = deepMerge(character, update);


1291    processUnsetSentinels(character, update);


1292


1293    const validator = new TavernCardValidator(character);


1294    //Accept either V1 or V2.


1295    if (!validator.validate()) {


1296        return { ok: false, error: validator.lastValidationError ?? 'Validation failed' };


1297    }


1298


1299    const targetImg = avatar.replace('.png', '');


1300    await writeCharacterData(avatarPath, JSON.stringify(character), targetImg, request);


1301    return { ok: true };


1302}


1303


1304/**


1305 * Handle a POST request to edit character properties.


1306 *


1307 * Operates in two modes depending on the request body:


1308 *


1309 * **Single mode** (default behavior) — when `avatar` (string) is present:


1310 *   Merges the request body with the selected character and validates the


1311 *   result against TavernCard V2 specification.


1312 *


1313 * **Bulk mode** — when `avatars` (array) is present:


1314 *   Applies the same merge to multiple characters in parallel. Supports:


1315 *   - An explicit list of avatars, or all characters when the array is empty


1316 *   - An optional server-side `filter` so only characters where a given


1317 *     JSON path exists and is non-null are updated


1318 *


1319 * In both modes, any value equal to the sentinel `__@@UNSET@@__` will cause


1320 * that key to be **deleted** from the character card instead of being set.


1321 *


1322 * @param {import("express").Request} request - The HTTP request object


1323 * @param {import("express").Response} response - The HTTP response object


1324 * @returns {void}


1325 */


1326router.post('/merge-attributes', getFileNameValidationFunction('avatar'), async function (request, response) {


1327    try {


1328        // ── Bulk mode: avatars array is present ──────────────────


1329        if (Array.isArray(request.body.avatars)) {


1330            const { avatars, data, filter } = request.body;


1331


1332            if (!_.isPlainObject(data)) {


1333                return response.status(400).send({ message: 'No valid update data provided.' });


1334            }


1335


1336            // Determine which avatar files to process


1337            let targetAvatars;


1338            if (avatars.length > 0) {


1339                for (const avatar of avatars) {


1340                    if (typeof avatar !== 'string' || forbiddenRegExp.test(avatar) || path.extname(avatar).toLowerCase() !== '.png') {


1341                        return response.status(400).send({ message: `Invalid avatar filename: ${avatar}` });


1342                    }


1343                }


1344                targetAvatars = avatars;


1345            } else {


1346                // Empty array → scan all characters in the directory


1347                const files = fs.readdirSync(request.user.directories.characters);


1348                targetAvatars = files.filter(file => path.extname(file).toLowerCase() === '.png');


1349            }


1350


1351            const updated = [];


1352            const skipped = [];


1353            const failed = [];


1354


1355            /**


1356             * Process a single character in bulk: read, filter, merge, validate, write.


1357             * @param {string} avatar Avatar filename


1358             */


1359            const processOne = async (avatar) => {


1360                const avatarPath = path.join(request.user.directories.characters, avatar);


1361


1362                try {


1363                    /** @type {(character: object) => boolean} */


1364                    let shouldSkip = () => false;


1365


1366                    // Apply optional server-side filter before updating the card


1367                    if (filter && typeof filter.path === 'string') {


1368                        shouldSkip = (character) => {


1369                            const value = _.get(character, filter.path);


1370                            return value === undefined;


1371                        };


1372                    }


1373


1374                    const result = await mergeCharacterUpdate(avatarPath, avatar, data, request, shouldSkip);


1375                    if (result.ok) {


1376                        updated.push(avatar);


1377                    } else if (result.skipped) {


1378                        skipped.push(avatar);


1379                    } else {


1380                        console.warn(`Bulk merge failed for ${avatar}:`, result.error);


1381                        failed.push(avatar);


1382                    }


1383                } catch (error) {


1384                    console.error(`Bulk merge failed for ${avatar}:`, error);


1385                    failed.push(avatar);


1386                }


1387            };


1388


1389            // Process in parallel with a concurrency limit


1390            for (let i = 0; i < targetAvatars.length; i += BULK_MERGE_CONCURRENCY) {


1391                const batch = targetAvatars.slice(i, i + BULK_MERGE_CONCURRENCY);


1392                await Promise.allSettled(batch.map(processOne));


1393            }


1394


1395            return response.send({ updated, skipped, failed });


1396        }


1397


1398        // ── Single mode (default behavior) ───────────────────────


1399        const update = request.body;


1400        const avatarPath = path.join(request.user.directories.characters, update.avatar);


1401


1402        const result = await mergeCharacterUpdate(avatarPath, update.avatar, update, request);


1403        if (result.ok) {


1404            response.sendStatus(200);


1405        } else {


1406            console.warn(result.error);


1407            response.status(400).send({ message: `Validation failed for ${update.avatar}`, error: result.error });


1408        }


1409    } catch (exception) {


1410        response.status(500).send({ message: 'Unexpected error while saving character.', error: exception.toString() });


1411    }


1412});


1413


1414router.post('/delete', validateAvatarUrlMiddleware, async function (request, response) {


1415    if (!request.body || !request.body.avatar_url) {


1416        return response.sendStatus(400);


1417    }


1418


1419    if (request.body.avatar_url !== sanitize(request.body.avatar_url)) {


1420        console.error('Malicious filename prevented');


1421        return response.sendStatus(403);


1422    }


1423


1424    const avatarPath = path.join(request.user.directories.characters, request.body.avatar_url);


1425    if (!fs.existsSync(avatarPath)) {


1426        return response.sendStatus(400);


1427    }


1428


1429    fs.unlinkSync(avatarPath);


1430    invalidateThumbnail(request.user.directories, 'avatar', request.body.avatar_url);


1431    let dir_name = (request.body.avatar_url.replace('.png', ''));


1432


1433    if (!dir_name.length) {


1434        console.error('Malicious dirname prevented');


1435        return response.sendStatus(403);


1436    }


1437


1438    if (request.body.delete_chats == true) {


1439        try {


1440            await fs.promises.rm(path.join(request.user.directories.chats, sanitize(dir_name)), { recursive: true, force: true });


1441        } catch (err) {


1442            console.error(err);


1443            return response.sendStatus(500);


1444        }


1445    }


1446


1447    return response.sendStatus(200);


1448});


1449


1450/**


1451 * HTTP POST endpoint for the "/api/characters/all" route.


1452 *


1453 * This endpoint is responsible for reading character files from the `charactersPath` directory,


1454 * parsing character data, calculating stats for each character and responding with the data.


1455 * Stats are calculated only on the first run, on subsequent runs the stats are fetched from


1456 * the `charStats` variable.


1457 * The stats are calculated by the `calculateStats` function.


1458 * The characters are processed by the `processCharacter` function.


1459 *


1460 * @param  {import("express").Request} request The HTTP request object.


1461 * @param  {import("express").Response} response The HTTP response object.


1462 * @return {void}


1463 */


1464router.post('/all', async function (request, response) {


1465    try {


1466        const files = fs.readdirSync(request.user.directories.characters);


1467        const pngFiles = files.filter(file => file.endsWith('.png'));


1468        const processingPromises = pngFiles.map(file => processCharacter(file, request.user.directories, { shallow: useShallowCharacters }));


1469        const data = (await Promise.all(processingPromises)).filter(c => c.name);


1470        return response.send(data);


1471    } catch (err) {


1472        console.error(err);


1473        const isRangeError = err instanceof RangeError;


1474        response.status(500).send({ overflow: isRangeError, error: true });


1475    }


1476});


1477


1478router.post('/get', validateAvatarUrlMiddleware, async function (request, response) {


1479    try {


1480        if (!request.body) return response.sendStatus(400);


1481        const item = request.body.avatar_url;


1482        const filePath = path.join(request.user.directories.characters, item);


1483


1484        if (!fs.existsSync(filePath)) {


1485            return response.sendStatus(404);


1486        }


1487


1488        const data = await processCharacter(item, request.user.directories, { shallow: false });


1489


1490        return response.send(data);


1491    } catch (err) {


1492        console.error(err);


1493        response.sendStatus(500);


1494    }


1495});


1496


1497router.post('/chats', validateAvatarUrlMiddleware, async function (request, response) {


1498    try {


1499        if (!request.body) return response.sendStatus(400);


1500


1501        const characterDirectory = (request.body.avatar_url).replace('.png', '');


1502        const chatsDirectory = path.join(request.user.directories.chats, characterDirectory);


1503


1504        if (!fs.existsSync(chatsDirectory)) {


1505            return response.send({ error: true });


1506        }


1507


1508        const files = fs.readdirSync(chatsDirectory, { withFileTypes: true });


1509        const jsonFiles = files.filter(file => file.isFile() && path.extname(file.name) === '.jsonl').map(file => file.name);


1510


1511        if (jsonFiles.length === 0) {


1512            return response.send([]);


1513        }


1514


1515        if (request.body.simple) {


1516            return response.send(jsonFiles.map(file => ({ file_name: file, file_id: path.parse(file).name })));


1517        }


1518


1519        const jsonFilesPromise = jsonFiles.map((file) => {


1520            const withMetadata = !!request.body.metadata;


1521            const pathToFile = path.join(request.user.directories.chats, characterDirectory, file);


1522            return getChatInfo(pathToFile, {}, withMetadata);


1523        });


1524


1525        const chatData = (await Promise.allSettled(jsonFilesPromise)).filter(x => x.status === 'fulfilled').map(x => x.value);


1526        const validFiles = chatData.filter(i => i.file_name);


1527


1528        return response.send(validFiles);


1529    } catch (error) {


1530        console.error(error);


1531        return response.send({ error: true });


1532    }


1533});


1534


1535/**


1536 * Gets the name for the uploaded PNG file.


1537 * @param {string} file File name


1538 * @param {import('../users.js').UserDirectoryList} directories User directories


1539 * @returns {string} - The name for the uploaded PNG file


1540 */


1541function getPngName(file, directories) {


1542    file = sanitize(file);


1543    return getUniqueName(file, (name) => fs.existsSync(path.join(directories.characters, `${name}.png`)),


1544        { nameBuilder: (base, i) => i === 0 ? base : `${base}${i}`, startIndex: 0, maxTries: 10000 }) ?? file;


1545}


1546


1547/**


1548 * Gets the preserved name for the uploaded file if the request is valid.


1549 * @param {import("express").Request} request - Express request object


1550 * @returns {string | undefined} - The preserved name if the request is valid, otherwise undefined


1551 */


1552function getPreservedName(request) {


1553    return typeof request.body.preserved_name === 'string' && request.body.preserved_name.length > 0


1554        ? path.parse(request.body.preserved_name).name


1555        : undefined;


1556}


1557


1558router.post('/import', async function (request, response) {


1559    if (!request.body || !request.file) return response.sendStatus(400);


1560


1561    const uploadPath = path.join(request.file.destination, request.file.filename);


1562    const format = request.body.file_type;


1563    const preservedFileName = getPreservedName(request);


1564


1565    const formatImportFunctions = {


1566        'yaml': importFromYaml,


1567        'yml': importFromYaml,


1568        'json': importFromJson,


1569        'png': importFromPng,


1570        'charx': importFromCharX,


1571        'byaf': importFromByaf,


1572    };


1573


1574    try {


1575        const importFunction = formatImportFunctions[format];


1576


1577        if (!importFunction) {


1578            throw new Error(`Unsupported format: ${format}`);


1579        }


1580


1581        const fileName = await importFunction(uploadPath, { request, response }, preservedFileName);


1582


1583        if (!fileName) {


1584            console.warn('Failed to import character');


1585            return response.sendStatus(400);


1586        }


1587


1588        if (preservedFileName) {


1589            invalidateThumbnail(request.user.directories, 'avatar', `${preservedFileName}.png`);


1590        }


1591


1592        response.send({ file_name: fileName });


1593    } catch (err) {


1594        console.error(err);


1595        response.send({ error: true });


1596    }


1597});


1598


1599router.post('/duplicate', validateAvatarUrlMiddleware, async function (request, response) {


1600    try {


1601        if (!request.body.avatar_url) {


1602            console.warn('avatar URL not found in request body');


1603            console.debug(request.body);


1604            return response.sendStatus(400);


1605        }


1606        let filename = path.join(request.user.directories.characters, sanitize(request.body.avatar_url));


1607        if (!fs.existsSync(filename)) {


1608            console.error('file for dupe not found', filename);


1609            return response.sendStatus(404);


1610        }


1611        let suffix = 1;


1612        let newFilename = filename;


1613


1614        // If filename ends with a _number, increment the number


1615        const nameParts = path.basename(filename, path.extname(filename)).split('_');


1616        const lastPart = nameParts[nameParts.length - 1];


1617


1618        let baseName;


1619


1620        if (!isNaN(Number(lastPart)) && nameParts.length > 1) {


1621            suffix = parseInt(lastPart) + 1;


1622            baseName = nameParts.slice(0, -1).join('_'); // construct baseName without suffix


1623        } else {


1624            baseName = nameParts.join('_'); // original filename is completely the baseName


1625        }


1626


1627        newFilename = path.join(request.user.directories.characters, `${baseName}_${suffix}${path.extname(filename)}`);


1628


1629        while (fs.existsSync(newFilename)) {


1630            let suffixStr = '_' + suffix;


1631            newFilename = path.join(request.user.directories.characters, `${baseName}${suffixStr}${path.extname(filename)}`);


1632            suffix++;


1633        }


1634


1635        fs.copyFileSync(filename, newFilename);


1636        console.info(`${filename} was copied to ${newFilename}`);


1637        response.send({ path: path.parse(newFilename).base });


1638    } catch (error) {


1639        console.error(error);


1640        return response.send({ error: true });


1641    }


1642});


1643


1644router.post('/export', validateAvatarUrlMiddleware, async function (request, response) {


1645    try {


1646        if (!request.body.format || !request.body.avatar_url) {


1647            return response.sendStatus(400);


1648        }


1649


1650        let filename = path.join(request.user.directories.characters, sanitize(request.body.avatar_url));


1651


1652        if (!fs.existsSync(filename)) {


1653            return response.sendStatus(404);


1654        }


1655


1656        switch (request.body.format) {


1657            case 'png': {


1658                const rawBuffer = await fsPromises.readFile(filename);


1659                const rawData = read(rawBuffer);


1660                const mutatedData = mutateJsonString(rawData, unsetPrivateFields);


1661                const mutatedBuffer = write(rawBuffer, mutatedData);


1662                const contentType = mime.lookup(filename) || 'image/png';


1663                response.setHeader('Content-Type', contentType);


1664                response.setHeader('Content-Disposition', `attachment; filename="${encodeURI(path.basename(filename))}"`);


1665                return response.send(mutatedBuffer);


1666            }


1667            case 'json': {


1668                try {


1669                    const json = await readCharacterData(filename);


1670                    if (json === undefined) return response.sendStatus(400);


1671                    const jsonObject = getCharaCardV2(JSON.parse(json), request.user.directories);


1672                    unsetPrivateFields(jsonObject);


1673                    return response.type('json').send(JSON.stringify(jsonObject, null, 4));


1674                } catch {


1675                    return response.sendStatus(400);


1676                }


1677            }


1678        }


1679


1680        return response.sendStatus(400);


1681    } catch (err) {


1682        console.error('Character export failed', err);


1683        response.sendStatus(500);


1684    }


1685});