Switch branch
feat/multi-feature-pack maindefault
View all branches
Blame Raw
Cohee · e3f41666 · · 1077 lines (40.2 KB)
1 contributor 
1import fs from 'node:fs';


2import path from 'node:path';


3import readline from 'node:readline';


4import process from 'node:process';


5


6import express from 'express';


7import sanitize from 'sanitize-filename';


8import { sync as writeFileAtomicSync } from 'write-file-atomic';


9import _ from 'lodash';


10


11import validateAvatarUrlMiddleware from '../middleware/validateFileName.js';


12import {


13    getConfigValue,


14    humanizedDateTime,


15    tryParse,


16    generateTimestamp,


17    removeOldBackups,


18    formatBytes,


19    tryWriteFileSync,


20    tryReadFileSync,


21    tryDeleteFile,


22    readFirstLine,


23    isPathUnderParent,


24} from '../util.js';


25


26const isBackupEnabled = !!getConfigValue('backups.chat.enabled', true, 'boolean');


27const maxTotalChatBackups = Number(getConfigValue('backups.chat.maxTotalBackups', -1, 'number'));


28const throttleInterval = Number(getConfigValue('backups.chat.throttleInterval', 10_000, 'number'));


29const checkIntegrity = !!getConfigValue('backups.chat.checkIntegrity', true, 'boolean');


30


31export const CHAT_BACKUPS_PREFIX = 'chat_';


32


33/**


34 * Saves a chat to the backups directory.


35 * @param {string} directory The user's backup directory.


36 * @param {string} name The name of the chat.


37 * @param {string} data The serialized chat to save.


38 * @param {string} backupPrefix The file prefix. Typically CHAT_BACKUPS_PREFIX.


39 * @returns


40 */


41function backupChat(directory, name, data, backupPrefix = CHAT_BACKUPS_PREFIX) {


42    try {


43        if (!isBackupEnabled) { return; }


44        if (!fs.existsSync(directory)) {


45            console.error(`The chat couldn't be backed up because no directory exists at ${directory}!`);


46        }


47        // replace non-alphanumeric characters with underscores


48        name = sanitize(name).replace(/[^a-z0-9]/gi, '_').toLowerCase();


49


50        const backupFile = path.join(directory, `${backupPrefix}${name}_${generateTimestamp()}.jsonl`);


51


52        tryWriteFileSync(backupFile, data);


53        removeOldBackups(directory, `${backupPrefix}${name}_`);


54        if (isNaN(maxTotalChatBackups) || maxTotalChatBackups < 0) {


55            return;


56        }


57        removeOldBackups(directory, backupPrefix, maxTotalChatBackups);


58    } catch (err) {


59        console.error(`Could not backup chat for ${name}`, err);


60    }


61}


62


63/**


64 * @type {Map<string, import('lodash').DebouncedFunc<typeof backupChat>>}


65 */


66const backupFunctions = new Map();


67


68/**


69 * Gets a backup function for a user.


70 * @param {string} handle User handle


71 * @returns {typeof backupChat} Backup function


72 */


73function getBackupFunction(handle) {


74    if (!backupFunctions.has(handle)) {


75        backupFunctions.set(handle, _.throttle(backupChat, throttleInterval, { leading: true, trailing: true }));


76    }


77    return backupFunctions.get(handle) || (() => { });


78}


79


80/**


81 * Gets a preview message from a chat message string.


82 * @param {string} [lastMessage] - The message to truncate


83 * @returns {string} A truncated preview of the last message or empty string if no messages


84 */


85function getPreviewMessage(lastMessage) {


86    const strlen = 400;


87


88    if (!lastMessage) {


89        return '';


90    }


91


92    return lastMessage.length > strlen


93        ? '...' + lastMessage.substring(lastMessage.length - strlen)


94        : lastMessage;


95}


96


97process.on('exit', () => {


98    for (const func of backupFunctions.values()) {


99        func.flush();


100    }


101});


102


103/**


104 * Imports a chat from Ooba's format.


105 * @param {string} userName User name


106 * @param {string} characterName Character name


107 * @param {object} jsonData JSON data


108 * @returns {string} Chat data


109 */


110function importOobaChat(userName, characterName, jsonData) {


111    /** @type {object[]} */


112    const chat = [{


113        chat_metadata: {},


114        user_name: 'unused',


115        character_name: 'unused',


116    }];


117


118    for (const arr of jsonData.data_visible) {


119        if (arr[0]) {


120            const userMessage = {


121                name: userName,


122                is_user: true,


123                send_date: new Date().toISOString(),


124                mes: arr[0],


125                extra: {},


126            };


127            chat.push(userMessage);


128        }


129        if (arr[1]) {


130            const charMessage = {


131                name: characterName,


132                is_user: false,


133                send_date: new Date().toISOString(),


134                mes: arr[1],


135                extra: {},


136            };


137            chat.push(charMessage);


138        }


139    }


140


141    return chat.map(obj => JSON.stringify(obj)).join('\n');


142}


143


144/**


145 * Imports a chat from Agnai's format.


146 * @param {string} userName User name


147 * @param {string} characterName Character name


148 * @param {object} jsonData Chat data


149 * @returns {string} Chat data


150 */


151function importAgnaiChat(userName, characterName, jsonData) {


152    /** @type {object[]} */


153    const chat = [{


154        chat_metadata: {},


155        user_name: 'unused',


156        character_name: 'unused',


157    }];


158


159    for (const message of jsonData.messages) {


160        const isUser = !!message.userId;


161        chat.push({


162            name: isUser ? userName : characterName,


163            is_user: isUser,


164            send_date: new Date().toISOString(),


165            mes: message.msg,


166            extra: {},


167        });


168    }


169


170    return chat.map(obj => JSON.stringify(obj)).join('\n');


171}


172


173/**


174 * Imports a chat from CAI Tools format.


175 * @param {string} userName User name


176 * @param {string} characterName Character name


177 * @param {object} jsonData JSON data


178 * @returns {string[]} Converted data


179 */


180function importCAIChat(userName, characterName, jsonData) {


181    /**


182     * Converts the chat data to suitable format.


183     * @param {object} history Imported chat data


184     * @returns {object[]} Converted chat data


185     */


186    function convert(history) {


187        const starter = {


188            chat_metadata: {},


189            user_name: 'unused',


190            character_name: 'unused',


191        };


192


193        const historyData = history.msgs.map((msg) => ({


194            name: msg.src.is_human ? userName : characterName,


195            is_user: msg.src.is_human,


196            send_date: new Date().toISOString(),


197            mes: msg.text,


198            extra: {},


199        }));


200


201        return [starter, ...historyData];


202    }


203


204    const newChats = (jsonData.histories.histories ?? []).map(history => newChats.push(convert(history).map(obj => JSON.stringify(obj)).join('\n')));


205    return newChats;


206}


207


208/**


209 * Imports a chat from Kobold Lite format.


210 * @param {string} _userName User name


211 * @param {string} _characterName Character name


212 * @param {object} data JSON data


213 * @returns {string} Chat data


214 */


215function importKoboldLiteChat(_userName, _characterName, data) {


216    const inputToken = '{{[INPUT]}}';


217    const outputToken = '{{[OUTPUT]}}';


218


219    /** @type {function(string): object} */


220    function processKoboldMessage(msg) {


221        const isUser = msg.includes(inputToken);


222        return {


223            name: isUser ? userName : characterName,


224            is_user: isUser,


225            mes: msg.replaceAll(inputToken, '').replaceAll(outputToken, '').trim(),


226            send_date: new Date().toISOString(),


227            extra: {},


228        };


229    }


230


231    // Create the header


232    const userName = String(data.savedsettings.chatname);


233    const characterName = String(data.savedsettings.chatopponent).split('||$||')[0];


234    const header = {


235        chat_metadata: {},


236        user_name: 'unused',


237        character_name: 'unused',


238    };


239    // Format messages


240    const formattedMessages = data.actions.map(processKoboldMessage);


241    // Add prompt if available


242    if (data.prompt) {


243        formattedMessages.unshift(processKoboldMessage(data.prompt));


244    }


245    // Combine header and messages


246    const chatData = [header, ...formattedMessages];


247    return chatData.map(obj => JSON.stringify(obj)).join('\n');


248}


249


250/**


251 * Flattens `msg` and `swipes` data from Chub Chat format.


252 * Only changes enough to make it compatible with the standard chat serialization format.


253 * @param {string} userName User name


254 * @param {string} characterName Character name


255 * @param {string[]} lines serialised JSONL data


256 * @returns {string} Converted data


257 */


258function flattenChubChat(userName, characterName, lines) {


259    function flattenSwipe(swipe) {


260        return swipe.message ? swipe.message : swipe;


261    }


262


263    function convert(line) {


264        const lineData = tryParse(line);


265        if (!lineData) return line;


266


267        if (lineData.mes && lineData.mes.message) {


268            lineData.mes = lineData?.mes.message;


269        }


270


271        if (lineData?.swipes && Array.isArray(lineData.swipes)) {


272            lineData.swipes = lineData.swipes.map(swipe => flattenSwipe(swipe));


273        }


274


275        return JSON.stringify(lineData);


276    }


277


278    return (lines ?? []).map(convert).join('\n');


279}


280


281/**


282 * Imports a chat from RisuAI format.


283 * @param {string} userName User name


284 * @param {string} characterName Character name


285 * @param {object} jsonData Imported chat data


286 * @returns {string} Chat data


287 */


288function importRisuChat(userName, characterName, jsonData) {


289    /** @type {object[]} */


290    const chat = [{


291        chat_metadata: {},


292        user_name: 'unused',


293        character_name: 'unused',


294    }];


295


296    for (const message of jsonData.data.message) {


297        const isUser = message.role === 'user';


298        chat.push({


299            name: message.name ?? (isUser ? userName : characterName),


300            is_user: isUser,


301            send_date: new Date(Number(message.time ?? Date.now())).toISOString(),


302            mes: message.data ?? '',


303            extra: {},


304        });


305    }


306


307    return chat.map(obj => JSON.stringify(obj)).join('\n');


308}


309


310/**


311 * Checks if the chat being saved has the same integrity as the one being loaded.


312 * @param {string} filePath Path to the chat file


313 * @param {string} integritySlug Integrity slug


314 * @returns {Promise<boolean>} Whether the chat is intact


315 */


316async function checkChatIntegrity(filePath, integritySlug) {


317    // If the chat file doesn't exist, assume it's intact


318    if (!fs.existsSync(filePath)) {


319        return true;


320    }


321


322    // Parse the first line of the chat file as JSON


323    const firstLine = await readFirstLine(filePath);


324    const jsonData = tryParse(firstLine);


325    const chatIntegrity = jsonData?.chat_metadata?.integrity;


326


327    // If the chat has no integrity metadata, assume it's intact


328    if (!chatIntegrity) {


329        console.debug(`File "${filePath}" does not have integrity metadata matching "${integritySlug}". The integrity validation has been skipped.`);


330        return true;


331    }


332


333    // Check if the integrity matches


334    return chatIntegrity === integritySlug;


335}


336


337/**


338 * @typedef {Object} ChatInfo


339 * @property {string} [file_id] - The name of the chat file (without extension)


340 * @property {string} [file_name] - The name of the chat file (with extension)


341 * @property {string} [file_size] - The size of the chat file in a human-readable format


342 * @property {number} [chat_items] - The number of chat items in the file


343 * @property {string} [mes] - The last message in the chat


344 * @property {number|string} [last_mes] - The timestamp of the last message


345 * @property {object} [chat_metadata] - Additional chat metadata


346 * @property {boolean} [match] - Whether the chat matches the search criteria


347 */


348


349/**


350 * Reads the information from a chat file.


351 * @param {string} pathToFile - Path to the chat file


352 * @param {object} additionalData - Additional data to include in the result


353 * @param {boolean} withMetadata - Whether to read chat metadata


354 * @param {ChatMatchFunction|null} matcher - Optional function to match messages


355 * @returns {Promise<ChatInfo>}


356 *


357 * @typedef {(textArray: string[]) => boolean} ChatMatchFunction


358 */


359export async function getChatInfo(pathToFile, additionalData = {}, withMetadata = false, matcher = null) {


360    return new Promise(async (res) => {


361        const parsedPath = path.parse(pathToFile);


362        const stats = await fs.promises.stat(pathToFile);


363        const hasMatcher = (typeof matcher === 'function');


364


365        const chatData = {


366            match: false,


367            file_id: parsedPath.name,


368            file_name: parsedPath.base,


369            file_size: formatBytes(stats.size),


370            chat_items: 0,


371            mes: '[The chat is empty]',


372            last_mes: stats.mtimeMs,


373            ...additionalData,


374        };


375


376        if (stats.size === 0) {


377            res(chatData);


378            return;


379        }


380


381        const fileStream = fs.createReadStream(pathToFile);


382        const rl = readline.createInterface({


383            input: fileStream,


384            crlfDelay: Infinity,


385        });


386


387        let lastLine;


388        let itemCounter = 0;


389        let hasAnyMatch = false;


390        let matchBuffer = [];


391        rl.on('line', (line) => {


392            if (withMetadata && itemCounter === 0) {


393                const jsonData = tryParse(line);


394                if (jsonData && _.isObjectLike(jsonData.chat_metadata)) {


395                    chatData.chat_metadata = jsonData.chat_metadata;


396                }


397            }


398            // Skip matching if any match was already found


399            if (hasMatcher && !hasAnyMatch && itemCounter > 0) {


400                const jsonData = tryParse(line);


401                if (jsonData) {


402                    matchBuffer.push(jsonData.mes || '');


403                    if (matcher(matchBuffer)) {


404                        hasAnyMatch = true;


405                        matchBuffer = [];


406                    }


407                }


408            }


409            itemCounter++;


410            lastLine = line;


411        });


412        rl.on('close', () => {


413            rl.close();


414


415            if (lastLine) {


416                const jsonData = tryParse(lastLine);


417                if (jsonData && (jsonData.name || jsonData.character_name || jsonData.chat_metadata)) {


418                    chatData.chat_items = (itemCounter - 1);


419                    chatData.mes = jsonData.mes || '[The message is empty]';


420                    chatData.last_mes = jsonData.send_date || new Date(Math.round(stats.mtimeMs)).toISOString();


421                    chatData.match = hasMatcher ? hasAnyMatch : true;


422


423                    res(chatData);


424                } else {


425                    console.warn('Found an invalid or corrupted chat file:', pathToFile);


426                    res({});


427                }


428            }


429        });


430    });


431}


432


433export const router = express.Router();


434


435// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error


436class IntegrityMismatchError extends Error {


437    constructor(...params) {


438        // Pass remaining arguments (including vendor specific ones) to parent constructor


439        super(...params);


440        // Maintains proper stack trace for where our error was thrown (non-standard)


441        if (Error.captureStackTrace) {


442            Error.captureStackTrace(this, IntegrityMismatchError);


443        }


444        this.date = new Date();


445    }


446}


447


448/**


449 * Tries to save the chat data to a file, performing an integrity check if required.


450 * @param {Array} chatData The chat array to save.


451 * @param {string} filePath Target file path for the data.


452 * @param {boolean} skipIntegrityCheck If undefined, the chat's integrity will not be checked.


453 * @param {string} handle The users handle, passed to getBackupFunction.


454 * @param {string} cardName Passed to backupChat.


455 * @param {string} backupDirectory Passed to backupChat.


456 */


457export async function trySaveChat(chatData, filePath, skipIntegrityCheck = false, handle, cardName, backupDirectory) {


458    const jsonlData = chatData?.map(m => JSON.stringify(m)).join('\n');


459


460    const doIntegrityCheck = (checkIntegrity && !skipIntegrityCheck);


461    const chatIntegritySlug = doIntegrityCheck ? chatData?.[0]?.chat_metadata?.integrity : undefined;


462


463    if (chatIntegritySlug && !await checkChatIntegrity(filePath, chatIntegritySlug)) {


464        throw new IntegrityMismatchError(`Chat integrity check failed for "${filePath}". The expected integrity slug was "${chatIntegritySlug}".`);


465    }


466    tryWriteFileSync(filePath, jsonlData);


467    getBackupFunction(handle)(backupDirectory, cardName, jsonlData);


468}


469


470router.post('/save', validateAvatarUrlMiddleware, async function (request, response) {


471    try {


472        const handle = request.user.profile.handle;


473        const cardName = String(request.body.avatar_url).replace('.png', '');


474        const chatData = request.body.chat;


475        const chatFileName = `${String(request.body.file_name)}.jsonl`;


476        const chatFilePath = path.join(request.user.directories.chats, cardName, sanitize(chatFileName));


477        if (!isPathUnderParent(request.user.directories.chats, chatFilePath)) {


478            return response.sendStatus(400);


479        }


480


481        if (Array.isArray(chatData)) {


482            await trySaveChat(chatData, chatFilePath, request.body.force, handle, cardName, request.user.directories.backups);


483            return response.send({ ok: true });


484        } else {


485            return response.status(400).send({ error: 'The request\'s body.chat is not an array.' });


486        }


487    } catch (error) {


488        if (error instanceof IntegrityMismatchError) {


489            console.error(error.message);


490            return response.status(400).send({ error: 'integrity' });


491        }


492        console.error(error);


493        return response.status(500).send({ error: 'An error has occurred, see the console logs for more information.' });


494    }


495});


496


497/**


498 * Gets the chat as an object.


499 * @param {string} chatFilePath The full chat file path.


500 * @returns {Array}} If the chatFilePath cannot be read, this will return [].


501 */


502export function getChatData(chatFilePath) {


503    let chatData = [];


504


505    const chatJSON = tryReadFileSync(chatFilePath) ?? '';


506    if (chatJSON.length > 0) {


507        const lines = chatJSON.split('\n');


508        // Iterate through the array of strings and parse each line as JSON


509        chatData = lines.map(line => tryParse(line)).filter(x => x);


510    } else {


511        console.warn(`File not found: ${chatFilePath}. The chat does not exist or is empty.`);


512    }


513


514    return chatData;


515}


516


517router.post('/get', validateAvatarUrlMiddleware, function (request, response) {


518    try {


519        const dirName = String(request.body.avatar_url).replace('.png', '');


520        const directoryPath = path.join(request.user.directories.chats, dirName);


521        if (!isPathUnderParent(request.user.directories.chats, directoryPath)) {


522            return response.sendStatus(400);


523        }


524        const chatDirExists = fs.existsSync(directoryPath);


525


526        //if no chat dir for the character is found, make one with the character name


527        if (!chatDirExists) {


528            fs.mkdirSync(directoryPath);


529            return response.send({});


530        }


531


532        if (!request.body.file_name) {


533            return response.send({});


534        }


535


536        const chatFileName = `${String(request.body.file_name)}.jsonl`;


537        const chatFilePath = path.join(directoryPath, sanitize(chatFileName));


538


539        return response.send(getChatData(chatFilePath));


540    } catch (error) {


541        console.error(error);


542        return response.send({});


543    }


544});


545


546router.post('/rename', validateAvatarUrlMiddleware, async function (request, response) {


547    try {


548        if (!request.body || !request.body.original_file || !request.body.renamed_file) {


549            return response.sendStatus(400);


550        }


551


552        const pathToFolder = request.body.is_group


553            ? request.user.directories.groupChats


554            : path.join(request.user.directories.chats, String(request.body.avatar_url).replace('.png', ''));


555        if (!request.body.is_group && !isPathUnderParent(request.user.directories.chats, pathToFolder)) {


556            return response.sendStatus(400);


557        }


558        const pathToOriginalFile = path.join(pathToFolder, sanitize(request.body.original_file));


559        const pathToRenamedFile = path.join(pathToFolder, sanitize(request.body.renamed_file));


560        const sanitizedFileName = path.parse(pathToRenamedFile).name;


561        console.debug('Old chat name', pathToOriginalFile);


562        console.debug('New chat name', pathToRenamedFile);


563


564        if (!fs.existsSync(pathToOriginalFile) || fs.existsSync(pathToRenamedFile)) {


565            console.error('Either Source or Destination files are not available');


566            return response.status(400).send({ error: true });


567        }


568


569        fs.copyFileSync(pathToOriginalFile, pathToRenamedFile);


570        fs.unlinkSync(pathToOriginalFile);


571        console.info('Successfully renamed chat file.');


572        return response.send({ ok: true, sanitizedFileName });


573    } catch (error) {


574        console.error('Error renaming chat file:', error);


575        return response.status(500).send({ error: true });


576    }


577});


578


579router.post('/delete', validateAvatarUrlMiddleware, function (request, response) {


580    try {


581        if (!path.extname(request.body.chatfile)) {


582            request.body.chatfile += '.jsonl';


583        }


584


585        const dirName = String(request.body.avatar_url).replace('.png', '');


586        const chatFileName = String(request.body.chatfile);


587        const chatFilePath = path.join(request.user.directories.chats, dirName, sanitize(chatFileName));


588        if (!isPathUnderParent(request.user.directories.chats, chatFilePath)) {


589            return response.sendStatus(400);


590        }


591        //Return success if the file was deleted.


592        if (tryDeleteFile(chatFilePath)) {


593            return response.send({ ok: true });


594        } else {


595            console.error('The chat file was not deleted.');


596            return response.sendStatus(400);


597        }


598    } catch (error) {


599        console.error(error);


600        return response.sendStatus(500);


601    }


602});


603


604router.post('/export', validateAvatarUrlMiddleware, async function (request, response) {


605    if (!request.body.file || (!request.body.avatar_url && request.body.is_group === false)) {


606        return response.sendStatus(400);


607    }


608    const pathToFolder = request.body.is_group


609        ? request.user.directories.groupChats


610        : path.join(request.user.directories.chats, String(request.body.avatar_url).replace('.png', ''));


611    const filename = path.join(pathToFolder, sanitize(request.body.file));


612    if (!request.body.is_group && !isPathUnderParent(request.user.directories.chats, filename)) {


613        return response.sendStatus(400);


614    }


615    let exportfilename = request.body.exportfilename;


616    if (!fs.existsSync(filename)) {


617        const errorMessage = {


618            message: `Could not find JSONL file to export. Source chat file: ${filename}.`,


619        };


620        console.error(errorMessage.message);


621        return response.status(404).json(errorMessage);


622    }


623    try {


624        // Short path for JSONL files


625        if (request.body.format === 'jsonl') {


626            try {


627                const rawFile = fs.readFileSync(filename, 'utf8');


628                const successMessage = {


629                    message: `Chat saved to ${exportfilename}`,


630                    result: rawFile,


631                };


632


633                console.info(`Chat exported as ${exportfilename}`);


634                return response.status(200).json(successMessage);


635            } catch (err) {


636                console.error(err);


637                const errorMessage = {


638                    message: `Could not read JSONL file to export. Source chat file: ${filename}.`,


639                };


640                console.error(errorMessage.message);


641                return response.status(500).json(errorMessage);


642            }


643        }


644


645        const readStream = fs.createReadStream(filename);


646        const rl = readline.createInterface({


647            input: readStream,


648        });


649        let buffer = '';


650        rl.on('line', (line) => {


651            const data = JSON.parse(line);


652            // Skip non-printable/prompt-hidden messages


653            if (data.is_system) {


654                return;


655            }


656            if (data.mes) {


657                const name = data.name;


658                const message = (data?.extra?.display_text || data?.mes || '').replace(/\r?\n/g, '\n');


659                buffer += (`${name}: ${message}\n\n`);


660            }


661        });


662        rl.on('close', () => {


663            const successMessage = {


664                message: `Chat saved to ${exportfilename}`,


665                result: buffer,


666            };


667            console.info(`Chat exported as ${exportfilename}`);


668            return response.status(200).json(successMessage);


669        });


670    } catch (err) {


671        console.error('chat export failed.', err);


672        return response.sendStatus(400);


673    }


674});


675


676router.post('/group/import', function (request, response) {


677    try {


678        const filedata = request.file;


679


680        if (!filedata) {


681            return response.sendStatus(400);


682        }


683


684        const chatname = humanizedDateTime();


685        const pathToUpload = path.join(filedata.destination, filedata.filename);


686        const pathToNewFile = path.join(request.user.directories.groupChats, `${chatname}.jsonl`);


687        fs.copyFileSync(pathToUpload, pathToNewFile);


688        fs.unlinkSync(pathToUpload);


689        return response.send({ res: chatname });


690    } catch (error) {


691        console.error(error);


692        return response.send({ error: true });


693    }


694});


695


696router.post('/import', validateAvatarUrlMiddleware, function (request, response) {


697    if (!request.body) return response.sendStatus(400);


698


699    const format = request.body.file_type;


700    const avatarUrl = (request.body.avatar_url).replace('.png', '');


701    const characterName = sanitize(request.body.character_name) || 'Character';


702    const userName = sanitize(request.body.user_name) || 'User';


703    const fileNames = [];


704


705    if (!request.file) {


706        return response.sendStatus(400);


707    }


708


709    const directoryPath = path.join(request.user.directories.chats, avatarUrl);


710    if (!isPathUnderParent(request.user.directories.chats, directoryPath)) {


711        return response.sendStatus(400);


712    }


713


714    try {


715        const pathToUpload = path.join(request.file.destination, request.file.filename);


716        const data = fs.readFileSync(pathToUpload, 'utf8');


717


718        if (format === 'json') {


719            fs.unlinkSync(pathToUpload);


720            const jsonData = JSON.parse(data);


721


722            /** @type {function(string, string, object): string|string[]} */


723            let importFunc;


724


725            if (jsonData.savedsettings !== undefined) { // Kobold Lite format


726                importFunc = importKoboldLiteChat;


727            } else if (jsonData.histories !== undefined) { // CAI Tools format


728                importFunc = importCAIChat;


729            } else if (Array.isArray(jsonData.data_visible)) { // oobabooga's format


730                importFunc = importOobaChat;


731            } else if (Array.isArray(jsonData.messages)) { // Agnai's format


732                importFunc = importAgnaiChat;


733            } else if (jsonData.type === 'risuChat') { // RisuAI format


734                importFunc = importRisuChat;


735            } else { // Unknown format


736                console.error('Incorrect chat format .json');


737                return response.send({ error: true });


738            }


739


740            const handleChat = (chat) => {


741                const fileName = `${characterName} - ${humanizedDateTime()} imported.jsonl`;


742                const filePath = path.join(directoryPath, fileName);


743                fileNames.push(fileName);


744                writeFileAtomicSync(filePath, chat, 'utf8');


745            };


746


747            const chat = importFunc(userName, characterName, jsonData);


748


749            if (Array.isArray(chat)) {


750                chat.forEach(handleChat);


751            } else {


752                handleChat(chat);


753            }


754


755            return response.send({ res: true, fileNames });


756        }


757


758        if (format === 'jsonl') {


759            let lines = data.split('\n');


760            const header = lines[0];


761


762            const jsonData = JSON.parse(header);


763


764            if (!(jsonData.user_name !== undefined || jsonData.name !== undefined || jsonData.chat_metadata !== undefined)) {


765                console.error('Incorrect chat format .jsonl');


766                return response.send({ error: true });


767            }


768


769            // Do a tiny bit of work to import Chub Chat data


770            // Processing the entire file is so fast that it's not worth checking if it's a Chub chat first


771            let flattenedChat = data;


772            try {


773                // flattening is unlikely to break, but it's not worth failing to


774                // import normal chats in an attempt to import a Chub chat


775                flattenedChat = flattenChubChat(userName, characterName, lines);


776            } catch (error) {


777                console.warn('Failed to flatten Chub Chat data: ', error);


778            }


779


780            const fileName = `${characterName} - ${humanizedDateTime()} imported.jsonl`;


781            const filePath = path.join(directoryPath, fileName);


782            fileNames.push(fileName);


783            if (flattenedChat !== data) {


784                writeFileAtomicSync(filePath, flattenedChat, 'utf8');


785            } else {


786                fs.copyFileSync(pathToUpload, filePath);


787            }


788            fs.unlinkSync(pathToUpload);


789            response.send({ res: true, fileNames });


790        }


791    } catch (error) {


792        console.error(error);


793        return response.send({ error: true });


794    }


795});


796


797router.post('/group/get', (request, response) => {


798    if (!request.body || !request.body.id) {


799        return response.sendStatus(400);


800    }


801


802    const id = request.body.id;


803    const chatFilePath = path.join(request.user.directories.groupChats, sanitize(`${id}.jsonl`));


804


805    return response.send(getChatData(chatFilePath));


806});


807


808router.post('/group/info', async (request, response) => {


809    try {


810        if (!request.body || !request.body.id) {


811            return response.sendStatus(400);


812        }


813


814        const id = request.body.id;


815        const chatFilePath = path.join(request.user.directories.groupChats, sanitize(`${id}.jsonl`));


816


817        const chatInfo = await getChatInfo(chatFilePath);


818        return response.send(chatInfo);


819    } catch (error) {


820        console.error(error);


821        return response.sendStatus(500);


822    }


823});


824


825router.post('/group/delete', (request, response) => {


826    try {


827        if (!request.body || !request.body.id) {


828            return response.sendStatus(400);


829        }


830


831        const id = request.body.id;


832        const chatFilePath = path.join(request.user.directories.groupChats, sanitize(`${id}.jsonl`));


833


834        //Return success if the file was deleted.


835        if (tryDeleteFile(chatFilePath)) {


836            return response.send({ ok: true });


837        } else {


838            console.error('The group chat file was not deleted.');


839            return response.sendStatus(400);


840        }


841    } catch (error) {


842        console.error(error);


843        return response.sendStatus(500);


844    }


845});


846


847router.post('/group/save', async function (request, response) {


848    try {


849        if (!request.body || !request.body.id) {


850            return response.sendStatus(400);


851        }


852


853        const id = request.body.id;


854        const handle = request.user.profile.handle;


855        const chatFilePath = path.join(request.user.directories.groupChats, sanitize(`${id}.jsonl`));


856        const chatData = request.body.chat;


857


858        if (Array.isArray(chatData)) {


859            await trySaveChat(chatData, chatFilePath, request.body.force, handle, String(id), request.user.directories.backups);


860            return response.send({ ok: true });


861        } else {


862            return response.status(400).send({ error: 'The request\'s body.chat is not an array.' });


863        }


864    } catch (error) {


865        if (error instanceof IntegrityMismatchError) {


866            console.error(error.message);


867            return response.status(400).send({ error: 'integrity' });


868        }


869        console.error(error);


870        return response.status(500).send({ error: 'An error has occurred, see the console logs for more information.' });


871    }


872});


873


874router.post('/search', validateAvatarUrlMiddleware, async function (request, response) {


875    try {


876        const { query, avatar_url, group_id } = request.body;


877


878        /** @type {string[]} */


879        let chatFiles = [];


880


881        if (group_id) {


882            // Find group's chat IDs first


883            const groupDir = path.join(request.user.directories.groups);


884            const groupFiles = fs.readdirSync(groupDir)


885                .filter(file => path.extname(file) === '.json');


886


887            let targetGroup;


888            for (const groupFile of groupFiles) {


889                try {


890                    const groupData = JSON.parse(fs.readFileSync(path.join(groupDir, groupFile), 'utf8'));


891                    if (groupData.id === group_id) {


892                        targetGroup = groupData;


893                        break;


894                    }


895                } catch (error) {


896                    console.warn(groupFile, 'group file is corrupted:', error);


897                }


898            }


899


900            if (!Array.isArray(targetGroup?.chats)) {


901                return response.send([]);


902            }


903


904            // Find group chat files for given group ID


905            const groupChatsDir = path.join(request.user.directories.groupChats);


906            chatFiles = targetGroup.chats


907                .map(chatId => path.join(groupChatsDir, `${chatId}.jsonl`))


908                .filter(fileName => fs.existsSync(fileName));


909        } else {


910            // Regular character chat directory


911            const character_name = avatar_url.replace('.png', '');


912            const directoryPath = path.join(request.user.directories.chats, character_name);


913


914            if (!fs.existsSync(directoryPath)) {


915                return response.send([]);


916            }


917


918            chatFiles = fs.readdirSync(directoryPath)


919                .filter(file => path.extname(file) === '.jsonl')


920                .map(fileName => path.join(directoryPath, fileName));


921        }


922


923        /**


924         * @type {SearchChatResult[]}


925         * @typedef {object} SearchChatResult


926         * @property {string} [file_name] - The name of the chat file


927         * @property {string} [file_size] - The size of the chat file in a human-readable format


928         * @property {number} [message_count] - The number of messages in the chat


929         * @property {number|string} [last_mes] - The timestamp of the last message


930         * @property {string} [preview_message] - A preview of the last message


931         */


932        const results = [];


933


934        /** @type {string[]} */


935        const fragments = query ? query.trim().toLowerCase().split(/\s+/).filter(x => x) : [];


936


937        /** @type {ChatMatchFunction} */


938        const hasTextMatch = (textArray) => {


939            if (fragments.length === 0) {


940                return true;


941            }


942            return fragments.every(fragment => textArray.some(text => String(text ?? '').toLowerCase().includes(fragment)));


943        };


944


945        for (const chatFile of chatFiles) {


946            const matcher = query ? hasTextMatch : null;


947            const chatInfo = await getChatInfo(chatFile, {}, false, matcher);


948            const hasMatch = chatInfo.match || hasTextMatch([chatInfo.file_id ?? '']);


949


950            // Skip corrupted or invalid chat files


951            if (!chatInfo.file_name) {


952                continue;


953            }


954


955            // Empty chats without a file name match are skipped when searching with a query


956            if (query && chatInfo.chat_items === 0 && !hasMatch) {


957                continue;


958            }


959


960            // If no search query or a match was found, include the chat in results


961            if (!query || hasMatch) {


962                results.push({


963                    file_name: chatInfo.file_id,


964                    file_size: chatInfo.file_size,


965                    message_count: chatInfo.chat_items,


966                    last_mes: chatInfo.last_mes,


967                    preview_message: getPreviewMessage(chatInfo.mes),


968                });


969            }


970        }


971


972        return response.send(results);


973    } catch (error) {


974        console.error('Chat search error:', error);


975        return response.status(500).json({ error: 'Search failed' });


976    }


977});


978


979router.post('/recent', async function (request, response) {


980    try {


981        /** @typedef {{pngFile?: string, groupId?: string, filePath: string, mtime: number}} ChatFile */


982        /** @type {ChatFile[]} */


983        const allChatFiles = [];


984        /** @type {import('../../public/scripts/welcome-screen.js').PinnedChat[]} */


985        const pinnedChats = Array.isArray(request.body.pinned) ? request.body.pinned : [];


986


987        const getCharacterChatFiles = async () => {


988            const pngDirents = await fs.promises.readdir(request.user.directories.characters, { withFileTypes: true });


989            const pngFiles = pngDirents.filter(e => e.isFile() && path.extname(e.name) === '.png').map(e => e.name);


990


991            for (const pngFile of pngFiles) {


992                const chatsDirectory = pngFile.replace('.png', '');


993                const pathToChats = path.join(request.user.directories.chats, chatsDirectory);


994                if (!fs.existsSync(pathToChats)) {


995                    continue;


996                }


997                const pathStats = await fs.promises.stat(pathToChats);


998                if (pathStats.isDirectory()) {


999                    const chatFiles = await fs.promises.readdir(pathToChats);


1000                    const jsonlFiles = chatFiles.filter(file => path.extname(file) === '.jsonl');


1001


1002                    for (const file of jsonlFiles) {


1003                        const filePath = path.join(pathToChats, file);


1004                        const stats = await fs.promises.stat(filePath);


1005                        allChatFiles.push({ pngFile, filePath, mtime: stats.mtimeMs });


1006                    }


1007                }


1008            }


1009        };


1010


1011        const getGroupChatFiles = async () => {


1012            const groupDirents = await fs.promises.readdir(request.user.directories.groups, { withFileTypes: true });


1013            const groups = groupDirents.filter(e => e.isFile() && path.extname(e.name) === '.json').map(e => e.name);


1014


1015            for (const group of groups) {


1016                try {


1017                    const groupPath = path.join(request.user.directories.groups, group);


1018                    const groupContents = await fs.promises.readFile(groupPath, 'utf8');


1019                    const groupData = JSON.parse(groupContents);


1020


1021                    if (Array.isArray(groupData.chats)) {


1022                        for (const chat of groupData.chats) {


1023                            const filePath = path.join(request.user.directories.groupChats, `${chat}.jsonl`);


1024                            if (!fs.existsSync(filePath)) {


1025                                continue;


1026                            }


1027                            const stats = await fs.promises.stat(filePath);


1028                            allChatFiles.push({ groupId: groupData.id, filePath, mtime: stats.mtimeMs });


1029                        }


1030                    }


1031                } catch (error) {


1032                    // Skip group files that can't be read or parsed


1033                    continue;


1034                }


1035            }


1036        };


1037


1038        const getRootChatFiles = async () => {


1039            const dirents = await fs.promises.readdir(request.user.directories.chats, { withFileTypes: true });


1040            const chatFiles = dirents.filter(e => e.isFile() && path.extname(e.name) === '.jsonl').map(e => e.name);


1041


1042            for (const file of chatFiles) {


1043                const filePath = path.join(request.user.directories.chats, file);


1044                const stats = await fs.promises.stat(filePath);


1045                allChatFiles.push({ filePath, mtime: stats.mtimeMs });


1046            }


1047        };


1048


1049        await Promise.allSettled([getCharacterChatFiles(), getGroupChatFiles(), getRootChatFiles()]);


1050


1051        const max = parseInt(request.body.max ?? Number.MAX_SAFE_INTEGER) + pinnedChats.length;


1052        const isPinned = (/** @type {ChatFile} */ chatFile) => pinnedChats.some(p => p.file_name === path.basename(chatFile.filePath) && (p.avatar === chatFile.pngFile || p.group === chatFile.groupId));


1053        const recentChats = allChatFiles.sort((a, b) => {


1054            const isAPinned = isPinned(a);


1055            const isBPinned = isPinned(b);


1056


1057            if (isAPinned && !isBPinned) return -1;


1058            if (!isAPinned && isBPinned) return 1;


1059


1060            return b.mtime - a.mtime;


1061        }).slice(0, max);


1062        const jsonFilesPromise = recentChats.map((file) => {


1063            const withMetadata = !!request.body.metadata;


1064            return file.groupId


1065                ? getChatInfo(file.filePath, { group: file.groupId }, withMetadata)


1066                : getChatInfo(file.filePath, { avatar: file.pngFile }, withMetadata);


1067        });


1068


1069        const chatData = (await Promise.allSettled(jsonFilesPromise)).filter(x => x.status === 'fulfilled').map(x => x.value);


1070        const validFiles = chatData.filter(i => i.file_name);


1071


1072        return response.send(validFiles);


1073    } catch (error) {


1074        console.error(error);


1075        return response.sendStatus(500);


1076    }


1077});