Backported refactor of the chats endpoint from `feat/chat-tree.` (#4870) * Backported cleanup and refactor of chats endpoint from `feat/chat-tree`. * https://github.com/SillyTavern/SillyTavern/pull/4870#discussion_r2599941781 * Error message. https://github.com/SillyTavern/SillyTavern/pull/4870#discussion_r2599941796 https://github.com/SillyTavern/SillyTavern/pull/4870#discussion_r2599941797 * chatIntegritySlug https://github.com/SillyTavern/SillyTavern/pull/4870#discussion_r2599958991 * /save should not hang on empty body.chat. https://github.com/SillyTavern/SillyTavern/pull/4870#discussion_r2599959001 https://github.com/SillyTavern/SillyTavern/pull/4870#discussion_r2599958997 * When the backup directory does not exist, log an error instead of silently failing. * chatData should be an array. https://github.com/SillyTavern/SillyTavern/pull/4870#discussion_r2599990728 * Skip body check. https://github.com/SillyTavern/SillyTavern/pull/4870#discussion_r2599987542 * Change integrity warning to debug. https://github.com/SillyTavern/SillyTavern/pull/4870#discussion_r2600005665 * Return null on error in `tryReadFileSync`. * Update warning message for missing chat file to clarify potential emptiness * Rename IntegrityMismatch to IntegrityMismatchError for clarity and consistency * Fix a type error and formatting * Improve debug messages and documentation for integrity checks in chat saving * Add newline before file writing function for improved readability * Improve debug message formatting for integrity validation in checkChatIntegrity function --------- Co-authored-by: user <user@exmaple.com> Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>

d5320f57cf2619a3a0812d45aa922be315aff690

DeclineThyself <235079501+DeclineThyself@users.noreply.github.com>

Signed

Parents: b5e20f2ff98708b13961d09130b63eda839c191b

src/endpoints/chats.js+127 -123 Viewed

     generateTimestamp,
     removeOldBackups,
     formatBytes,
 +    tryWriteFileSync,
 +    tryReadFileSync,
 +    tryDeleteFile,
 +    readFirstLine,
 } from '../util.js';
 const isBackupEnabled = !!getConfigValue('backups.chat.enabled', true, 'boolean');

 /**
  * Saves a chat to the backups directory.
  * @param {string} directory The user's backupsbackup directory.
  * @param {string} name The name of the chat.
  * @param {string} chatdata The serialized chat to save.
 + * @param {string} backupPrefix The file prefix. Typically CHAT_BACKUPS_PREFIX.
 + * @returns
  */
 function backupChat(directory, name, chatdata, backupPrefix = CHAT_BACKUPS_PREFIX) {
     try {
         if (!isBackupEnabled || !fs.existsSync(directory)) { return; }
 -            return;
 +        if (!fs.existsSync(directory)) {
 +            console.error(`The chat couldn't be backed up because no directory exists at ${directory}!`);
+        }
+-
         // replace non-alphanumeric characters with underscores
         name = sanitize(name).replace(/[^a-z0-9]/gi, '_').toLowerCase();
         const backupFile = path.join(directory, `${CHAT_BACKUPS_PREFIXbackupPrefix}${name}_${generateTimestamp()}.jsonl`);
 -        writeFileAtomicSync(backupFile, chat, 'utf-8');
+-
 -        removeOldBackups(directory, `${CHAT_BACKUPS_PREFIX}${name}_`);
 +        tryWriteFileSync(backupFile, data);
 +        removeOldBackups(directory, `${backupPrefix}${name}_`);
         if (isNaN(maxTotalChatBackups) || maxTotalChatBackups < 0) {
             return;
+        }
+-
 +        removeOldBackups(directory, backupPrefix, maxTotalChatBackups);
 -        removeOldBackups(directory, CHAT_BACKUPS_PREFIX, maxTotalChatBackups);
     } catch (err) {
         console.error(`Could not backup chat for ${name}`, err);
+    }
+}
 /**
  * @type {Map<string, import('lodash').DebouncedFunc<function(string, string, string):typeof voidbackupChat>>}
  */
 const backupFunctions = new Map();
 /**
  * Gets a backup function for a user.
  * @param {string} handle User handle
  * @returns {function(string, string, string):typeof voidbackupChat} Backup function
  */
 function getBackupFunction(handle) {
     if (!backupFunctions.has(handle)) {

+}
 /**
 - * Reads the first line of a file asynchronously.
 - * @param {string} filePath Path to the file
 - * @returns {Promise<string>} The first line of the file
 - */
 -function readFirstLine(filePath) {
 -    const stream = fs.createReadStream(filePath, { encoding: 'utf8' });
 -    const rl = readline.createInterface({ input: stream });
 -    return new Promise((resolve, reject) => {
 -        let resolved = false;
 -        rl.on('line', line => {
 -            resolved = true;
 -            rl.close();
 -            stream.close();
 -            resolve(line);
 -        });
+-
 -        rl.on('error', error => {
 -            resolved = true;
 -            reject(error);
 -        });
+-
 -        // Handle empty files
 -        stream.on('end', () => {
 -            if (!resolved) {
 -                resolved = true;
 -                resolve('');
 -            }
 -        });
 -    });
 -}
+-
 -/**
  * Checks if the chat being saved has the same integrity as the one being loaded.
  * @param {string} filePath Path to the chat file
  * @param {string} integritySlug Integrity slug

     // If the chat has no integrity metadata, assume it's intact
     if (!chatIntegrity) {
 +        console.debug(`File "${filePath}" does not have integrity metadata matching "${integritySlug}". The integrity validation has been skipped.`);
         return true;
+    }

 export const router = express.Router();
 +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error
 +class IntegrityMismatchError extends Error {
 +    constructor(...params) {
 +        // Pass remaining arguments (including vendor specific ones) to parent constructor
 +        super(...params);
 +        // Maintains proper stack trace for where our error was thrown (non-standard)
 +        if (Error.captureStackTrace) {
 +            Error.captureStackTrace(this, IntegrityMismatchError);
 +        }
 +        this.date = new Date();
 +    }
 +}
++
 +/**
 + * Tries to save the chat data to a file, performing an integrity check if required.
 + * @param {Array} chatData The chat array to save.
 + * @param {string} filePath Target file path for the data.
 + * @param {boolean} skipIntegrityCheck If undefined, the chat's integrity will not be checked.
 + * @param {string} handle The users handle, passed to getBackupFunction.
 + * @param {string} cardName Passed to backupChat.
 + * @param {string} backupDirectory Passed to backupChat.
 + */
 +export async function trySaveChat(chatData, filePath, skipIntegrityCheck = false, handle, cardName, backupDirectory) {
 +    const jsonlData = chatData?.map(m => JSON.stringify(m)).join('\n');
++
 +    const doIntegrityCheck = (checkIntegrity && !skipIntegrityCheck);
 +    const chatIntegritySlug = doIntegrityCheck ? chatData?.[0]?.chat_metadata?.integrity : undefined;
++
 +    if (chatIntegritySlug && !await checkChatIntegrity(filePath, chatIntegritySlug)) {
 +        throw new IntegrityMismatchError(`Chat integrity check failed for "${filePath}". The expected integrity slug was "${chatIntegritySlug}".`);
 +    }
 +    tryWriteFileSync(filePath, jsonlData);
 +    getBackupFunction(handle)(backupDirectory, cardName, jsonlData);
 +}
++
 router.post('/save', validateAvatarUrlMiddleware, async function (request, response) {
     try {
         const directoryNamehandle = String(request.body.avatar_url)user.replace('profile.png', '')handle;
 +        const cardName = String(request.body.avatar_url).replace('.png', '');
         const chatData = request.body.chat;
 -        const jsonlData = chatData.map(JSON.stringify).join('\n');
 +        const chatFileName = `${String(request.body.file_name)}.jsonl`;
 -        const fileName = `${String(request.body.file_name)}.jsonl`;
 +        const chatFilePath = path.join(request.user.directories.chats, cardName, sanitize(chatFileName));
 -        const filePath = path.join(request.user.directories.chats, directoryName, sanitize(fileName));
++
         if (checkIntegrity && !request.bodyArray.forceisArray(chatData)) {
 -            const integritySlug = chatData?.[0]?.chat_metadata?.integrity;
 +            await trySaveChat(chatData, chatFilePath, request.body.force, handle, cardName, request.user.directories.backups);
 -            const isIntact = await checkChatIntegrity(filePath, integritySlug);
 +            return response.send({ ok: true });
 -            if (!isIntact) {
 +        } else {
 -                console.error(`Chat integrity check failed for ${filePath}`);
 +            return response.status(400).send({ error: 'The request\'s body.chat is not an array.' });
 -                return response.status(400).send({ error: 'integrity' });
 -            }
+        }
 -        writeFileAtomicSync(filePath, jsonlData, 'utf8');
 -        getBackupFunction(request.user.profile.handle)(request.user.directories.backups, directoryName, jsonlData);
 -        return response.send({ result: 'ok' });
     } catch (error) {
 +        if (error instanceof IntegrityMismatchError) {
 +            console.error(error.message);
 +            return response.status(400).send({ error: 'integrity' });
 +        }
         console.error(error);
 -        return response.send(error);
 +        return response.status(500).send({ error: 'An error has occurred, see the console logs for more information.' });
+    }
 });
 +/**
 + * Gets the chat as an object.
 + * @param {string} chatFilePath The full chat file path.
 + * @returns {Array}} If the chatFilePath cannot be read, this will return [].
 + */
 +export function getChatData(chatFilePath) {
 +    let chatData = [];
++
 +    const chatJSON = tryReadFileSync(chatFilePath) ?? '';
 +    if (chatJSON.length > 0) {
 +        const lines = chatJSON.split('\n');
 +        // Iterate through the array of strings and parse each line as JSON
 +        chatData = lines.map(line => tryParse(line)).filter(x => x);
 +    } else {
 +        console.warn(`File not found: ${chatFilePath}. The chat does not exist or is empty.`);
 +    }
++
 +    return chatData;
 +}
++
 router.post('/get', validateAvatarUrlMiddleware, function (request, response) {
     try {
         const dirName = String(request.body.avatar_url).replace('.png', '');

             return response.send({});
+        }
         const fileNamechatFileName = `${String(request.body.file_name)}.jsonl`;
         const filePathchatFilePath = path.join(directoryPath, sanitize(fileNamechatFileName));
 -        const chatFileExists = fs.existsSync(filePath);
 -        if (!chatFileExists) {
 +        return response.send(getChatData(chatFilePath));
 -            return response.send({});
 -        }
+-
 -        const data = fs.readFileSync(filePath, 'utf8');
 -        const lines = data.split('\n');
+-
 -        // Iterate through the array of strings and parse each line as JSON
 -        const jsonData = lines.map((l) => { try { return JSON.parse(l); } catch (_) { return; } }).filter(x => x);
 -        return response.send(jsonData);
     } catch (error) {
         console.error(error);
         return response.send({});

+        }
         const dirName = String(request.body.avatar_url).replace('.png', '');
         const fileNamechatFileName = String(request.body.chatfile);
         const filePathchatFilePath = path.join(request.user.directories.chats, dirName, sanitize(fileNamechatFileName));
 -        const chatFileExists = fs.existsSync(filePath);
 +        //Return success if the file was deleted.
+-
 +        if (tryDeleteFile(chatFilePath)) {
 -        if (!chatFileExists) {
 +            return response.send({ ok: true });
 -            console.error(`Chat file not found '${filePath}'`);
 +        } else {
 +            console.error('The chat file was not deleted.');
             return response.sendStatus(400);
+        }
+-
 -        fs.unlinkSync(filePath);
 -        console.info(`Deleted chat file: ${filePath}`);
 -        return response.send('ok');
     } catch (error) {
         console.error(error);
         return response.sendStatus(500);

+    }
     const id = request.body.id;
     const pathToFilechatFilePath = path.join(request.user.directories.groupChats, `${id}.jsonl`);
+-
 -    if (fs.existsSync(pathToFile)) {
 -        const data = fs.readFileSync(pathToFile, 'utf8');
 -        const lines = data.split('\n');
 -        // Iterate through the array of strings and parse each line as JSON
 +    return response.send(getChatData(chatFilePath));
 -        const jsonData = lines.map(line => tryParse(line)).filter(x => x);
 -        return response.send(jsonData);
 -    } else {
 -        return response.send([]);
 -    }
 });
 router.post('/group/delete', (request, response) => {
 +    try {
         if (!request.body || !request.body.id) {
             return response.sendStatus(400);
+        }
         const id = request.body.id;
             const pathToFilechatFilePath = path.join(request.user.directories.groupChats, `${id}.jsonl`);
 -    if (fs.existsSync(pathToFile)) {
 +        //Return success if the file was deleted.
 -        fs.unlinkSync(pathToFile);
 +        if (tryDeleteFile(chatFilePath)) {
             return response.send({ ok: true });
 +        } else {
 +            console.error('The group chat file was not deleted.\'');
 +            return response.sendStatus(400);
 +        }
 +    } catch (error) {
 +        console.error(error);
 +        return response.sendStatus(500);
+    }
+-
 -    return response.send({ error: true });
 });
 router.post('/group/save', async function (request, response) => {
     try {
         if (!request.body || !request.body.id) {
             return response.sendStatus(400);
+        }
         const id = request.body.id;
         const filePathhandle = path.join(request.user.directories.groupChats, sanitize(`${id}profile.jsonl`))handle;
+-
 +        const chatFilePath = path.join(request.user.directories.groupChats, sanitize(`${id}.jsonl`));
 -        if (!fs.existsSync(request.user.directories.groupChats)) {
 -            fs.mkdirSync(request.user.directories.groupChats, { recursive: true });
 -        }
+-
         const chatData = request.body.chat;
 -        const jsonlData = chatData.map(JSON.stringify).join('\n');
         if (checkIntegrity && !request.bodyArray.forceisArray(chatData)) {
 -            const integritySlug = chatData?.[0]?.chat_metadata?.integrity;
 +            await trySaveChat(chatData, chatFilePath, request.body.force, handle, String(id), request.user.directories.backups);
 -            const isIntact = await checkChatIntegrity(filePath, integritySlug);
 +            return response.send({ ok: true });
 -            if (!isIntact) {
 -                console.error(`Chat integrity check failed for ${filePath}`);
 -                return response.status(400).send({ error: 'integrity' });
+        }
 +        else {
 +            return response.status(400).send({ error: 'The request\'s body.chat is not an array.' });
+        }
+-
 -        writeFileAtomicSync(filePath, jsonlData, 'utf8');
 -        getBackupFunction(request.user.profile.handle)(request.user.directories.backups, String(id), jsonlData);
 -        return response.send({ ok: true });
     } catch (error) {
 +        if (error instanceof IntegrityMismatchError) {
 +            console.error(error.message);
 +            return response.status(400).send({ error: 'integrity' });
 +        }
         console.error(error);
         return response.status(500).send({ error: true'An error has occurred, see the console logs for more information.' });
+    }
 });

         // Search logic
         for (const chatFile of chatFiles) {
             const data = fs.readFileSyncgetChatData(chatFile.path, 'utf8');
             const messages = data.splitfilter(x => x && typeof x.mes === '\nstring');
 -                .map(line => { try { return JSON.parse(line); } catch (_) { return null; } })
 -                .filter(x => x && typeof x.mes === 'string');
             if (query && messages.length === 0) {
                 continue;

src/util.js+80 -0 Viewed

 import { promises as dnsPromise } from 'node:dns';
 import os from 'node:os';
 import crypto from 'node:crypto';
 +import readline from 'node:readline';
 import yaml from 'yaml';
 import { sync as commandExistsSync } from 'command-exists';

 import bytes from 'bytes';
 import { LOG_LEVELS, CHAT_COMPLETION_SOURCES, MEDIA_REQUEST_TYPE } from './constants.js';
 import { serverDirectory } from './server-directory.js';
 +import { sync as writeFileAtomicSync } from 'write-file-atomic';
 import { isFirefox } from './express-common.js';
 /**

+}
 /**
 + * Writes to a file, creating it's parent directories if needed.
 + * @param {string} filePath
 + * @param {string} data
 + */
 +export function tryWriteFileSync(filePath, data) {
 +    const directory = path.dirname(filePath);
 +    //Ensure the directory exists.
 +    if (!fs.existsSync(directory)) {
 +        fs.mkdirSync(directory, { recursive: true });
 +    }
 +    writeFileAtomicSync(filePath, data, 'utf8');
 +}
++
 +/**
 +* Attempts to read a file as utf8.
 +* @param {string} filePath
 +* @returns {string|null}
 +*/
 +export function tryReadFileSync(filePath) {
 +    try {
 +        if (fs.existsSync(filePath)) {
 +            return fs.readFileSync(filePath, 'utf8');
 +        }
 +    } catch (error) {
 +        console.error(`Error reading ${filePath}: ${error.message}`);
 +    }
 +    return null;
 +}
++
 +/**
 +* Attempts to delete a file.
 +* @param {string} filePath Target file.
 +* @returns {boolean} Returns true if the file was found and deleted.
 +*/
 +export function tryDeleteFile(filePath) {
 +    if (fs.existsSync(filePath)) {
 +        fs.unlinkSync(filePath);
 +        console.info(`Deleted file: ${filePath}`);
 +        return true;
 +    } else {
 +        console.error(`File not found '${filePath}'`);
 +        return false;
 +    }
 +}
++
 +/**
 + * Reads the first line of a file asynchronously.
 + * @param {string} filePath Path to the file
 + * @returns {Promise<string>} The first line of the file
 + */
 +export function readFirstLine(filePath) {
 +    const stream = fs.createReadStream(filePath, { encoding: 'utf8' });
 +    const rl = readline.createInterface({ input: stream });
 +    return new Promise((resolve, reject) => {
 +        let resolved = false;
 +        rl.on('line', line => {
 +            resolved = true;
 +            rl.close();
 +            stream.close();
 +            resolve(line);
 +        });
++
 +        rl.on('error', error => {
 +            resolved = true;
 +            reject(error);
 +        });
++
 +        // Handle empty files
 +        stream.on('end', () => {
 +            if (!resolved) {
 +                resolved = true;
 +                resolve('');
 +            }
 +        });
 +    });
 +}
++
 +/**
  * If the file is an image, and the request's user agent matches Firefox, then the response's headers are set to invalidate the cache.
  * Without this, Firefox ignores updated images even after a refresh.
  * https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Cache-Control