Backported refactor of the chats endpoint from `feat/chat-tree.` (#4870) * Backported cleanup and refactor of chats endpoint from `feat/chat-tree`. * https://github.com/SillyTavern/SillyTavern/pull/4870#discussion_r2599941781 * Error message. https://github.com/SillyTavern/SillyTavern/pull/4870#discussion_r2599941796 https://github.com/SillyTavern/SillyTavern/pull/4870#discussion_r2599941797 * chatIntegritySlug https://github.com/SillyTavern/SillyTavern/pull/4870#discussion_r2599958991 * /save should not hang on empty body.chat. https://github.com/SillyTavern/SillyTavern/pull/4870#discussion_r2599959001 https://github.com/SillyTavern/SillyTavern/pull/4870#discussion_r2599958997 * When the backup directory does not exist, log an error instead of silently failing. * chatData should be an array. https://github.com/SillyTavern/SillyTavern/pull/4870#discussion_r2599990728 * Skip body check. https://github.com/SillyTavern/SillyTavern/pull/4870#discussion_r2599987542 * Change integrity warning to debug. https://github.com/SillyTavern/SillyTavern/pull/4870#discussion_r2600005665 * Return null on error in `tryReadFileSync`. * Update warning message for missing chat file to clarify potential emptiness * Rename IntegrityMismatch to IntegrityMismatchError for clarity and consistency * Fix a type error and formatting * Improve debug messages and documentation for integrity checks in chat saving * Add newline before file writing function for improved readability * Improve debug message formatting for integrity validation in checkChatIntegrity function --------- Co-authored-by: user <user@exmaple.com> Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>

d5320f57cf2619a3a0812d45aa922be315aff690

DeclineThyself <235079501+DeclineThyself@users.noreply.github.com>

Signed

Parents: b5e20f2ff98708b13961d09130b63eda839c191b

src/endpoints/chats.js+127 -123 Viewed

     generateTimestamp,
     removeOldBackups,
     formatBytes,
+    tryWriteFileSync,
+    tryReadFileSync,
+    tryDeleteFile,
+    readFirstLine,
 } from '../util.js';
 const isBackupEnabled = !!getConfigValue('backups.chat.enabled', true, 'boolean');

 /**
  * Saves a chat to the backups directory.
- * @param {string} directory The user's backups directory.
+ * @param {string} directory The user's backup directory.
  * @param {string} name The name of the chat.
- * @param {string} chat The serialized chat to save.
+ * @param {string} data The serialized chat to save.
+ * @param {string} backupPrefix The file prefix. Typically CHAT_BACKUPS_PREFIX.
+ * @returns
  */
-function backupChat(directory, name, chat) {
+function backupChat(directory, name, data, backupPrefix = CHAT_BACKUPS_PREFIX) {
     try {
-        if (!isBackupEnabled || !fs.existsSync(directory)) {
+        if (!isBackupEnabled) { return; }
-            return;
+        if (!fs.existsSync(directory)) {
+            console.error(`The chat couldn't be backed up because no directory exists at ${directory}!`);
+        }
         // replace non-alphanumeric characters with underscores
         name = sanitize(name).replace(/[^a-z0-9]/gi, '_').toLowerCase();
-        const backupFile = path.join(directory, `${CHAT_BACKUPS_PREFIX}${name}_${generateTimestamp()}.jsonl`);
+        const backupFile = path.join(directory, `${backupPrefix}${name}_${generateTimestamp()}.jsonl`);
-        writeFileAtomicSync(backupFile, chat, 'utf-8');
-        removeOldBackups(directory, `${CHAT_BACKUPS_PREFIX}${name}_`);
+        tryWriteFileSync(backupFile, data);
+        removeOldBackups(directory, `${backupPrefix}${name}_`);
         if (isNaN(maxTotalChatBackups) || maxTotalChatBackups < 0) {
             return;
+        }
+        removeOldBackups(directory, backupPrefix, maxTotalChatBackups);
-        removeOldBackups(directory, CHAT_BACKUPS_PREFIX, maxTotalChatBackups);
     } catch (err) {
         console.error(`Could not backup chat for ${name}`, err);
+    }
+}
 /**
- * @type {Map<string, import('lodash').DebouncedFunc<function(string, string, string): void>>}
+ * @type {Map<string, import('lodash').DebouncedFunc<typeof backupChat>>}
  */
 const backupFunctions = new Map();
 /**
  * Gets a backup function for a user.
  * @param {string} handle User handle
- * @returns {function(string, string, string): void} Backup function
+ * @returns {typeof backupChat} Backup function
  */
 function getBackupFunction(handle) {
     if (!backupFunctions.has(handle)) {

+}
 /**
- * Reads the first line of a file asynchronously.
- * @param {string} filePath Path to the file
- * @returns {Promise<string>} The first line of the file
- */
-function readFirstLine(filePath) {
-    const stream = fs.createReadStream(filePath, { encoding: 'utf8' });
-    const rl = readline.createInterface({ input: stream });
-    return new Promise((resolve, reject) => {
-        let resolved = false;
-        rl.on('line', line => {
-            resolved = true;
-            rl.close();
-            stream.close();
-            resolve(line);
-        });
-        rl.on('error', error => {
-            resolved = true;
-            reject(error);
-        });
-        // Handle empty files
-        stream.on('end', () => {
-            if (!resolved) {
-                resolved = true;
-                resolve('');
-            }
-        });
-    });
-}
-/**
  * Checks if the chat being saved has the same integrity as the one being loaded.
  * @param {string} filePath Path to the chat file
  * @param {string} integritySlug Integrity slug

     // If the chat has no integrity metadata, assume it's intact
     if (!chatIntegrity) {
+        console.debug(`File "${filePath}" does not have integrity metadata matching "${integritySlug}". The integrity validation has been skipped.`);
         return true;
+    }

 export const router = express.Router();
+// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error
+class IntegrityMismatchError extends Error {
+    constructor(...params) {
+        // Pass remaining arguments (including vendor specific ones) to parent constructor
+        super(...params);
+        // Maintains proper stack trace for where our error was thrown (non-standard)
+        if (Error.captureStackTrace) {
+            Error.captureStackTrace(this, IntegrityMismatchError);
+        }
+        this.date = new Date();
+    }
+}
+/**
+ * Tries to save the chat data to a file, performing an integrity check if required.
+ * @param {Array} chatData The chat array to save.
+ * @param {string} filePath Target file path for the data.
+ * @param {boolean} skipIntegrityCheck If undefined, the chat's integrity will not be checked.
+ * @param {string} handle The users handle, passed to getBackupFunction.
+ * @param {string} cardName Passed to backupChat.
+ * @param {string} backupDirectory Passed to backupChat.
+ */
+export async function trySaveChat(chatData, filePath, skipIntegrityCheck = false, handle, cardName, backupDirectory) {
+    const jsonlData = chatData?.map(m => JSON.stringify(m)).join('\n');
+    const doIntegrityCheck = (checkIntegrity && !skipIntegrityCheck);
+    const chatIntegritySlug = doIntegrityCheck ? chatData?.[0]?.chat_metadata?.integrity : undefined;
+    if (chatIntegritySlug && !await checkChatIntegrity(filePath, chatIntegritySlug)) {
+        throw new IntegrityMismatchError(`Chat integrity check failed for "${filePath}". The expected integrity slug was "${chatIntegritySlug}".`);
+    }
+    tryWriteFileSync(filePath, jsonlData);
+    getBackupFunction(handle)(backupDirectory, cardName, jsonlData);
+}
 router.post('/save', validateAvatarUrlMiddleware, async function (request, response) {
     try {
-        const directoryName = String(request.body.avatar_url).replace('.png', '');
+        const handle = request.user.profile.handle;
+        const cardName = String(request.body.avatar_url).replace('.png', '');
         const chatData = request.body.chat;
-        const jsonlData = chatData.map(JSON.stringify).join('\n');
+        const chatFileName = `${String(request.body.file_name)}.jsonl`;
-        const fileName = `${String(request.body.file_name)}.jsonl`;
+        const chatFilePath = path.join(request.user.directories.chats, cardName, sanitize(chatFileName));
-        const filePath = path.join(request.user.directories.chats, directoryName, sanitize(fileName));
-        if (checkIntegrity && !request.body.force) {
+        if (Array.isArray(chatData)) {
-            const integritySlug = chatData?.[0]?.chat_metadata?.integrity;
+            await trySaveChat(chatData, chatFilePath, request.body.force, handle, cardName, request.user.directories.backups);
-            const isIntact = await checkChatIntegrity(filePath, integritySlug);
+            return response.send({ ok: true });
-            if (!isIntact) {
+        } else {
-                console.error(`Chat integrity check failed for ${filePath}`);
+            return response.status(400).send({ error: 'The request\'s body.chat is not an array.' });
-                return response.status(400).send({ error: 'integrity' });
-            }
+        }
-        writeFileAtomicSync(filePath, jsonlData, 'utf8');
-        getBackupFunction(request.user.profile.handle)(request.user.directories.backups, directoryName, jsonlData);
-        return response.send({ result: 'ok' });
     } catch (error) {
+        if (error instanceof IntegrityMismatchError) {
+            console.error(error.message);
+            return response.status(400).send({ error: 'integrity' });
+        }
         console.error(error);
-        return response.send(error);
+        return response.status(500).send({ error: 'An error has occurred, see the console logs for more information.' });
+    }
 });
+/**
+ * Gets the chat as an object.
+ * @param {string} chatFilePath The full chat file path.
+ * @returns {Array}} If the chatFilePath cannot be read, this will return [].
+ */
+export function getChatData(chatFilePath) {
+    let chatData = [];
+    const chatJSON = tryReadFileSync(chatFilePath) ?? '';
+    if (chatJSON.length > 0) {
+        const lines = chatJSON.split('\n');
+        // Iterate through the array of strings and parse each line as JSON
+        chatData = lines.map(line => tryParse(line)).filter(x => x);
+    } else {
+        console.warn(`File not found: ${chatFilePath}. The chat does not exist or is empty.`);
+    }
+    return chatData;
+}
 router.post('/get', validateAvatarUrlMiddleware, function (request, response) {
     try {
         const dirName = String(request.body.avatar_url).replace('.png', '');

             return response.send({});
+        }
-        const fileName = `${String(request.body.file_name)}.jsonl`;
+        const chatFileName = `${String(request.body.file_name)}.jsonl`;
-        const filePath = path.join(directoryPath, sanitize(fileName));
+        const chatFilePath = path.join(directoryPath, sanitize(chatFileName));
-        const chatFileExists = fs.existsSync(filePath);
-        if (!chatFileExists) {
+        return response.send(getChatData(chatFilePath));
-            return response.send({});
-        }
-        const data = fs.readFileSync(filePath, 'utf8');
-        const lines = data.split('\n');
-        // Iterate through the array of strings and parse each line as JSON
-        const jsonData = lines.map((l) => { try { return JSON.parse(l); } catch (_) { return; } }).filter(x => x);
-        return response.send(jsonData);
     } catch (error) {
         console.error(error);
         return response.send({});

+        }
         const dirName = String(request.body.avatar_url).replace('.png', '');
-        const fileName = String(request.body.chatfile);
+        const chatFileName = String(request.body.chatfile);
-        const filePath = path.join(request.user.directories.chats, dirName, sanitize(fileName));
+        const chatFilePath = path.join(request.user.directories.chats, dirName, sanitize(chatFileName));
-        const chatFileExists = fs.existsSync(filePath);
+        //Return success if the file was deleted.
+        if (tryDeleteFile(chatFilePath)) {
-        if (!chatFileExists) {
+            return response.send({ ok: true });
-            console.error(`Chat file not found '${filePath}'`);
+        } else {
+            console.error('The chat file was not deleted.');
             return response.sendStatus(400);
+        }
-        fs.unlinkSync(filePath);
-        console.info(`Deleted chat file: ${filePath}`);
-        return response.send('ok');
     } catch (error) {
         console.error(error);
         return response.sendStatus(500);

+    }
     const id = request.body.id;
-    const pathToFile = path.join(request.user.directories.groupChats, `${id}.jsonl`);
+    const chatFilePath = path.join(request.user.directories.groupChats, `${id}.jsonl`);
-    if (fs.existsSync(pathToFile)) {
-        const data = fs.readFileSync(pathToFile, 'utf8');
-        const lines = data.split('\n');
-        // Iterate through the array of strings and parse each line as JSON
+    return response.send(getChatData(chatFilePath));
-        const jsonData = lines.map(line => tryParse(line)).filter(x => x);
-        return response.send(jsonData);
-    } else {
-        return response.send([]);
-    }
 });
 router.post('/group/delete', (request, response) => {
+    try {
         if (!request.body || !request.body.id) {
             return response.sendStatus(400);
+        }
         const id = request.body.id;
-    const pathToFile = path.join(request.user.directories.groupChats, `${id}.jsonl`);
+        const chatFilePath = path.join(request.user.directories.groupChats, `${id}.jsonl`);
-    if (fs.existsSync(pathToFile)) {
+        //Return success if the file was deleted.
-        fs.unlinkSync(pathToFile);
+        if (tryDeleteFile(chatFilePath)) {
             return response.send({ ok: true });
+        } else {
+            console.error('The group chat file was not deleted.\'');
+            return response.sendStatus(400);
+        }
+    } catch (error) {
+        console.error(error);
+        return response.sendStatus(500);
+    }
-    return response.send({ error: true });
 });
-router.post('/group/save', async (request, response) => {
+router.post('/group/save', async function (request, response) {
     try {
         if (!request.body || !request.body.id) {
             return response.sendStatus(400);
+        }
         const id = request.body.id;
-        const filePath = path.join(request.user.directories.groupChats, sanitize(`${id}.jsonl`));
+        const handle = request.user.profile.handle;
+        const chatFilePath = path.join(request.user.directories.groupChats, sanitize(`${id}.jsonl`));
-        if (!fs.existsSync(request.user.directories.groupChats)) {
-            fs.mkdirSync(request.user.directories.groupChats, { recursive: true });
-        }
         const chatData = request.body.chat;
-        const jsonlData = chatData.map(JSON.stringify).join('\n');
-        if (checkIntegrity && !request.body.force) {
+        if (Array.isArray(chatData)) {
-            const integritySlug = chatData?.[0]?.chat_metadata?.integrity;
+            await trySaveChat(chatData, chatFilePath, request.body.force, handle, String(id), request.user.directories.backups);
-            const isIntact = await checkChatIntegrity(filePath, integritySlug);
+            return response.send({ ok: true });
-            if (!isIntact) {
-                console.error(`Chat integrity check failed for ${filePath}`);
-                return response.status(400).send({ error: 'integrity' });
+        }
+        else {
+            return response.status(400).send({ error: 'The request\'s body.chat is not an array.' });
+        }
-        writeFileAtomicSync(filePath, jsonlData, 'utf8');
-        getBackupFunction(request.user.profile.handle)(request.user.directories.backups, String(id), jsonlData);
-        return response.send({ ok: true });
     } catch (error) {
+        if (error instanceof IntegrityMismatchError) {
+            console.error(error.message);
+            return response.status(400).send({ error: 'integrity' });
+        }
         console.error(error);
-        return response.send({ error: true });
+        return response.status(500).send({ error: 'An error has occurred, see the console logs for more information.' });
+    }
 });

         // Search logic
         for (const chatFile of chatFiles) {
-            const data = fs.readFileSync(chatFile.path, 'utf8');
+            const data = getChatData(chatFile.path);
-            const messages = data.split('\n')
+            const messages = data.filter(x => x && typeof x.mes === 'string');
-                .map(line => { try { return JSON.parse(line); } catch (_) { return null; } })
-                .filter(x => x && typeof x.mes === 'string');
             if (query && messages.length === 0) {
                 continue;

src/util.js+80 -0 Viewed

 import { promises as dnsPromise } from 'node:dns';
 import os from 'node:os';
 import crypto from 'node:crypto';
+import readline from 'node:readline';
 import yaml from 'yaml';
 import { sync as commandExistsSync } from 'command-exists';

 import bytes from 'bytes';
 import { LOG_LEVELS, CHAT_COMPLETION_SOURCES, MEDIA_REQUEST_TYPE } from './constants.js';
 import { serverDirectory } from './server-directory.js';
+import { sync as writeFileAtomicSync } from 'write-file-atomic';
 import { isFirefox } from './express-common.js';
 /**

+}
 /**
+ * Writes to a file, creating it's parent directories if needed.
+ * @param {string} filePath
+ * @param {string} data
+ */
+export function tryWriteFileSync(filePath, data) {
+    const directory = path.dirname(filePath);
+    //Ensure the directory exists.
+    if (!fs.existsSync(directory)) {
+        fs.mkdirSync(directory, { recursive: true });
+    }
+    writeFileAtomicSync(filePath, data, 'utf8');
+}
+/**
+* Attempts to read a file as utf8.
+* @param {string} filePath
+* @returns {string|null}
+*/
+export function tryReadFileSync(filePath) {
+    try {
+        if (fs.existsSync(filePath)) {
+            return fs.readFileSync(filePath, 'utf8');
+        }
+    } catch (error) {
+        console.error(`Error reading ${filePath}: ${error.message}`);
+    }
+    return null;
+}
+/**
+* Attempts to delete a file.
+* @param {string} filePath Target file.
+* @returns {boolean} Returns true if the file was found and deleted.
+*/
+export function tryDeleteFile(filePath) {
+    if (fs.existsSync(filePath)) {
+        fs.unlinkSync(filePath);
+        console.info(`Deleted file: ${filePath}`);
+        return true;
+    } else {
+        console.error(`File not found '${filePath}'`);
+        return false;
+    }
+}
+/**
+ * Reads the first line of a file asynchronously.
+ * @param {string} filePath Path to the file
+ * @returns {Promise<string>} The first line of the file
+ */
+export function readFirstLine(filePath) {
+    const stream = fs.createReadStream(filePath, { encoding: 'utf8' });
+    const rl = readline.createInterface({ input: stream });
+    return new Promise((resolve, reject) => {
+        let resolved = false;
+        rl.on('line', line => {
+            resolved = true;
+            rl.close();
+            stream.close();
+            resolve(line);
+        });
+        rl.on('error', error => {
+            resolved = true;
+            reject(error);
+        });
+        // Handle empty files
+        stream.on('end', () => {
+            if (!resolved) {
+                resolved = true;
+                resolve('');
+            }
+        });
+    });
+}
+/**
  * If the file is an image, and the request's user agent matches Firefox, then the response's headers are set to invalidate the cache.
  * Without this, Firefox ignores updated images even after a refresh.
  * https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Cache-Control