Merge pull request #3698 from SillyTavern/integrity Add integrity check when saving solo chat files

e2f94896848410c41cc5611caadeb456a0ed6d7f

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

Parents: 03d590415b9f600082965b1f52ab39bacfa3f5b4 c92ca8dbfbab1bd1d221ebf86680f05cc2591efd

default/config.yaml+2 -0 Viewed

   chat:
     # Enable automatic chat backups
     enabled: true
 +    # Verify integrity of chat files before saving
 +    checkIntegrity: true
     # Maximum number of chat backups to keep per user (starting from the most recent). Set to -1 to keep all backups.
     maxTotalBackups: -1
     # Interval in milliseconds to throttle chat backups per user

public/script.js+62 -37 Viewed

     copyText,
     escapeHtml,
     saveBase64AsFile,
 +    uuidv4,
 } from './scripts/utils.js';
 import { debounce_timeout } from './scripts/constants.js';

     }, DEFAULT_SAVE_EDIT_TIMEOUT);
+}
 -export async function saveChat(chatName, withMetadata, mesId) {
 +/**
 + * Saves the chat to the server.
 + * @param {object} [options] - Additional options.
 + * @param {string} [options.chatName] The name of the chat file to save to
 + * @param {object} [options.withMetadata] Additional metadata to save with the chat
 + * @param {number} [options.mesId] The message ID to save the chat up to
 + * @param {boolean} [options.force] Force the saving despire the integrity check result
 + *
 + * @returns {Promise<void>}
 + */
 +export async function saveChat({ chatName, withMetadata, mesId, force = false } = {}) {
 +    if (arguments.length > 1 && typeof arguments[0] !== 'object') {
 +        console.trace('saveChat called with positional arguments. Please use an object instead.');
 +        [chatName, withMetadata, mesId, force] = arguments;
 +    }
++
     const metadata = { ...chat_metadata, ...(withMetadata || {}) };
     const fileName = chatName ?? characters[this_chid]?.chat;

             toastr.error(t`Trying to save group chat with regular saveChat function. Aborting to prevent corruption.`);
             throw new Error('Group chat saved from saveChat');
+        }
 -        /*
 -        if (item.is_user) {
 -            //var str = item.mes.replace(`${name1}:`, `${name1}:`);
 -            //chat[i].mes = str;
 -            //chat[i].name = name1;
 -        } else if (i !== chat.length - 1 && chat[i].swipe_id !== undefined) {
 -            //  delete chat[i].swipes;
 -            //  delete chat[i].swipe_id;
 -        }
 -        */
     });
     const trimmed_chattrimmedChat = (mesId !== undefined && mesId >= 0 && mesId < chat.length)
         ? chat.slice(0, parseIntNumber(mesId) + 1)
         : chat.slice();
     varconst save_chatchatToSave = [
+        {
             user_name: name1,
             character_name: name2,
             create_date: chat_create_date,
             chat_metadata: metadata,
         },
         ...trimmed_chattrimmedChat,
     ];
 -    return jQuery.ajax({
 -        type: 'POST',
 -        url: '/api/chats/save',
 -        data: JSON.stringify({
 -            ch_name: characters[this_chid].name,
 -            file_name: fileName,
 -            chat: save_chat,
 -            avatar_url: characters[this_chid].avatar,
 -        }),
 -        beforeSend: function () {
 -        },
 +    try {
 -        cache: false,
 +        const result = await fetch('/api/chats/save', {
         dataType            method: 'jsonPOST',
         contentType            cache: 'application/jsonno-cache',
 -        success: function (data) { },
 +            headers: getRequestHeaders(),
 -        error: function (jqXHR, exception) {
 +            body: JSON.stringify({
 -            toastr.error(t`Check the server connection and reload the page to prevent data loss.`, t`Chat could not be saved`);
 +                ch_name: characters[this_chid].name,
 -            console.log(exception);
 +                file_name: fileName,
 -            console.log(jqXHR);
 +                chat: chatToSave,
 -        },
 +                avatar_url: characters[this_chid].avatar,
 -    });
 +                force: force,
 +            }),
 +        });
++
 +        if (result.ok) {
 +            return;
 +        }
++
 +        const errorData = await result.json();
 +        const isIntegrityError = errorData?.error === 'integrity' && !force;
 +        if (!isIntegrityError) {
 +            throw new Error(result.statusText);
 +        }
++
 +        const forceSaveConfirmed = await Popup.show.confirm(
 +            t`ERROR: Chat integrity check failed.`,
 +            t`Continuing the operation may result in data loss. Would you like to overwrite the chat file anyway? Pressing "NO" will cancel the save operation.`,
 +            { okButton: t`Yes, overwrite`, cancelButton: t`No, cancel` },
 +        ) === POPUP_RESULT.AFFIRMATIVE;
++
 +        if (forceSaveConfirmed) {
 +            await saveChat({ chatName, withMetadata, mesId, force: true });
 +        }
 +    } catch (error) {
 +        console.error(error);
 +        toastr.error(t`Check the server connection and reload the page to prevent data loss.`, t`Chat could not be saved`);
 +    }
+}
 async function read_avatar_load(input) {

         } else {
             chat_create_date = humanizedDateTime();
+        }
 +        if (!chat_metadata['integrity']) {
 +            chat_metadata['integrity'] = uuidv4();
 +        }
         await getChatResult();
         eventSource.emit('chatLoaded', { detail: { id: this_chid, character: characters[this_chid] } });

public/scripts/bookmarks.js+2 -2 Viewed

     if (selected_group) {
         await saveGroupBookmarkChat(selected_group, name, newMetadata, mesId);
     } else {
         await saveChat({ chatName: name, withMetadata: newMetadata, mesId });
+    }
     // append to branches list if it exists
     // otherwise create it

     if (selected_group) {
         await saveGroupBookmarkChat(selected_group, name, newMetadata, mesId);
     } else {
         await saveChat({ chatName: name, withMetadata: newMetadata, mesId });
+    }
     lastMes.extra['bookmark_link'] = name;

src/endpoints/chats.js+68 -1 Viewed

 const isBackupEnabled = !!getConfigValue('backups.chat.enabled', true, 'boolean');
 const maxTotalChatBackups = Number(getConfigValue('backups.chat.maxTotalBackups', -1, 'number'));
 const throttleInterval = Number(getConfigValue('backups.chat.throttleInterval', 10_000, 'number'));
 +const checkIntegrity = !!getConfigValue('backups.chat.checkIntegrity', true, 'boolean');
 /**
  * Saves a chat to the backups directory.

     return chat.map(obj => JSON.stringify(obj)).join('\n');
+}
 +/**
 + * Reads the first line of a file asynchronously.
 + * @param {string} filePath Path to the file
 + * @returns {Promise<string>} The first line of the file
 + */
 +function readFirstLine(filePath) {
 +    const stream = fs.createReadStream(filePath, { encoding: 'utf8' });
 +    const rl = readline.createInterface({ input: stream });
 +    return new Promise((resolve, reject) => {
 +        let resolved = false;
 +        rl.on('line', line => {
 +            resolved = true;
 +            rl.close();
 +            stream.close();
 +            resolve(line);
 +        });
++
 +        rl.on('error', error => {
 +            resolved = true;
 +            reject(error);
 +        });
++
 +        // Handle empty files
 +        stream.on('end', () => {
 +            if (!resolved) {
 +                resolved = true;
 +                resolve('');
 +            }
 +        });
 +    });
 +}
++
 +/**
 + * Checks if the chat being saved has the same integrity as the one being loaded.
 + * @param {string} filePath Path to the chat file
 + * @param {string} integritySlug Integrity slug
 + * @returns {Promise<boolean>} Whether the chat is intact
 + */
 +async function checkChatIntegrity(filePath, integritySlug) {
 +    // If the chat file doesn't exist, assume it's intact
 +    if (!fs.existsSync(filePath)) {
 +        return true;
 +    }
++
 +    // Parse the first line of the chat file as JSON
 +    const firstLine = await readFirstLine(filePath);
 +    const jsonData = tryParse(firstLine);
 +    const chatIntegrity = jsonData?.chat_metadata?.integrity;
++
 +    // If the chat has no integrity metadata, assume it's intact
 +    if (!chatIntegrity) {
 +        return true;
 +    }
++
 +    // Check if the integrity matches
 +    return chatIntegrity === integritySlug;
 +}
++
 export const router = express.Router();
 router.post('/save', validateAvatarUrlMiddleware, async function (request, response) {
     try {
         const directoryName = String(request.body.avatar_url).replace('.png', '');
         const chatData = request.body.chat;
         const jsonlData = chatData.map(JSON.stringify).join('\n');
         const fileName = `${String(request.body.file_name)}.jsonl`;
         const filePath = path.join(request.user.directories.chats, directoryName, sanitize(fileName));
 +        if (checkIntegrity && !request.body.force) {
 +            const integritySlug = chatData?.[0]?.chat_metadata?.integrity;
 +            const isIntact = await checkChatIntegrity(filePath, integritySlug);
 +            if (!isIntact) {
 +                console.error(`Chat integrity check failed for ${filePath}`);
 +                return response.status(400).send({ error: 'integrity' });
 +            }
 +        }
         writeFileAtomicSync(filePath, jsonlData, 'utf8');
         getBackupFunction(request.user.profile.handle)(request.user.directories.backups, directoryName, jsonlData);
         return response.send({ result: 'ok' });