Refactor /search to use per-line async parsing (#5085) * Refactor /search to use per-line async parsing Supersedes #5047 * Use named type * Cache query fragments, add file name match * Skip invalid chat files in search * Remove extensions from /search results * Use file_id when appropriate * Revert "Use file_id when appropriate" This reverts commit aaa0274b53da6a677183b6f923163053cfd6d569. * Clean-up file extension handling * Move extension append down for non-group

3f8acaad4e2ac6251e9b714726eed180c2de7fb7

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

Parents: cf587bc76643febe026c01a90926b62f7cc468ff

public/script.js+10 -19 Viewed

 async function displayChats(searchQuery, currentChat, displayName, avatarImg, selected_group, highlightNames) {
     try {
-        const trimExtension = (fileName) => String(fileName).replace('.jsonl', '');
         const response = await fetch('/api/chats/search', {
             method: 'POST',
             headers: getRequestHeaders(),

         filteredData.sort((a, b) => sortMoments(timestampToMoment(a.last_mes), timestampToMoment(b.last_mes)));
         for (const chat of filteredData) {
-            const isSelected = trimExtension(currentChat) === trimExtension(chat.file_name);
+            const isSelected = currentChat === chat.file_name;
             const template = $('#past_chat_template .select_chat_block_wrapper').clone();
             template.find('.select_chat_block').attr('file_name', chat.file_name);
             template.find('.avatar img').attr('src', avatarImg);

         if (group) {
             await deleteGroupChat(group, chatFile);
         } else {
-            await delChat(chatFile);
+            await delChat(`${chatFile}.jsonl`);
+        }
         if (fromSlashCommand) {  // When called from `/delchat` command, don't re-open the history view.

     $(document).on('click', '.PastChat_cross', async function (e, { fromSlashCommand = false } = {}) {
         e.stopPropagation();
-        chat_file_for_del = $(this).attr('file_name');
+        const deleteFileName = $(this).attr('file_name');
-        console.debug('detected cross click for' + chat_file_for_del);
+        console.debug('detected cross click for' + deleteFileName);
         // Skip confirmation if called from a slash command.
         if (fromSlashCommand) {
-            await handleDeleteChat(chat_file_for_del, selected_group, true);
+            await handleDeleteChat(deleteFileName, selected_group, true);
             return;
+        }
         const result = await callGenericPopup('<h3>' + t`Delete the Chat File?` + '</h3>', POPUP_TYPE.CONFIRM);
         if (result === POPUP_RESULT.AFFIRMATIVE) {
-            await handleDeleteChat(chat_file_for_del, selected_group, false);
+            await handleDeleteChat(deleteFileName, selected_group, false);
+        }
     });

         $('#character_popup').css('display', 'none');
     });
-    $('#dialogue_popup_ok').on('click', async function (_e, customData) {
+    $('#dialogue_popup_ok').on('click', async function (_e) {
-        const fromSlashCommand = customData?.fromSlashCommand || false;
         dialogueCloseStop = false;
         $('#shadow_popup').transition({
             opacity: 0,

             $('#dialogue_popup').removeClass('wide_dialogue_popup');
         }, animation_duration);
-        if (popup_type == 'del_chat') {
-            await handleDeleteChat(chat_file_for_del, selected_group, fromSlashCommand);
-        }
         if (dialogueResolve) {
             if (popup_type == 'input') {
                 dialogueResolve($('#dialogue_popup_input').val());

     $(document).on('click', '.renameChatButton', async function (e) {
         e.stopPropagation();
-        const oldFileNameFull = $(this).closest('.select_chat_block_wrapper').find('.select_chat_block_filename').text();
+        const oldFileName = $(this).closest('.select_chat_block_wrapper').find('.select_chat_block_filename').text();
-        const oldFileName = oldFileNameFull.replace('.jsonl', '');
         const popupText = await renderTemplateAsync('chatRename');
         const newName = await callGenericPopup(popupText, POPUP_TYPE.INPUT, oldFileName);

         e.stopPropagation();
         const format = $(this).data('format') || 'txt';
         await saveChatConditional();
-        const filenamefull = $(this).closest('.select_chat_block_wrapper').find('.select_chat_block_filename').text();
+        const filename = $(this).closest('.select_chat_block_wrapper').find('.select_chat_block_filename').text();
-        console.log(`exporting ${filenamefull} in ${format} format`);
+        console.log(`exporting ${filename} in ${format} format`);
-        const filename = filenamefull.replace('.jsonl', '');
         const body = {
             is_group: !!selected_group,
             avatar_url: characters[this_chid]?.avatar,

public/scripts/bookmarks.js+1 -1 Viewed

         const fileName = $(this).hasClass('mes_bookmark')
             ? $(this).closest('.mes').attr('bookmark_link')
-            : $(this).attr('file_name').replace('.jsonl', '');
+            : $(this).attr('file_name');
         if (!fileName) {
             return;

src/endpoints/chats.js+63 -60 Viewed

+}
 /**
- * Gets a preview message from an array of chat messages
+ * Gets a preview message from a chat message string.
- * @param {Array<Object>} messages - Array of chat messages, each with a 'mes' property
+ * @param {string} [lastMessage] - The message to truncate
  * @returns {string} A truncated preview of the last message or empty string if no messages
  */
-function getPreviewMessage(messages) {
+function getPreviewMessage(lastMessage) {
     const strlen = 400;
-    const lastMessage = messages[messages.length - 1]?.mes;
     if (!lastMessage) {
         return '';

  * @property {string} [file_size] - The size of the chat file in a human-readable format
  * @property {number} [chat_items] - The number of chat items in the file
  * @property {string} [mes] - The last message in the chat
- * @property {number} [last_mes] - The timestamp of the last message
+ * @property {number|string} [last_mes] - The timestamp of the last message
  * @property {object} [chat_metadata] - Additional chat metadata
+ * @property {boolean} [match] - Whether the chat matches the search criteria
  */
 /**

  * @param {string} pathToFile - Path to the chat file
  * @param {object} additionalData - Additional data to include in the result
  * @param {boolean} withMetadata - Whether to read chat metadata
+ * @param {ChatMatchFunction|null} matcher - Optional function to match messages
  * @returns {Promise<ChatInfo>}
+ *
+ * @typedef {(text: string) => boolean} ChatMatchFunction
  */
-export async function getChatInfo(pathToFile, additionalData = {}, withMetadata = false) {
+export async function getChatInfo(pathToFile, additionalData = {}, withMetadata = false, matcher = null) {
     return new Promise(async (res) => {
         const parsedPath = path.parse(pathToFile);
         const stats = await fs.promises.stat(pathToFile);
+        const hasMatcher = (typeof matcher === 'function');
         const chatData = {
+            match: false,
             file_id: parsedPath.name,
             file_name: parsedPath.base,
             file_size: formatBytes(stats.size),

         let lastLine;
         let itemCounter = 0;
+        let hasAnyMatch = false;
         rl.on('line', (line) => {
             if (withMetadata && itemCounter === 0) {
                 const jsonData = tryParse(line);

                     chatData.chat_metadata = jsonData.chat_metadata;
+                }
+            }
+            // Skip matching if any match was already found
+            if (hasMatcher && !hasAnyMatch && itemCounter > 0) {
+                const jsonData = tryParse(line);
+                if (jsonData && matcher(jsonData.mes || '')) {
+                    hasAnyMatch = true;
+                }
+            }
             itemCounter++;
             lastLine = line;
         });

                     chatData.chat_items = (itemCounter - 1);
                     chatData.mes = jsonData.mes || '[The message is empty]';
                     chatData.last_mes = jsonData.send_date || new Date(Math.round(stats.mtimeMs)).toISOString();
+                    chatData.match = hasMatcher ? hasAnyMatch : true;
                     res(chatData);
                 } else {

+    }
 });
-router.post('/search', validateAvatarUrlMiddleware, function (request, response) {
+router.post('/search', validateAvatarUrlMiddleware, async function (request, response) {
     try {
         const { query, avatar_url, group_id } = request.body;
+        /** @type {string[]} */
         let chatFiles = [];
         if (group_id) {
             // Find group's chat IDs first
             const groupDir = path.join(request.user.directories.groups);
             const groupFiles = fs.readdirSync(groupDir)
-                .filter(file => file.endsWith('.json'));
+                .filter(file => path.extname(file) === '.json');
             let targetGroup;
             for (const groupFile of groupFiles) {

+                }
+            }
-            if (!targetGroup?.chats) {
+            if (!Array.isArray(targetGroup?.chats)) {
                 return response.send([]);
+            }
             // Find group chat files for given group ID
             const groupChatsDir = path.join(request.user.directories.groupChats);
             chatFiles = targetGroup.chats
-                .map(chatId => {
+                .map(chatId => path.join(groupChatsDir, `${chatId}.jsonl`))
-                    const filePath = path.join(groupChatsDir, `${chatId}.jsonl`);
+                .filter(fileName => fs.existsSync(fileName));
-                    if (!fs.existsSync(filePath)) return null;
-                    const stats = fs.statSync(filePath);
-                    return {
-                        file_name: chatId,
-                        file_size: formatBytes(stats.size),
-                        path: filePath,
-                    };
-                })
-                .filter(x => x);
         } else {
             // Regular character chat directory
             const character_name = avatar_url.replace('.png', '');

+            }
             chatFiles = fs.readdirSync(directoryPath)
-                .filter(file => file.endsWith('.jsonl'))
+                .filter(file => path.extname(file) === '.jsonl')
-                .map(fileName => {
+                .map(fileName => path.join(directoryPath, fileName));
-                    const filePath = path.join(directoryPath, fileName);
-                    const stats = fs.statSync(filePath);
-                    return {
-                        file_name: fileName,
-                        file_size: formatBytes(stats.size),
-                        path: filePath,
-                    };
-                });
+        }
+        /**
+         * @type {SearchChatResult[]}
+         * @typedef {object} SearchChatResult
+         * @property {string} [file_name] - The name of the chat file
+         * @property {string} [file_size] - The size of the chat file in a human-readable format
+         * @property {number} [message_count] - The number of messages in the chat
+         * @property {number|string} [last_mes] - The timestamp of the last message
+         * @property {string} [preview_message] - A preview of the last message
+         */
         const results = [];
-        // Search logic
+        /** @type {string[]} */
-        for (const chatFile of chatFiles) {
+        const fragments = query ? query.trim().toLowerCase().split(/\s+/).filter(x => x) : [];
-            const data = getChatData(chatFile.path);
-            const messages = data.filter(x => x && typeof x.mes === 'string');
-            if (query && messages.length === 0) {
+        /** @type {ChatMatchFunction} */
-                continue;
+        const hasTextMatch = (text) => {
+            if (fragments.length === 0) {
+                return true;
+            }
+            const loweredText = String(text).toLowerCase();
+            return fragments.every(fragment => loweredText.includes(fragment));
+        };
-            const lastMessage = messages[messages.length - 1];
+        for (const chatFile of chatFiles) {
-            const lastMesDate = lastMessage?.send_date || new Date(fs.statSync(chatFile.path).mtimeMs).toISOString();
+            const chatInfo = await getChatInfo(chatFile, {}, false, hasTextMatch);
+            const hasMatch = hasTextMatch(chatInfo.file_id ?? '') || chatInfo.match;
-            // If no search query, just return metadata
+            // Skip corrupted or invalid chat files
-            if (!query) {
+            if (!chatInfo.file_name) {
-                results.push({
-                    file_name: chatFile.file_name,
-                    file_size: chatFile.file_size,
-                    message_count: messages.length,
-                    last_mes: lastMesDate,
-                    preview_message: getPreviewMessage(messages),
-                });
                 continue;
+            }
-            // Search through title and messages of the chat
+            // Empty chats without a file name match are skipped when searching with a query
-            const fragments = query.trim().toLowerCase().split(/\s+/).filter(x => x);
+            if (query && chatInfo.chat_items === 0 && !hasMatch) {
-            const text = [path.parse(chatFile.path).name, ...messages.map(message => message?.mes)].join('\n').toLowerCase();
+                continue;
-            const hasMatch = fragments.every(fragment => text.includes(fragment));
+            }
-            if (hasMatch) {
+            // If no search query or a match was found, include the chat in results
+            if (!query || hasMatch) {
                 results.push({
-                    file_name: chatFile.file_name,
+                    file_name: chatInfo.file_id,
-                    file_size: chatFile.file_size,
+                    file_size: chatInfo.file_size,
-                    message_count: messages.length,
+                    message_count: chatInfo.chat_items,
-                    last_mes: lastMesDate,
+                    last_mes: chatInfo.last_mes,
-                    preview_message: getPreviewMessage(messages),
+                    preview_message: getPreviewMessage(chatInfo.mes),
                 });
+            }
+        }
-        // Sort by last message date descending
-        results.sort((a, b) => new Date(b.last_mes).getTime() - new Date(a.last_mes).getTime());
         return response.send(results);
     } catch (error) {
         console.error('Chat search error:', error);
         return response.status(500).json({ error: 'Search failed' });