Refactor /search to use per-line async parsing (#5085) * Refactor /search to use per-line async parsing Supersedes #5047 * Use named type * Cache query fragments, add file name match * Skip invalid chat files in search * Remove extensions from /search results * Use file_id when appropriate * Revert "Use file_id when appropriate" This reverts commit aaa0274b53da6a677183b6f923163053cfd6d569. * Clean-up file extension handling * Move extension append down for non-group

3f8acaad4e2ac6251e9b714726eed180c2de7fb7

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

Parents: cf587bc76643febe026c01a90926b62f7cc468ff

public/script.js+10 -19 Viewed

 async function displayChats(searchQuery, currentChat, displayName, avatarImg, selected_group, highlightNames) {
     try {
 -        const trimExtension = (fileName) => String(fileName).replace('.jsonl', '');
+-
         const response = await fetch('/api/chats/search', {
             method: 'POST',
             headers: getRequestHeaders(),

         filteredData.sort((a, b) => sortMoments(timestampToMoment(a.last_mes), timestampToMoment(b.last_mes)));
         for (const chat of filteredData) {
             const isSelected = trimExtension(currentChat) === trimExtension(chat.file_name);
             const template = $('#past_chat_template .select_chat_block_wrapper').clone();
             template.find('.select_chat_block').attr('file_name', chat.file_name);
             template.find('.avatar img').attr('src', avatarImg);

         if (group) {
             await deleteGroupChat(group, chatFile);
         } else {
             await delChat(`${chatFile}.jsonl`);
+        }
         if (fromSlashCommand) {  // When called from `/delchat` command, don't re-open the history view.

     $(document).on('click', '.PastChat_cross', async function (e, { fromSlashCommand = false } = {}) {
         e.stopPropagation();
         chat_file_for_delconst deleteFileName = $(this).attr('file_name');
         console.debug('detected cross click for' + chat_file_for_deldeleteFileName);
         // Skip confirmation if called from a slash command.
         if (fromSlashCommand) {
             await handleDeleteChat(chat_file_for_deldeleteFileName, selected_group, true);
             return;
+        }
         const result = await callGenericPopup('<h3>' + t`Delete the Chat File?` + '</h3>', POPUP_TYPE.CONFIRM);
         if (result === POPUP_RESULT.AFFIRMATIVE) {
             await handleDeleteChat(chat_file_for_deldeleteFileName, selected_group, false);
+        }
     });

         $('#character_popup').css('display', 'none');
     });
     $('#dialogue_popup_ok').on('click', async function (_e, customData) {
 -        const fromSlashCommand = customData?.fromSlashCommand || false;
         dialogueCloseStop = false;
         $('#shadow_popup').transition({
             opacity: 0,

             $('#dialogue_popup').removeClass('wide_dialogue_popup');
         }, animation_duration);
 -        if (popup_type == 'del_chat') {
 -            await handleDeleteChat(chat_file_for_del, selected_group, fromSlashCommand);
 -        }
+-
         if (dialogueResolve) {
             if (popup_type == 'input') {
                 dialogueResolve($('#dialogue_popup_input').val());

     $(document).on('click', '.renameChatButton', async function (e) {
         e.stopPropagation();
         const oldFileNameFulloldFileName = $(this).closest('.select_chat_block_wrapper').find('.select_chat_block_filename').text();
 -        const oldFileName = oldFileNameFull.replace('.jsonl', '');
         const popupText = await renderTemplateAsync('chatRename');
         const newName = await callGenericPopup(popupText, POPUP_TYPE.INPUT, oldFileName);

         e.stopPropagation();
         const format = $(this).data('format') || 'txt';
         await saveChatConditional();
         const filenamefullfilename = $(this).closest('.select_chat_block_wrapper').find('.select_chat_block_filename').text();
         console.log(`exporting ${filenamefullfilename} in ${format} format`);
 -        const filename = filenamefull.replace('.jsonl', '');
         const body = {
             is_group: !!selected_group,
             avatar_url: characters[this_chid]?.avatar,

public/scripts/bookmarks.js+1 -1 Viewed

         const fileName = $(this).hasClass('mes_bookmark')
             ? $(this).closest('.mes').attr('bookmark_link')
             : $(this).attr('file_name').replace('.jsonl', '');
         if (!fileName) {
             return;

src/endpoints/chats.js+63 -60 Viewed

+}
 /**
  * Gets a preview message from an array ofa chat messagesmessage string.
  * @param {Array<Object>string} messages[lastMessage] - Array of chat messages, each withThe amessage 'mes'to propertytruncate
  * @returns {string} A truncated preview of the last message or empty string if no messages
  */
 function getPreviewMessage(messageslastMessage) {
     const strlen = 400;
 -    const lastMessage = messages[messages.length - 1]?.mes;
     if (!lastMessage) {
         return '';

  * @property {string} [file_size] - The size of the chat file in a human-readable format
  * @property {number} [chat_items] - The number of chat items in the file
  * @property {string} [mes] - The last message in the chat
  * @property {number|string} [last_mes] - The timestamp of the last message
  * @property {object} [chat_metadata] - Additional chat metadata
 + * @property {boolean} [match] - Whether the chat matches the search criteria
  */
 /**

  * @param {string} pathToFile - Path to the chat file
  * @param {object} additionalData - Additional data to include in the result
  * @param {boolean} withMetadata - Whether to read chat metadata
 + * @param {ChatMatchFunction|null} matcher - Optional function to match messages
  * @returns {Promise<ChatInfo>}
 + *
 + * @typedef {(text: string) => boolean} ChatMatchFunction
  */
 export async function getChatInfo(pathToFile, additionalData = {}, withMetadata = false, matcher = null) {
     return new Promise(async (res) => {
         const parsedPath = path.parse(pathToFile);
         const stats = await fs.promises.stat(pathToFile);
 +        const hasMatcher = (typeof matcher === 'function');
         const chatData = {
 +            match: false,
             file_id: parsedPath.name,
             file_name: parsedPath.base,
             file_size: formatBytes(stats.size),

         let lastLine;
         let itemCounter = 0;
 +        let hasAnyMatch = false;
         rl.on('line', (line) => {
             if (withMetadata && itemCounter === 0) {
                 const jsonData = tryParse(line);

                     chatData.chat_metadata = jsonData.chat_metadata;
+                }
+            }
 +            // Skip matching if any match was already found
 +            if (hasMatcher && !hasAnyMatch && itemCounter > 0) {
 +                const jsonData = tryParse(line);
 +                if (jsonData && matcher(jsonData.mes || '')) {
 +                    hasAnyMatch = true;
 +                }
 +            }
             itemCounter++;
             lastLine = line;
         });

                     chatData.chat_items = (itemCounter - 1);
                     chatData.mes = jsonData.mes || '[The message is empty]';
                     chatData.last_mes = jsonData.send_date || new Date(Math.round(stats.mtimeMs)).toISOString();
 +                    chatData.match = hasMatcher ? hasAnyMatch : true;
                     res(chatData);
                 } else {

+    }
 });
 router.post('/search', validateAvatarUrlMiddleware, async function (request, response) {
     try {
         const { query, avatar_url, group_id } = request.body;
++
 +        /** @type {string[]} */
         let chatFiles = [];
         if (group_id) {
             // Find group's chat IDs first
             const groupDir = path.join(request.user.directories.groups);
             const groupFiles = fs.readdirSync(groupDir)
                 .filter(file => filepath.endsWithextname(file) === '.json'));
             let targetGroup;
             for (const groupFile of groupFiles) {

+                }
+            }
             if (!Array.isArray(targetGroup?.chats)) {
                 return response.send([]);
+            }
             // Find group chat files for given group ID
             const groupChatsDir = path.join(request.user.directories.groupChats);
             chatFiles = targetGroup.chats
                 .map(chatId => path.join(groupChatsDir, `${chatId}.jsonl`))
 -                    const filePath = path.join(groupChatsDir, `${chatId}.jsonl`);
 +                .filter(fileName => fs.existsSync(fileName));
 -                    if (!fs.existsSync(filePath)) return null;
 -                    const stats = fs.statSync(filePath);
 -                    return {
 -                        file_name: chatId,
 -                        file_size: formatBytes(stats.size),
 -                        path: filePath,
 -                    };
 -                })
 -                .filter(x => x);
         } else {
             // Regular character chat directory
             const character_name = avatar_url.replace('.png', '');

+            }
             chatFiles = fs.readdirSync(directoryPath)
                 .filter(file => filepath.endsWithextname(file) === '.jsonl'))
                 .map(fileName => {path.join(directoryPath, fileName));
 -                    const filePath = path.join(directoryPath, fileName);
 -                    const stats = fs.statSync(filePath);
 -                    return {
 -                        file_name: fileName,
 -                        file_size: formatBytes(stats.size),
 -                        path: filePath,
 -                    };
 -                });
+        }
 +        /**
 +         * @type {SearchChatResult[]}
 +         * @typedef {object} SearchChatResult
 +         * @property {string} [file_name] - The name of the chat file
 +         * @property {string} [file_size] - The size of the chat file in a human-readable format
 +         * @property {number} [message_count] - The number of messages in the chat
 +         * @property {number|string} [last_mes] - The timestamp of the last message
 +         * @property {string} [preview_message] - A preview of the last message
 +         */
         const results = [];
 -        // Search logic
 +        /** @type {string[]} */
 -        for (const chatFile of chatFiles) {
 +        const fragments = query ? query.trim().toLowerCase().split(/\s+/).filter(x => x) : [];
 -            const data = getChatData(chatFile.path);
 -            const messages = data.filter(x => x && typeof x.mes === 'string');
 -            if (query && messages.length === 0) {
 +        /** @type {ChatMatchFunction} */
 -                continue;
 +        const hasTextMatch = (text) => {
 +            if (fragments.length === 0) {
 +                return true;
+            }
 +            const loweredText = String(text).toLowerCase();
 +            return fragments.every(fragment => loweredText.includes(fragment));
 +        };
 -            const lastMessage = messages[messages.length - 1];
 +        for (const chatFile of chatFiles) {
 -            const lastMesDate = lastMessage?.send_date || new Date(fs.statSync(chatFile.path).mtimeMs).toISOString();
 +            const chatInfo = await getChatInfo(chatFile, {}, false, hasTextMatch);
 +            const hasMatch = hasTextMatch(chatInfo.file_id ?? '') || chatInfo.match;
             // If noSkip searchcorrupted query,or justinvalid returnchat metadatafiles
             if (!querychatInfo.file_name) {
 -                results.push({
 -                    file_name: chatFile.file_name,
 -                    file_size: chatFile.file_size,
 -                    message_count: messages.length,
 -                    last_mes: lastMesDate,
 -                    preview_message: getPreviewMessage(messages),
 -                });
                 continue;
+            }
 -            // Search through title and messages of the chat
 +            // Empty chats without a file name match are skipped when searching with a query
 -            const fragments = query.trim().toLowerCase().split(/\s+/).filter(x => x);
 +            if (query && chatInfo.chat_items === 0 && !hasMatch) {
 -            const text = [path.parse(chatFile.path).name, ...messages.map(message => message?.mes)].join('\n').toLowerCase();
 +                continue;
 -            const hasMatch = fragments.every(fragment => text.includes(fragment));
 +            }
 -            if (hasMatch) {
 +            // If no search query or a match was found, include the chat in results
 +            if (!query || hasMatch) {
                 results.push({
                     file_name: chatFilechatInfo.file_namefile_id,
                     file_size: chatFilechatInfo.file_size,
                     message_count: messageschatInfo.lengthchat_items,
                     last_mes: lastMesDatechatInfo.last_mes,
                     preview_message: getPreviewMessage(messageschatInfo.mes),
                 });
+            }
+        }
 -        // Sort by last message date descending
 -        results.sort((a, b) => new Date(b.last_mes).getTime() - new Date(a.last_mes).getTime());
         return response.send(results);
+-
     } catch (error) {
         console.error('Chat search error:', error);
         return response.status(500).json({ error: 'Search failed' });