Merge pull request #2875 from ceruleandeep/feature/handleChubChatData Fix crasher in script.js:displayChats, add minimal support for importing chat histories in Chub format

ed47d51c93e6a123d8ffd8ae85b4a0012bfba3e7

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

Parents: 90169858274431ed4cdf8bc0cb9ab9c2fba7df6c a2d9526cdfa73d2e4eac0fe662479fe77c5e0e8f

public/script.js+3 -5 Viewed

+            }
             // Check whether `text` {string} includes all of the `fragments` {string[]}.
             function matchFragments(fragments, text) {
                 if (!text || !text.toLowerCase) {return false;
 -                    return false;
 +                return fragments.every(item => text.toLowerCase().includes(item));
 -                }
 -                return fragments.every(item => text.includes(item));
+            }
             const fragments = makeQueryFragments(searchQuery);
             // At least one chat message must match *all* the fragments.
             // Currently, this doesn't match if the fragment matches are distributed across several chat messages.
             return chatContent && Object.values(chatContent).some(message => matchFragments(fragments, message?.mes?.toLowerCase()));
         });
         console.debug(filteredData);

src/endpoints/chats.js+59 -16 Viewed

@@ -92,8 +92,7 @@ function importOobaChat(userName, characterName, jsonData) {
92	92	}
93	93	}
94	94
95	95	const chatContent =return chat.map(obj => JSON.stringify(obj)).join('\n');
96		- return chatContent;
97	96	}
98	97
99	98	/**

@@ -121,8 +120,7 @@ function importAgnaiChat(userName, characterName, jsonData) {
121	120	});
122	121	}
123	122
124	123	const chatContent =return chat.map(obj => JSON.stringify(obj)).join('\n');
125		- return chatContent;
126	124	}
127	125
128	126	/**

     return newChats;
+}
 +/**
 + * Flattens `msg` and `swipes` data from Chub Chat format.
 + * Only changes enough to make it compatible with the standard chat serialization format.
 + * @param {string} userName User name
 + * @param {string} characterName Character name
 + * @param {string[]} lines serialised JSONL data
 + * @returns {string} Converted data
 + */
 +function flattenChubChat(userName, characterName, lines) {
 +    function flattenSwipe(swipe) {
 +        return swipe.message ? swipe.message : swipe;
 +    }
++
 +    function convert(line) {
 +        const lineData = tryParse(line);
 +        if (!lineData) return line;
++
 +        if (lineData.mes && lineData.mes.message) {
 +            lineData.mes = lineData?.mes.message;
 +        }
++
 +        if (lineData?.swipes && Array.isArray(lineData.swipes)) {
 +            lineData.swipes = lineData.swipes.map(swipe => flattenSwipe(swipe));
 +        }
++
 +        return JSON.stringify(lineData);
 +    }
++
 +    return (lines ?? []).map(convert).join('\n');
 +}
++
 const router = express.Router();
 router.post('/save', jsonParser, function (request, response) {

+    }
     try {
         // Short path for JSONL files
         if (request.body.format === 'jsonl') {
             try {
                 const rawFile = fs.readFileSync(filename, 'utf8');
                 const successMessage = {

                 console.log(`Chat exported as ${exportfilename}`);
                 return response.status(200).json(successMessage);
 -            }
 +            } catch (err) {
 -            catch (err) {
                 console.error(err);
                 const errorMessage = {
                     message: `Could not read JSONL file to export. Source chat file: ${filename}.`,

             console.log(`Chat exported as ${exportfilename}`);
             return response.status(200).json(successMessage);
         });
 -    }
 +    } catch (err) {
 -    catch (err) {
         console.log('chat export failed.');
         console.log(err);
         return response.sendStatus(400);

+        }
         if (format === 'jsonl') {
             constlet linelines = data.split('\n')[0];
 +            const header = lines[0];
++
 +            const jsonData = JSON.parse(header);
++
 +            if (!(jsonData.user_name !== undefined || jsonData.name !== undefined)) {
 +                console.log('Incorrect chat format .jsonl');
 +                return response.send({ error: true });
 +            }
 -            const jsonData = JSON.parse(line);
 +            // Do a tiny bit of work to import Chub Chat data
 +            // Processing the entire file is so fast that it's not worth checking if it's a Chub chat first
 +            let flattenedChat;
 +            try {
 +                // flattening is unlikely to break, but it's not worth failing to
 +                // import normal chats in an attempt to import a Chub chat
 +                flattenedChat = flattenChubChat(userName, characterName, lines);
 +            } catch (error) {
 +                console.warn('Failed to flatten Chub Chat data: ', error);
 +            }
 -            if (jsonData.user_name !== undefined || jsonData.name !== undefined) {
             const fileName = `${characterName} - ${humanizedISO8601DateTime()} imported.jsonl`;
             const filePath = path.join(request.user.directories.chats, avatarUrl, fileName);
 +            if (flattenedChat !== data) {
 +                writeFileAtomicSync(filePath, flattenedChat, 'utf8');
 +            } else {
                 fs.copyFileSync(pathToUpload, filePath);
 +            }
             fs.unlinkSync(pathToUpload);
             response.send({ res: true });
 -            } else {
 -                console.log('Incorrect chat format .jsonl');
 -                return response.send({ error: true });
 -            }
+        }
     } catch (error) {
         console.error(error);