Merge pull request #2875 from ceruleandeep/feature/handleChubChatData Fix crasher in script.js:displayChats, add minimal support for importing chat histories in Chub format

ed47d51c93e6a123d8ffd8ae85b4a0012bfba3e7

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

Parents: 90169858274431ed4cdf8bc0cb9ab9c2fba7df6c a2d9526cdfa73d2e4eac0fe662479fe77c5e0e8f

public/script.js+3 -5 Viewed

+            }
             // Check whether `text` {string} includes all of the `fragments` {string[]}.
             function matchFragments(fragments, text) {
-                if (!text) {
+                if (!text || !text.toLowerCase) return false;
-                    return false;
+                return fragments.every(item => text.toLowerCase().includes(item));
-                }
-                return fragments.every(item => text.includes(item));
+            }
             const fragments = makeQueryFragments(searchQuery);
             // At least one chat message must match *all* the fragments.
             // Currently, this doesn't match if the fragment matches are distributed across several chat messages.
-            return chatContent && Object.values(chatContent).some(message => matchFragments(fragments, message?.mes?.toLowerCase()));
+            return chatContent && Object.values(chatContent).some(message => matchFragments(fragments, message?.mes));
         });
         console.debug(filteredData);

src/endpoints/chats.js+59 -16 Viewed

@@ -92,8 +92,7 @@ function importOobaChat(userName, characterName, jsonData) {
92	}	92	}
93	}	93	}
94		94
95	const chatContent = chat.map(obj => JSON.stringify(obj)).join('\n');	95	return chat.map(obj => JSON.stringify(obj)).join('\n');
96	return chatContent;
97	}	96	}
98		97
99	/**	98	/**

@@ -121,8 +120,7 @@ function importAgnaiChat(userName, characterName, jsonData) {
121	});	120	});
122	}	121	}
123		122
124	const chatContent = chat.map(obj => JSON.stringify(obj)).join('\n');	123	return chat.map(obj => JSON.stringify(obj)).join('\n');
125	return chatContent;
126	}	124	}
127		125
128	/**	126	/**

     return newChats;
+}
+/**
+ * Flattens `msg` and `swipes` data from Chub Chat format.
+ * Only changes enough to make it compatible with the standard chat serialization format.
+ * @param {string} userName User name
+ * @param {string} characterName Character name
+ * @param {string[]} lines serialised JSONL data
+ * @returns {string} Converted data
+ */
+function flattenChubChat(userName, characterName, lines) {
+    function flattenSwipe(swipe) {
+        return swipe.message ? swipe.message : swipe;
+    }
+    function convert(line) {
+        const lineData = tryParse(line);
+        if (!lineData) return line;
+        if (lineData.mes && lineData.mes.message) {
+            lineData.mes = lineData?.mes.message;
+        }
+        if (lineData?.swipes && Array.isArray(lineData.swipes)) {
+            lineData.swipes = lineData.swipes.map(swipe => flattenSwipe(swipe));
+        }
+        return JSON.stringify(lineData);
+    }
+    return (lines ?? []).map(convert).join('\n');
+}
 const router = express.Router();
 router.post('/save', jsonParser, function (request, response) {

+    }
     try {
         // Short path for JSONL files
-        if (request.body.format == 'jsonl') {
+        if (request.body.format === 'jsonl') {
             try {
                 const rawFile = fs.readFileSync(filename, 'utf8');
                 const successMessage = {

                 console.log(`Chat exported as ${exportfilename}`);
                 return response.status(200).json(successMessage);
-            }
+            } catch (err) {
-            catch (err) {
                 console.error(err);
                 const errorMessage = {
                     message: `Could not read JSONL file to export. Source chat file: ${filename}.`,

             console.log(`Chat exported as ${exportfilename}`);
             return response.status(200).json(successMessage);
         });
-    }
+    } catch (err) {
-    catch (err) {
         console.log('chat export failed.');
         console.log(err);
         return response.sendStatus(400);

+        }
         if (format === 'jsonl') {
-            const line = data.split('\n')[0];
+            let lines = data.split('\n');
+            const header = lines[0];
+            const jsonData = JSON.parse(header);
+            if (!(jsonData.user_name !== undefined || jsonData.name !== undefined)) {
+                console.log('Incorrect chat format .jsonl');
+                return response.send({ error: true });
+            }
-            const jsonData = JSON.parse(line);
+            // Do a tiny bit of work to import Chub Chat data
+            // Processing the entire file is so fast that it's not worth checking if it's a Chub chat first
+            let flattenedChat;
+            try {
+                // flattening is unlikely to break, but it's not worth failing to
+                // import normal chats in an attempt to import a Chub chat
+                flattenedChat = flattenChubChat(userName, characterName, lines);
+            } catch (error) {
+                console.warn('Failed to flatten Chub Chat data: ', error);
+            }
-            if (jsonData.user_name !== undefined || jsonData.name !== undefined) {
             const fileName = `${characterName} - ${humanizedISO8601DateTime()} imported.jsonl`;
             const filePath = path.join(request.user.directories.chats, avatarUrl, fileName);
+            if (flattenedChat !== data) {
+                writeFileAtomicSync(filePath, flattenedChat, 'utf8');
+            } else {
                 fs.copyFileSync(pathToUpload, filePath);
+            }
             fs.unlinkSync(pathToUpload);
             response.send({ res: true });
-            } else {
-                console.log('Incorrect chat format .jsonl');
-                return response.send({ error: true });
-            }
+        }
     } catch (error) {
         console.error(error);