data-maid.js · permissionBRICK/SillyTavern

Blame Raw
· · · 816 lines (33.8 KB)
0 contributors
1 import crypto from 'node:crypto';
2 import fs from 'node:fs';
3 import path from 'node:path';
4 import express from 'express';
5 import mime from 'mime-types';
6 import { getSettingsBackupFilePrefix } from './settings.js';
7 import { CHAT_BACKUPS_PREFIX } from './chats.js';
8 import { isPathUnderParent, tryParse } from '../util.js';
9 import { SETTINGS_FILE } from '../constants.js';
10 
11 const sha256 = str => crypto.createHash('sha256').update(str).digest('hex');
12 
13 /**
14  * @typedef {object} DataMaidRawReport
15  * @property {string[]} images - List of loose user images
16  * @property {string[]} files - List of loose user files
17  * @property {string[]} chats - List of loose character chats
18  * @property {string[]} groupChats - List of loose group chats
19  * @property {string[]} avatarThumbnails - List of loose avatar thumbnails
20  * @property {string[]} backgroundThumbnails - List of loose background thumbnails
21  * @property {string[]} personaThumbnails - List of loose persona thumbnails
22  * @property {string[]} chatBackups - List of chat backups
23  * @property {string[]} settingsBackups - List of settings backups
24  */
25 
26 /**
27  * @typedef {object} DataMaidSanitizedRecord - The entry excluding the sensitive paths.
28  * @property {string} name - The name of the file.
29  * @property {string} hash - The SHA-256 hash of the file path.
30  * @property {string} [parent] - The name of the parent directory, if applicable.
31  * @property {number} [size] - The size of the file in bytes, if available.
32  * @property {number} [mtime] - The last modification time of the file, if available.
33  */
34 
35 /**
36  * @typedef {object} DataMaidSanitizedReport - The report containing loose user data.
37  * @property {DataMaidSanitizedRecord[]} images - List of sanitized loose user images
38  * @property {DataMaidSanitizedRecord[]} files - List of sanitized loose user files
39  * @property {DataMaidSanitizedRecord[]} chats - List of sanitized loose character chats
40  * @property {DataMaidSanitizedRecord[]} groupChats - List of sanitized loose group chats
41  * @property {DataMaidSanitizedRecord[]} avatarThumbnails - List of sanitized loose avatar thumbnails
42  * @property {DataMaidSanitizedRecord[]} backgroundThumbnails - List of sanitized loose background thumbnails
43  * @property {DataMaidSanitizedRecord[]} personaThumbnails - List of sanitized loose persona thumbnails
44  * @property {DataMaidSanitizedRecord[]} chatBackups - List of sanitized chat backups
45  * @property {DataMaidSanitizedRecord[]} settingsBackups - List of sanitized settings backups
46  */
47 
48 /**
49  * @typedef {object} DataMaidMessage - The chat message object.
50  * @property {DataMaidMessageExtra} [extra] - The extra data object.
51  * @property {DataMaidChatMetadata} [chat_metadata] - The chat metadata object.
52  */
53 
54 /**
55  * @typedef {object} DataMaidFile - The file object.
56  * @property {string} url - The file URL
57  */
58 
59 /**
60  * @typedef {object} DataMaidMedia - The media object.
61  * @property {string} url - The media URL
62  */
63 
64 /**
65  * @typedef {object} DataMaidChatMetadata - The chat metadata object.
66  * @property {DataMaidFile[]} [attachments] - The array of attachments, if any.
67  * @property {string[]} [chat_backgrounds] - The array of chat background image links, if any.
68  */
69 
70 /**
71  * @typedef {object} DataMaidMessageExtra - The extra data object.
72  * @property {string} [image] - The link to the image, if any - DEPRECATED, use `media` instead.
73  * @property {string} [video] - The link to the video, if any - DEPRECATED, use `media` instead.
74  * @property {string[]} [image_swipes] - The links to the image swipes, if any - DEPRECATED, use `media` instead.
75  * @property {DataMaidMedia[]} [media] - The links to the media, if any.
76  * @property {DataMaidFile} [file] - The file object, if any - DEPRECATED, use `files` instead.
77  * @property {DataMaidFile[]} [files] - The array of file objects, if any.
78  */
79 
80 /**
81  * @typedef {object} DataMaidTokenEntry
82  * @property {string} handle - The user's handle or identifier.
83  * @property {{path: string, hash: string}[]} paths - The list of file paths and their hashes that can be cleaned up.
84  */
85 
86 /**
87  * Service for detecting and managing loose user data files.
88  * Helps identify orphaned files that are no longer referenced by the application.
89  */
90 export class DataMaidService {
91     /**
92      * @type {Map<string, DataMaidTokenEntry>} Map clean-up tokens to user IDs
93      */
94     static TOKENS = new Map();
95 
96     /**
97      * Creates a new DataMaidService instance for a specific user.
98      * @param {string} handle - The user's handle.
99      * @param {import('../users.js').UserDirectoryList} directories - List of user directories to scan for loose data.
100      */
101     constructor(handle, directories) {
102         this.handle = handle;
103         this.directories = directories;
104     }
105 
106     /**
107      * Generates a report of loose user data.
108      * @returns {Promise<DataMaidRawReport>} A report containing lists of loose user data.
109      */
110     async generateReport() {
111         /** @type {DataMaidRawReport} */
112         const report = {
113             images: await this.#collectImages(),
114             files: await this.#collectFiles(),
115             chats: await this.#collectChats(),
116             groupChats: await this.#collectGroupChats(),
117             avatarThumbnails: await this.#collectAvatarThumbnails(),
118             backgroundThumbnails: await this.#collectBackgroundThumbnails(),
119             personaThumbnails: await this.#collectPersonaThumbnails(),
120             chatBackups: await this.#collectChatBackups(),
121             settingsBackups: await this.#collectSettingsBackups(),
122         };
123 
124         return report;
125     }
126 
127 
128     /**
129      * Sanitizes a record by hashing the file name and removing sensitive information.
130      * Additionally, adds metadata like size and modification time.
131      * @param {string} name The file or directory name to sanitize.
132      * @param {boolean} withParent If the model should include the parent directory name.
133      * @returns {Promise<DataMaidSanitizedRecord>} A sanitized record with the file name, hash, parent directory name, size, and modification time.
134      */
135     async #sanitizeRecord(name, withParent) {
136         const stat = fs.existsSync(name) ? await fs.promises.stat(name) : null;
137         return {
138             name: path.basename(name),
139             hash: sha256(name),
140             parent: withParent ? path.basename(path.dirname(name)) : void 0,
141             size: stat?.size,
142             mtime: stat?.mtimeMs,
143         };
144     }
145 
146     /**
147      * Sanitizes the report by hashing the file paths and removing sensitive information.
148      * @param {DataMaidRawReport} report - The raw report containing loose user data.
149      * @returns {Promise<DataMaidSanitizedReport>} A sanitized report with sensitive paths removed.
150      */
151     async sanitizeReport(report) {
152         const sanitizedReport = {
153             images: await Promise.all(report.images.map(i => this.#sanitizeRecord(i, true))),
154             files: await Promise.all(report.files.map(i => this.#sanitizeRecord(i, false))),
155             chats: await Promise.all(report.chats.map(i => this.#sanitizeRecord(i, true))),
156             groupChats: await Promise.all(report.groupChats.map(i => this.#sanitizeRecord(i, false))),
157             avatarThumbnails: await Promise.all(report.avatarThumbnails.map(i => this.#sanitizeRecord(i, false))),
158             backgroundThumbnails: await Promise.all(report.backgroundThumbnails.map(i => this.#sanitizeRecord(i, false))),
159             personaThumbnails: await Promise.all(report.personaThumbnails.map(i => this.#sanitizeRecord(i, false))),
160             chatBackups: await Promise.all(report.chatBackups.map(i => this.#sanitizeRecord(i, false))),
161             settingsBackups: await Promise.all(report.settingsBackups.map(i => this.#sanitizeRecord(i, false))),
162         };
163 
164         return sanitizedReport;
165     }
166 
167     /**
168      * Collects loose user images from the provided directories.
169      * Images are considered loose if they exist in the user images directory
170      * but are not referenced in any chat messages.
171      * @returns {Promise<string[]>} List of paths to loose user images
172      */
173     async #collectImages() {
174         const result = [];
175 
176         try {
177             const messages = await this.#parseAllChats(x => !!x?.extra?.image || !!x?.extra?.video || Array.isArray(x?.extra?.image_swipes) || Array.isArray(x?.extra?.media));
178             const knownImages = new Set();
179             for (const message of messages) {
180                 if (message?.extra?.image) {
181                     knownImages.add(message.extra.image);
182                 }
183                 if (message?.extra?.video) {
184                     knownImages.add(message.extra.video);
185                 }
186                 if (Array.isArray(message?.extra?.image_swipes)) {
187                     for (const swipe of message.extra.image_swipes) {
188                         knownImages.add(swipe);
189                     }
190                 }
191                 if (Array.isArray(message?.extra?.media)) {
192                     for (const media of message.extra.media) {
193                         if (media?.url) {
194                             knownImages.add(media.url);
195                         }
196                     }
197                 }
198             }
199             const metadata = await this.#parseAllMetadata(x => Array.isArray(x?.chat_backgrounds) && x.chat_backgrounds.length > 0);
200             for (const meta of metadata) {
201                 if (Array.isArray(meta?.chat_backgrounds)) {
202                     for (const background of meta.chat_backgrounds) {
203                         if (background) {
204                             knownImages.add(background);
205                         }
206                     }
207                 }
208             }
209             const knownImageFullPaths = new Set();
210             knownImages.forEach(image => {
211                 if (image.startsWith('http') || image.startsWith('data:')) {
212                     return; // Skip URLs and data URIs
213                 }
214                 knownImageFullPaths.add(path.normalize(path.join(this.directories.root, image)));
215             });
216             const images = await fs.promises.readdir(this.directories.userImages, { withFileTypes: true });
217             for (const dirent of images) {
218                 const direntPath = path.join(dirent.parentPath, dirent.name);
219                 if (dirent.isFile() && !knownImageFullPaths.has(direntPath)) {
220                     result.push(direntPath);
221                 }
222                 if (dirent.isDirectory()) {
223                     const subdirFiles = await fs.promises.readdir(direntPath, { withFileTypes: true });
224                     for (const file of subdirFiles) {
225                         const subdirFilePath = path.join(direntPath, file.name);
226                         if (file.isFile() && !knownImageFullPaths.has(subdirFilePath)) {
227                             result.push(subdirFilePath);
228                         }
229                     }
230                 }
231             }
232         } catch (error) {
233             console.error('[Data Maid] Error collecting user images:', error);
234         }
235 
236         return result;
237     }
238 
239     /**
240      * Collects loose user files from the provided directories.
241      * Files are considered loose if they exist in the files directory
242      * but are not referenced in chat messages, metadata, or settings.
243      * @returns {Promise<string[]>} List of paths to loose user files
244      */
245     async #collectFiles() {
246         const result = [];
247 
248         try {
249             const messages = await this.#parseAllChats(x => !!x?.extra?.file?.url || (Array.isArray(x?.extra?.files) && x.extra.files.length > 0));
250             const knownFiles = new Set();
251             for (const message of messages) {
252                 if (message?.extra?.file?.url) {
253                     knownFiles.add(message.extra.file.url);
254                 }
255                 if (Array.isArray(message?.extra?.files)) {
256                     for (const file of message.extra.files) {
257                         if (file?.url) {
258                             knownFiles.add(file.url);
259                         }
260                     }
261                 }
262             }
263             const metadata = await this.#parseAllMetadata(x => Array.isArray(x?.attachments) && x.attachments.length > 0);
264             for (const meta of metadata) {
265                 if (Array.isArray(meta?.attachments)) {
266                     for (const attachment of meta.attachments) {
267                         if (attachment?.url) {
268                             knownFiles.add(attachment.url);
269                         }
270                     }
271                 }
272             }
273             const pathToSettings = path.join(this.directories.root, SETTINGS_FILE);
274             if (fs.existsSync(pathToSettings)) {
275                 try {
276                     const settingsContent = await fs.promises.readFile(pathToSettings, 'utf-8');
277                     const settings = tryParse(settingsContent);
278                     if (Array.isArray(settings?.extension_settings?.attachments)) {
279                         for (const file of settings.extension_settings.attachments) {
280                             if (file?.url) {
281                                 knownFiles.add(file.url);
282                             }
283                         }
284                     }
285                     if (typeof settings?.extension_settings?.character_attachments === 'object') {
286                         for (const files of Object.values(settings.extension_settings.character_attachments)) {
287                             if (!Array.isArray(files)) {
288                                 continue;
289                             }
290                             for (const file of files) {
291                                 if (file?.url) {
292                                     knownFiles.add(file.url);
293                                 }
294                             }
295                         }
296                     }
297                 } catch (error) {
298                     console.error('[Data Maid] Error reading settings file:', error);
299                 }
300             }
301             const knownFileFullPaths = new Set();
302             knownFiles.forEach(file => {
303                 knownFileFullPaths.add(path.normalize(path.join(this.directories.root, file)));
304             });
305             const files = await fs.promises.readdir(this.directories.files, { withFileTypes: true });
306             for (const file of files) {
307                 const filePath = path.join(this.directories.files, file.name);
308                 if (file.isFile() && !knownFileFullPaths.has(filePath)) {
309                     result.push(filePath);
310                 }
311             }
312         } catch (error) {
313             console.error('[Data Maid] Error collecting user files:', error);
314         }
315 
316         return result;
317     }
318 
319     /**
320      * Collects loose character chats from the provided directories.
321      * Chat folders are considered loose if they don't have corresponding character files.
322      * @returns {Promise<string[]>} List of paths to loose character chats
323      */
324     async #collectChats() {
325         const result = [];
326 
327         try {
328             const knownChatFolders = new Set();
329             const characters = await fs.promises.readdir(this.directories.characters, { withFileTypes: true });
330             for (const file of characters) {
331                 if (file.isFile() && path.parse(file.name).ext === '.png') {
332                     knownChatFolders.add(file.name.replace('.png', ''));
333                 }
334             }
335             const chatFolders = await fs.promises.readdir(this.directories.chats, { withFileTypes: true });
336             for (const folder of chatFolders) {
337                 if (folder.isDirectory() && !knownChatFolders.has(folder.name)) {
338                     const chatFiles = await fs.promises.readdir(path.join(this.directories.chats, folder.name), { withFileTypes: true });
339                     for (const file of chatFiles) {
340                         if (file.isFile() && path.parse(file.name).ext === '.jsonl') {
341                             result.push(path.join(this.directories.chats, folder.name, file.name));
342                         }
343                     }
344                 }
345             }
346         } catch (error) {
347             console.error('[Data Maid] Error collecting character chats:', error);
348         }
349 
350         return result;
351     }
352 
353     /**
354      * Collects loose group chats from the provided directories.
355      * Group chat files are considered loose if they're not referenced by any group definition.
356      * @returns {Promise<string[]>} List of paths to loose group chats
357      */
358     async #collectGroupChats() {
359         const result = [];
360 
361         try {
362             const groups = await fs.promises.readdir(this.directories.groups, { withFileTypes: true });
363             const knownGroupChats = new Set();
364             for (const file of groups) {
365                 if (file.isFile() && path.parse(file.name).ext === '.json') {
366                     try {
367                         const pathToFile = path.join(this.directories.groups, file.name);
368                         const fileContent = await fs.promises.readFile(pathToFile, 'utf-8');
369                         const groupData = tryParse(fileContent);
370                         if (groupData?.chat_id) {
371                             knownGroupChats.add(groupData.chat_id);
372                         }
373                         if (Array.isArray(groupData?.chats)) {
374                             for (const chat of groupData.chats) {
375                                 knownGroupChats.add(chat);
376                             }
377                         }
378                     } catch (error) {
379                         console.error(`[Data Maid] Error parsing group chat file ${file.name}:`, error);
380                     }
381                 }
382             }
383             const groupChats = await fs.promises.readdir(this.directories.groupChats, { withFileTypes: true });
384             for (const file of groupChats) {
385                 if (file.isFile() && path.parse(file.name).ext === '.jsonl') {
386                     if (!knownGroupChats.has(path.parse(file.name).name)) {
387                         result.push(path.join(this.directories.groupChats, file.name));
388                     }
389                 }
390             }
391         } catch (error) {
392             console.error('[Data Maid] Error collecting group chats:', error);
393         }
394 
395         return result;
396     }
397 
398     /**
399      * Collects loose avatar thumbnails from the provided directories.
400      * @returns {Promise<string[]>} List of paths to loose avatar thumbnails
401      */
402     async #collectAvatarThumbnails() {
403         const result = [];
404 
405         try {
406             const knownAvatars = new Set();
407             const avatars = await fs.promises.readdir(this.directories.characters, { withFileTypes: true });
408             for (const file of avatars) {
409                 if (file.isFile()) {
410                     knownAvatars.add(file.name);
411                 }
412             }
413             const avatarThumbnails = await fs.promises.readdir(this.directories.thumbnailsAvatar, { withFileTypes: true });
414             for (const file of avatarThumbnails) {
415                 if (file.isFile() && !knownAvatars.has(file.name)) {
416                     result.push(path.join(this.directories.thumbnailsAvatar, file.name));
417                 }
418             }
419         } catch (error) {
420             console.error('[Data Maid] Error collecting avatar thumbnails:', error);
421         }
422 
423         return result;
424     }
425 
426     /**
427      * Collects loose background thumbnails from the provided directories.
428      * @returns {Promise<string[]>} List of paths to loose background thumbnails
429      */
430     async #collectBackgroundThumbnails() {
431         const result = [];
432 
433         try {
434             const knownBackgrounds = new Set();
435             const backgrounds = await fs.promises.readdir(this.directories.backgrounds, { withFileTypes: true });
436             for (const file of backgrounds) {
437                 if (file.isFile()) {
438                     knownBackgrounds.add(file.name);
439                 }
440             }
441             const backgroundThumbnails = await fs.promises.readdir(this.directories.thumbnailsBg, { withFileTypes: true });
442             for (const file of backgroundThumbnails) {
443                 if (file.isFile() && !knownBackgrounds.has(file.name)) {
444                     result.push(path.join(this.directories.thumbnailsBg, file.name));
445                 }
446             }
447         } catch (error) {
448             console.error('[Data Maid] Error collecting background thumbnails:', error);
449         }
450 
451         return result;
452     }
453 
454     /**
455      * Collects loose persona thumbnails from the provided directories.
456      * @returns {Promise<string[]>} List of paths to loose persona thumbnails
457      */
458     async #collectPersonaThumbnails() {
459         const result = [];
460 
461         try {
462             const knownPersonas = new Set();
463             const personas = await fs.promises.readdir(this.directories.avatars, { withFileTypes: true });
464             for (const file of personas) {
465                 if (file.isFile()) {
466                     knownPersonas.add(file.name);
467                 }
468             }
469             const personaThumbnails = await fs.promises.readdir(this.directories.thumbnailsPersona, { withFileTypes: true });
470             for (const file of personaThumbnails) {
471                 if (file.isFile() && !knownPersonas.has(file.name)) {
472                     result.push(path.join(this.directories.thumbnailsPersona, file.name));
473                 }
474             }
475         } catch (error) {
476             console.error('[Data Maid] Error collecting persona thumbnails:', error);
477         }
478 
479         return result;
480     }
481 
482     /**
483      * Collects chat backups from the provided directories.
484      * @returns {Promise<string[]>} List of paths to chat backups
485      */
486     async #collectChatBackups() {
487         const result = [];
488 
489         try {
490             const prefix = CHAT_BACKUPS_PREFIX;
491             const backups = await fs.promises.readdir(this.directories.backups, { withFileTypes: true });
492             for (const file of backups) {
493                 if (file.isFile() && file.name.startsWith(prefix)) {
494                     result.push(path.join(this.directories.backups, file.name));
495                 }
496             }
497         } catch (error) {
498             console.error('[Data Maid] Error collecting chat backups:', error);
499         }
500 
501         return result;
502     }
503 
504     /**
505      * Collects settings backups from the provided directories.
506      * @returns {Promise<string[]>} List of paths to settings backups
507      */
508     async #collectSettingsBackups() {
509         const result = [];
510 
511         try {
512             const prefix = getSettingsBackupFilePrefix(this.handle);
513             const backups = await fs.promises.readdir(this.directories.backups, { withFileTypes: true });
514             for (const file of backups) {
515                 if (file.isFile() && file.name.startsWith(prefix)) {
516                     result.push(path.join(this.directories.backups, file.name));
517                 }
518             }
519         } catch (error) {
520             console.error('[Data Maid] Error collecting settings backups:', error);
521         }
522 
523         return result;
524     }
525 
526     /**
527      * Parses all chat files and returns an array of chat messages.
528      * Searches both individual character chats and group chats.
529      * @param {function(DataMaidMessage): boolean} filterFn - Filter function to apply to each message.
530      * @returns {Promise<DataMaidMessage[]>} Array of chat messages
531      */
532     async #parseAllChats(filterFn) {
533         try {
534             const allChats = [];
535 
536             const groupChats = await fs.promises.readdir(this.directories.groupChats, { withFileTypes: true });
537             for (const file of groupChats) {
538                 if (file.isFile() && path.parse(file.name).ext === '.jsonl') {
539                     const chatMessages = await this.#parseChatFile(path.join(this.directories.groupChats, file.name));
540                     allChats.push(...chatMessages.filter(filterFn));
541                 }
542             }
543 
544             const chatDirectories = await fs.promises.readdir(this.directories.chats, { withFileTypes: true });
545             for (const directory of chatDirectories) {
546                 if (directory.isDirectory()) {
547                     const chatFiles = await fs.promises.readdir(path.join(this.directories.chats, directory.name), { withFileTypes: true });
548                     for (const file of chatFiles) {
549                         if (file.isFile() && path.parse(file.name).ext === '.jsonl') {
550                             const chatMessages = await this.#parseChatFile(path.join(this.directories.chats, directory.name, file.name));
551                             allChats.push(...chatMessages.filter(filterFn));
552                         }
553                     }
554                 }
555             }
556 
557             return allChats;
558         } catch (error) {
559             console.error('[Data Maid] Error parsing chats:', error);
560             return [];
561         }
562     }
563 
564     /**
565      * Parses all metadata from chat files and group definitions.
566      * Extracts metadata from both active and historical chat data.
567      * @param {function(DataMaidChatMetadata): boolean} filterFn - Filter function to apply to each metadata entry.
568      * @returns {Promise<DataMaidChatMetadata[]>} Parsed chat metadata as an array.
569      */
570     async #parseAllMetadata(filterFn) {
571         try {
572             const allMetadata = [];
573 
574             const groups = await fs.promises.readdir(this.directories.groups, { withFileTypes: true });
575             for (const file of groups) {
576                 if (file.isFile() && path.parse(file.name).ext === '.json') {
577                     try {
578                         const pathToFile = path.join(this.directories.groups, file.name);
579                         const fileContent = await fs.promises.readFile(pathToFile, 'utf-8');
580                         const groupData = tryParse(fileContent);
581                         if (groupData?.chat_metadata && filterFn(groupData.chat_metadata)) {
582                             console.warn('Found group chat metadata in group definition - this is deprecated behavior.');
583                             allMetadata.push(groupData.chat_metadata);
584                         }
585                         if (groupData?.past_metadata) {
586                             console.warn('Found group past chat metadata in group definition - this is deprecated behavior.');
587                             allMetadata.push(...Object.values(groupData.past_metadata).filter(filterFn));
588                         }
589                     } catch (error) {
590                         console.error(`[Data Maid] Error parsing group chat file ${file.name}:`, error);
591                     }
592                 }
593             }
594 
595             const groupChats = await fs.promises.readdir(this.directories.groupChats, { withFileTypes: true });
596             for (const file of groupChats) {
597                 if (file.isFile() && path.parse(file.name).ext === '.jsonl') {
598                     const chatMessages = await this.#parseChatFile(path.join(this.directories.groupChats, file.name));
599                     const chatMetadata = chatMessages?.[0]?.chat_metadata;
600                     if (chatMetadata && filterFn(chatMetadata)) {
601                         allMetadata.push(chatMetadata);
602                     }
603                 }
604             }
605 
606             const chatDirectories = await fs.promises.readdir(this.directories.chats, { withFileTypes: true });
607             for (const directory of chatDirectories) {
608                 if (directory.isDirectory()) {
609                     const chatFiles = await fs.promises.readdir(path.join(this.directories.chats, directory.name), { withFileTypes: true });
610                     for (const file of chatFiles) {
611                         if (file.isFile() && path.parse(file.name).ext === '.jsonl') {
612                             const chatMessages = await this.#parseChatFile(path.join(this.directories.chats, directory.name, file.name));
613                             const chatMetadata = chatMessages?.[0]?.chat_metadata;
614                             if (chatMetadata && filterFn(chatMetadata)) {
615                                 allMetadata.push(chatMetadata);
616                             }
617                         }
618                     }
619                 }
620             }
621 
622             return allMetadata;
623         } catch (error) {
624             console.error('[Data Maid] Error parsing chats:', error);
625             return [];
626         }
627     }
628 
629     /**
630      * Parses a single chat file and returns an array of chat messages.
631      * Each line in the JSONL file represents one message.
632      * @param {string} filePath Path to the chat file to parse.
633      * @returns {Promise<DataMaidMessage[]>} Parsed chat messages as an array.
634      */
635     async #parseChatFile(filePath) {
636         try {
637             const content = await fs.promises.readFile(filePath, 'utf-8');
638             const chatData = content.split('\n').map(tryParse).filter(Boolean);
639             return chatData;
640         } catch (error) {
641             console.error(`[Data Maid] Error reading chat file ${filePath}:`, error);
642             return [];
643         }
644     }
645 
646     /**
647      * Generates a unique token for the user to clean up their data.
648      * Replaces any existing token for the same user.
649      * @param {string} handle - The user's handle or identifier.
650      * @param {DataMaidRawReport} report - The report containing loose user data.
651      * @returns {string} A unique token.
652      */
653     static generateToken(handle, report) {
654         // Remove any existing token for this user
655         for (const [token, entry] of this.TOKENS.entries()) {
656             if (entry.handle === handle) {
657                 this.TOKENS.delete(token);
658             }
659         }
660 
661         const token = crypto.randomBytes(32).toString('hex');
662         const tokenEntry = {
663             handle,
664             paths: Object.values(report).filter(v => Array.isArray(v)).flat().map(x => ({ path: x, hash: sha256(x) })),
665         };
666         this.TOKENS.set(token, tokenEntry);
667         return token;
668     }
669 }
670 
671 export const router = express.Router();
672 
673 router.post('/report', async (req, res) => {
674     try {
675         if (!req.user || !req.user.directories) {
676             return res.sendStatus(403);
677         }
678 
679         const dataMaid = new DataMaidService(req.user.profile.handle, req.user.directories);
680         const rawReport = await dataMaid.generateReport();
681 
682         const report = await dataMaid.sanitizeReport(rawReport);
683         const token = DataMaidService.generateToken(req.user.profile.handle, rawReport);
684 
685         return res.json({ report, token });
686     } catch (error) {
687         console.error('[Data Maid] Error generating data maid report:', error);
688         return res.sendStatus(500);
689     }
690 });
691 
692 router.post('/finalize', async (req, res) => {
693     try {
694         if (!req.user || !req.user.directories) {
695             return res.sendStatus(403);
696         }
697 
698         if (!req.body.token) {
699             return res.sendStatus(400);
700         }
701 
702         const token = req.body.token.toString();
703         if (!DataMaidService.TOKENS.has(token)) {
704             return res.sendStatus(403);
705         }
706 
707         const tokenEntry = DataMaidService.TOKENS.get(token);
708         if (!tokenEntry || tokenEntry.handle !== req.user.profile.handle) {
709             return res.sendStatus(403);
710         }
711 
712         // Remove the token after finalization
713         DataMaidService.TOKENS.delete(token);
714         return res.sendStatus(204);
715     } catch (error) {
716         console.error('[Data Maid] Error finalizing the token:', error);
717         return res.sendStatus(500);
718     }
719 });
720 
721 router.get('/view', async (req, res) => {
722     try {
723         if (!req.user || !req.user.directories) {
724             return res.sendStatus(403);
725         }
726 
727         if (!req.query.token || !req.query.hash) {
728             return res.sendStatus(400);
729         }
730 
731         const token = req.query.token.toString();
732         const hash = req.query.hash.toString();
733 
734         if (!DataMaidService.TOKENS.has(token)) {
735             return res.sendStatus(403);
736         }
737 
738         const tokenEntry = DataMaidService.TOKENS.get(token);
739         if (!tokenEntry || tokenEntry.handle !== req.user.profile.handle) {
740             return res.sendStatus(403);
741         }
742 
743         const fileEntry = tokenEntry.paths.find(entry => entry.hash === hash);
744         if (!fileEntry) {
745             return res.sendStatus(404);
746         }
747 
748         if (!isPathUnderParent(req.user.directories.root, fileEntry.path)) {
749             console.warn('[Data Maid] Attempted access to a file outside of the user directory:', fileEntry.path);
750             return res.sendStatus(403);
751         }
752 
753         const pathToFile = fileEntry.path;
754         const fileExists = fs.existsSync(pathToFile);
755 
756         if (!fileExists) {
757             return res.sendStatus(404);
758         }
759 
760         const fileBuffer = await fs.promises.readFile(pathToFile);
761         const mimeType = mime.lookup(pathToFile) || 'text/plain';
762         res.setHeader('Content-Type', mimeType);
763         return res.send(fileBuffer);
764     } catch (error) {
765         console.error('[Data Maid] Error viewing file:', error);
766         return res.sendStatus(500);
767     }
768 });
769 
770 router.post('/delete', async (req, res) => {
771     try {
772         if (!req.user || !req.user.directories) {
773             return res.sendStatus(403);
774         }
775 
776         const { token, hashes } = req.body;
777         if (!token || !Array.isArray(hashes) || hashes.length === 0) {
778             return res.sendStatus(400);
779         }
780 
781         if (!DataMaidService.TOKENS.has(token)) {
782             return res.sendStatus(403);
783         }
784 
785         const tokenEntry = DataMaidService.TOKENS.get(token);
786         if (!tokenEntry || tokenEntry.handle !== req.user.profile.handle) {
787             return res.sendStatus(403);
788         }
789 
790         for (const hash of hashes) {
791             const fileEntry = tokenEntry.paths.find(entry => entry.hash === hash);
792             if (!fileEntry) {
793                 continue;
794             }
795 
796             if (!isPathUnderParent(req.user.directories.root, fileEntry.path)) {
797                 console.warn('[Data Maid] Attempted deletion of a file outside of the user directory:', fileEntry.path);
798                 continue;
799             }
800 
801             const pathToFile = fileEntry.path;
802             const fileExists = fs.existsSync(pathToFile);
803 
804             if (!fileExists) {
805                 continue;
806             }
807 
808             await fs.promises.unlink(pathToFile);
809         }
810 
811         return res.sendStatus(204);
812     } catch (error) {
813         console.error('[Data Maid] Error deleting files:', error);
814         return res.sendStatus(500);
815     }
816 });