Implement data clean-up dialog (#4072) * [wip] Add user data cleanup service * Add clean-up report viewer * Fix review comments * Add function comments * Implement item actions * Fix UI styles * Add placeholder for empty results, update category description view * Add displayEmptyPlaceholder method to show message when results list is empty * Adjust menu buttons row * Delete char-scoped data bank attachments on character deletion * Data Bank: Handle character attachments on rename * Remove line breaks in description strings * Drop the category when the last item is deleted * Skip invalid hashes instead of bailing

3ec9b1a09994e33d663e47e425339df39235c973

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: 7f47f84c9f7ec29ba245aa95d96c25c264eb80b9
15 files changed, +1475 -62Ignore whitespace
public/css/data-maid.css+149 -0
@@ -0,0 +1,149 @@
1+.dataMaidDialogContainer {
2+ height: 100%;
3+ overflow: hidden;
4+}
5+
6+.dataMaidDialog {
7+ display: flex;
8+ flex-direction: column;
9+ gap: 5px;
10+ height: 100%;
11+ overflow: hidden;
12+}
13+
14+.dataMaidDialogHeader {
15+ display: flex;
16+ gap: 10px;
17+ align-items: center;
18+ text-align: left;
19+}
20+
21+.dataMaidHeaderInfo {
22+ flex: 1;
23+ margin: 0;
24+ padding: 5px 10px;
25+}
26+
27+.dataMaidTextView {
28+ width: 100%;
29+ height: 100%;
30+ font-family: var(--monoFontFamily);
31+ resize: none;
32+ font-size: 0.95em;
33+}
34+
35+.dataMaidImageView {
36+ width: 100%;
37+ height: 100%;
38+ object-fit: contain;
39+}
40+
41+.dataMaidSpinner {
42+ display: flex;
43+ justify-content: center;
44+ align-items: center;
45+ height: 100%;
46+}
47+
48+.dataMaidPlaceholder {
49+ display: flex;
50+ justify-content: center;
51+ align-items: center;
52+ font-size: 1.05em;
53+}
54+
55+.dataMaidResultsList:empty {
56+ display: none;
57+}
58+
59+.dataMaidResultsList {
60+ text-align: left;
61+ display: flex;
62+ flex-direction: column;
63+ gap: 2px;
64+ overflow-y: auto;
65+ height: 100%;
66+ flex-grow: 1;
67+}
68+
69+.dataMaidCategory {
70+ border: 1px solid var(--SmartThemeBorderColor);
71+ border-radius: 10px;
72+ padding: 0 10px;
73+}
74+
75+.dataMaidCategoryHeader {
76+ width: 100%;
77+ display: flex;
78+ align-items: center;
79+ justify-content: space-between;
80+ margin-right: 5px;
81+ padding: 0 5px;
82+}
83+
84+.dataMaidCategoryDetails {
85+ display: flex;
86+ flex-direction: column;
87+ gap: 1px;
88+}
89+
90+.dataMaidCategoryName {
91+ flex: 3;
92+ font-weight: bold;
93+ font-size: 1.1em;
94+}
95+
96+.dataMaidCategoryInfo {
97+ flex: 1;
98+ display: flex;
99+ align-items: baseline;
100+ gap: 5px;
101+}
102+
103+.dataMaidCategoryContent {
104+ border: 1px solid var(--SmartThemeBorderColor);
105+ padding: 5px;
106+ border-radius: 10px;
107+ background-color: var(--black30a);
108+ margin: 10px 0;
109+}
110+
111+.dataMaidCategoryContent>.info-block {
112+ white-space: pre-wrap;
113+}
114+
115+.dataMaidItem {
116+ display: flex;
117+ flex-direction: column;
118+ padding: 5px;
119+ width: 100%;
120+ border-bottom: 1px solid var(--SmartThemeBorderColor);
121+}
122+
123+.dataMaidItem:last-child {
124+ border-bottom: none;
125+}
126+
127+.dataMaidItemHeader {
128+ display: flex;
129+ align-items: center;
130+ gap: 5px;
131+}
132+
133+.dataMaidItemName {
134+ display: flex;
135+ flex: 1;
136+ align-items: baseline;
137+ gap: 2px;
138+ word-break: break-all;
139+}
140+
141+.dataMaidItemActions {
142+ display: flex;
143+ align-items: center;
144+ gap: 5px;
145+}
146+
147+.dataMaidItemActions>button {
148+ font-size: 0.9em;
149+}
public/css/st-tailwind.css+4 -0
@@ -556,6 +556,10 @@ textarea:disabled {
556556 min-width: fit-content;
557557}
558558
559+.flexGap2 {
560+ gap: 2px;
561+}
562+
559563.flexGap5 {
560564 gap: 5px;
561565}
public/index.html+4 -1
@@ -4738,13 +4738,16 @@
47384738
47394739 <div name="MiscellaneousToggles">
47404740 <h4><span data-i18n="Miscellaneous">Miscellaneous</span></h4>
47414741 <div class="flex-container flexGap2">
47424742 <div id="reload_chat" class="menu_button whitespacenowrap" data-i18n="[title]Reload and redraw the currently open chat" title="Reload and redraw the currently open chat.">
47434743 <small data-i18n="Reload Chat">Reload Chat</small>
47444744 </div>
47454745 <div id="debug_menu" class="menu_button whitespacenowrap">
47464746 <small data-i18n="Debug Menu">Debug Menu</small>
47474747 </div>
4748+ <div id="data_maid_button" class="menu_button whitespacenowrap" title="Find and delete backups, unused chats, files, images, etc." data-i18n="[title]Find and delete backups, unused chats, files, images, etc.">
4749+ <small data-i18n="Clean-Up">Clean-Up</small>
4750+ </div>
47484751 </div>
47494752 <label class="checkbox_label flexWrap" for="smooth_streaming">
47504753 <input id="smooth_streaming" type="checkbox" />
public/script.js+2 -0
@@ -283,6 +283,7 @@ import { getContext } from './scripts/st-context.js';
283283import { extractReasoningFromData, initReasoning, parseReasoningInSwipes, PromptReasoning, ReasoningHandler, removeReasoningFromString, updateReasoningUI } from './scripts/reasoning.js';
284284import { accountStorage } from './scripts/util/AccountStorage.js';
285285import { initWelcomeScreen, openPermanentAssistantChat, openPermanentAssistantCard, getPermanentAssistantAvatar } from './scripts/welcome-screen.js';
286+import { initDataMaid } from './scripts/data-maid.js';
286287
287288// API OBJECT FOR EXTERNAL WIRING
288289globalThis.SillyTavern = {
@@ -1026,6 +1027,7 @@ async function firstLoadInit() {
10261027 initWelcomeScreen();
10271028 await initScrapers();
10281029 initCustomSelectedSamplers();
1030+ initDataMaid();
10291031 addDebugFunctions();
10301032 doDailyExtensionUpdatesCheck();
10311033 await hideLoader();
public/scripts/data-maid.js+393 -0
@@ -0,0 +1,393 @@
1+import { getRequestHeaders } from '../script.js';
2+import { t } from './i18n.js';
3+import { callGenericPopup, Popup, POPUP_TYPE } from './popup.js';
4+import { renderTemplateAsync } from './templates.js';
5+import { humanFileSize, timestampToMoment } from './utils.js';
6+
7+/**
8+ * @typedef {object} DataMaidReportResult
9+ * @property {import('../../src/endpoints/data-maid.js').DataMaidSanitizedReport} report - The sanitized report of the Data Maid.
10+ * @property {string} token - The token to use for the Data Maid report.
11+ */
12+
13+/**
14+ * Data Maid Dialog class for managing the cleanup dialog interface.
15+ */
16+class DataMaidDialog {
17+ constructor() {
18+ this.token = null;
19+ this.container = null;
20+ this.isScanning = false;
21+
22+ this.DATA_MAID_CATEGORIES = {
23+ files: {
24+ name: t`Files`,
25+ description: t`Files that are not associated with chat messages or Data Bank. WILL DELETE MANUAL UPLOADS!`,
26+ },
27+ images: {
28+ name: t`Images`,
29+ description: t`Images that are not associated with chat messages. WILL DELETE MANUAL UPLOADS!`,
30+ },
31+ chats: {
32+ name: t`Chats`,
33+ description: t`Chat files associated with deleted characters.`,
34+ },
35+ groupChats: {
36+ name: t`Group Chats`,
37+ description: t`Chat files associated with deleted groups.`,
38+ },
39+ avatarThumbnails: {
40+ name: t`Avatar Thumbnails`,
41+ description: t`Thumbnails for avatars of missing or deleted characters.`,
42+ },
43+ backgroundThumbnails: {
44+ name: t`Background Thumbnails`,
45+ description: t`Thumbnails for missing or deleted backgrounds.`,
46+ },
47+ chatBackups: {
48+ name: t`Chat Backups`,
49+ description: t`Automatically generated chat backups.`,
50+ },
51+ settingsBackups: {
52+ name: t`Settings Backups`,
53+ description: t`Automatically generated settings backups.`,
54+ },
55+ };
56+ }
57+
58+ /**
59+ * Returns a promise that resolves to the Data Maid report.
60+ * @returns {Promise<DataMaidReportResult>}
61+ * @private
62+ */
63+ async getReport() {
64+ const response = await fetch('/api/data-maid/report', {
65+ method: 'POST',
66+ headers: getRequestHeaders(),
67+ });
68+
69+ if (!response.ok) {
70+ throw new Error(`Error fetching Data Maid report: ${response.statusText}`);
71+ }
72+
73+ return await response.json();
74+ }
75+
76+ /**
77+ * Finalizes the Data Maid process by sending a request to the server.
78+ * @returns {Promise<void>}
79+ * @private
80+ */
81+ async finalize() {
82+ const response = await fetch('/api/data-maid/finalize', {
83+ method: 'POST',
84+ headers: getRequestHeaders(),
85+ body: JSON.stringify({ token: this.token }),
86+ });
87+
88+ if (!response.ok) {
89+ throw new Error(`Error finalizing Data Maid: ${response.statusText}`);
90+ }
91+ }
92+
93+ /**
94+ * Sets up the dialog UI elements and event listeners.
95+ * @private
96+ */
97+ async setupDialogUI() {
98+ const template = await renderTemplateAsync('dataMaidDialog');
99+ this.container = document.createElement('div');
100+ this.container.classList.add('dataMaidDialogContainer');
101+ this.container.innerHTML = template;
102+
103+ const startButton = this.container.querySelector('.dataMaidStartButton');
104+ startButton.addEventListener('click', () => this.handleScanClick());
105+ }
106+
107+ /**
108+ * Handles the scan button click event.
109+ * @private
110+ */
111+ async handleScanClick() {
112+ if (this.isScanning) {
113+ toastr.warning(t`The scan is already running. Please wait for it to finish.`);
114+ return;
115+ }
116+
117+ try {
118+ const resultsList = this.container.querySelector('.dataMaidResultsList');
119+ resultsList.innerHTML = '';
120+ this.showSpinner();
121+ this.isScanning = true;
122+
123+ const report = await this.getReport();
124+
125+ this.hideSpinner();
126+ await this.renderReport(report, resultsList);
127+ this.token = report.token;
128+ } catch (error) {
129+ this.hideSpinner();
130+ toastr.error(t`An error has occurred. Check the console for details.`);
131+ console.error('Error generating Data Maid report:', error);
132+ } finally {
133+ this.isScanning = false;
134+ }
135+ }
136+
137+ /**
138+ * Shows the loading spinner and hides the placeholder.
139+ * @private
140+ */
141+ showSpinner() {
142+ const spinner = this.container.querySelector('.dataMaidSpinner');
143+ const placeholder = this.container.querySelector('.dataMaidPlaceholder');
144+ placeholder.classList.add('displayNone');
145+ spinner.classList.remove('displayNone');
146+ }
147+
148+ /**
149+ * Hides the loading spinner.
150+ * @private
151+ */
152+ hideSpinner() {
153+ const spinner = this.container.querySelector('.dataMaidSpinner');
154+ spinner.classList.add('displayNone');
155+ }
156+
157+ /**
158+ * Renders the Data Maid report into the results list.
159+ * @param {DataMaidReportResult} report
160+ * @param {Element} resultsList
161+ * @private
162+ */
163+ async renderReport(report, resultsList) {
164+ for (const [prop, data] of Object.entries(this.DATA_MAID_CATEGORIES)) {
165+ const category = await this.renderCategory(prop, data.name, data.description, report.report[prop]);
166+ if (!category) {
167+ continue;
168+ }
169+ resultsList.appendChild(category);
170+ }
171+ this.displayEmptyPlaceholder();
172+ }
173+
174+ /**
175+ * Displays a placeholder message if no items are found in the results list.
176+ * @private
177+ */
178+ displayEmptyPlaceholder() {
179+ const resultsList = this.container.querySelector('.dataMaidResultsList');
180+ if (resultsList.children.length === 0) {
181+ const placeholder = this.container.querySelector('.dataMaidPlaceholder');
182+ placeholder.classList.remove('displayNone');
183+ placeholder.textContent = t`No items found to clean up. Come back later!`;
184+ }
185+ }
186+
187+ /**
188+ * Renders a single Data Maid category into a DOM element.
189+ * @param {string} prop Property name for the category
190+ * @param {string} name Name of the category
191+ * @param {string} description Description of the category
192+ * @param {import('../../src/endpoints/data-maid.js').DataMaidSanitizedRecord[]} items List of items in the category
193+ * @return {Promise<Element|null>} A promise that resolves to a DOM element containing the rendered category
194+ * @private
195+ */
196+ async renderCategory(prop, name, description, items) {
197+ if (!Array.isArray(items) || items.length === 0) {
198+ return null;
199+ }
200+
201+ const viewModel = {
202+ name: name,
203+ description: description,
204+ totalSize: humanFileSize(items.reduce((sum, item) => sum + item.size, 0)),
205+ totalItems: items.length,
206+ items: items.sort((a, b) => b.mtime - a.mtime).map(item => ({
207+ ...item,
208+ size: humanFileSize(item.size),
209+ date: timestampToMoment(item.mtime).format('L LT'),
210+ })),
211+ };
212+
213+ const template = await renderTemplateAsync('dataMaidCategory', viewModel);
214+ const categoryElement = document.createElement('div');
215+ categoryElement.innerHTML = template;
216+ categoryElement.querySelectorAll('.dataMaidItemView').forEach(button => {
217+ button.addEventListener('click', async () => {
218+ const item = button.closest('.dataMaidItem');
219+ const hash = item?.getAttribute('data-hash');
220+ if (hash) {
221+ await this.view(prop, hash);
222+ }
223+ });
224+ });
225+ categoryElement.querySelectorAll('.dataMaidItemDownload').forEach(button => {
226+ button.addEventListener('click', async () => {
227+ const item = button.closest('.dataMaidItem');
228+ const hash = item?.getAttribute('data-hash');
229+ if (hash) {
230+ await this.download(items, hash);
231+ }
232+ });
233+ });
234+ categoryElement.querySelectorAll('.dataMaidDeleteAll').forEach(button => {
235+ button.addEventListener('click', async (event) => {
236+ event.stopPropagation();
237+ const confirm = await Popup.show.confirm(t`Are you sure?`, t`This will permanently delete all files in this category. THIS CANNOT BE UNDONE!`);
238+ if (!confirm) {
239+ return;
240+ }
241+
242+ const hashes = items.map(item => item.hash).filter(hash => hash);
243+ await this.delete(hashes);
244+
245+ categoryElement.remove();
246+ this.displayEmptyPlaceholder();
247+ });
248+
249+ });
250+ categoryElement.querySelectorAll('.dataMaidItemDelete').forEach(button => {
251+ button.addEventListener('click', async () => {
252+ const item = button.closest('.dataMaidItem');
253+ const hash = item?.getAttribute('data-hash');
254+ if (hash) {
255+ const confirm = await Popup.show.confirm(t`Are you sure?`, t`This will permanently delete the file. THIS CANNOT BE UNDONE!`);
256+ if (!confirm) {
257+ return;
258+ }
259+ if (await this.delete([hash])) {
260+ item.remove();
261+ items.splice(items.findIndex(i => i.hash === hash), 1);
262+ if (items.length === 0) {
263+ categoryElement.remove();
264+ this.displayEmptyPlaceholder();
265+ }
266+ }
267+ }
268+ });
269+ });
270+ return categoryElement;
271+ }
272+
273+ /**
274+ * Constructs the URL for viewing an item by its hash.
275+ * @param {string} hash Hash of the item to view
276+ * @returns {string} URL to view the item
277+ * @private
278+ */
279+ getViewUrl(hash) {
280+ return `/api/data-maid/view?hash=${encodeURIComponent(hash)}&token=${encodeURIComponent(this.token)}`;
281+ }
282+
283+ /**
284+ * Downloads an item by its hash.
285+ * @param {import('../../src/endpoints/data-maid.js').DataMaidSanitizedRecord[]} items List of items in the category
286+ * @param {string} hash Hash of the item to download
287+ * @private
288+ */
289+ async download(items, hash) {
290+ const item = items.find(i => i.hash === hash);
291+ if (!item) {
292+ return;
293+ }
294+ const url = this.getViewUrl(hash);
295+ const a = document.createElement('a');
296+ a.href = url;
297+ a.download = item?.name || hash;
298+ document.body.appendChild(a);
299+ a.click();
300+ document.body.removeChild(a);
301+ }
302+
303+ /**
304+ * Opens the item view for a specific hash.
305+ * @param {string} prop Property name for the category
306+ * @param {string} hash Item hash to view
307+ * @private
308+ */
309+ async view(prop, hash) {
310+ const url = this.getViewUrl(hash);
311+ const isImage = ['images', 'avatarThumbnails', 'backgroundThumbnails'].includes(prop);
312+ const element = isImage
313+ ? await this.getViewElement(url)
314+ : await this.getTextViewElement(url);
315+ await callGenericPopup(element, POPUP_TYPE.DISPLAY, '', { large: true, wide: true });
316+ }
317+
318+ /**
319+ * Deletes an item by its file path hash.
320+ * @param {string[]} hashes Hashes of items to delete
321+ * @return {Promise<boolean>} True if the deletion was successful, false otherwise
322+ * @private
323+ */
324+ async delete(hashes) {
325+ try {
326+ const response = await fetch('/api/data-maid/delete', {
327+ method: 'POST',
328+ headers: getRequestHeaders(),
329+ body: JSON.stringify({ hashes: hashes, token: this.token }),
330+ });
331+
332+ if (!response.ok) {
333+ throw new Error(`Error deleting item: ${response.statusText}`);
334+ }
335+
336+ return true;
337+ } catch (error) {
338+ console.error('Error deleting item:', error);
339+ return false;
340+ }
341+ }
342+
343+ /**
344+ * Gets an image element for viewing images.
345+ * @param {string} url View URL
346+ * @returns {Promise<HTMLElement>} Image element
347+ * @private
348+ */
349+ async getViewElement(url) {
350+ const img = document.createElement('img');
351+ img.src = url;
352+ img.classList.add('dataMaidImageView');
353+ return img;
354+ }
355+
356+ /**
357+ * Gets an iframe element for viewing text content.
358+ * @param {string} url View URL
359+ * @returns {Promise<HTMLTextAreaElement>} Frame element
360+ * @private
361+ */
362+ async getTextViewElement(url) {
363+ const response = await fetch(url);
364+ const text = await response.text();
365+ const element = document.createElement('textarea');
366+ element.classList.add('dataMaidTextView');
367+ element.readOnly = true;
368+ element.textContent = text;
369+ return element;
370+ }
371+
372+ /**
373+ * Opens the Data Maid dialog and handles the interaction.
374+ */
375+ async open() {
376+ await this.setupDialogUI();
377+ await callGenericPopup(this.container, POPUP_TYPE.TEXT, '', { wide: true, large: true });
378+
379+ if (this.token) {
380+ await this.finalize();
381+ }
382+ }
383+}
384+
385+export function initDataMaid() {
386+ const dataMaidButton = document.getElementById('data_maid_button');
387+ if (!dataMaidButton) {
388+ console.warn('Data Maid button not found');
389+ return;
390+ }
391+
392+ dataMaidButton.addEventListener('click', () => new DataMaidDialog().open());
393+}
public/scripts/extensions/attachments/index.js+29 -0
@@ -216,8 +216,37 @@ function cleanUpAttachments() {
216216 }
217217}
218218
219+/**
220+ * Clean up character attachments when a character is deleted.
221+ * @param {{character: import('../../char-data.js').v1CharData}} data Event data
222+ */
223+function cleanUpCharacterAttachments(data) {
224+ const avatar = data?.character?.avatar;
225+ if (!avatar) return;
226+ if (Array.isArray(extension_settings?.character_attachments?.[avatar])) {
227+ delete extension_settings.character_attachments[avatar];
228+ saveSettingsDebounced();
229+ }
230+}
231+
232+/**
233+ * Handle character rename event to update character attachments.
234+ * @param {string} oldAvatar Old avatar name
235+ * @param {string} newAvatar New avatar name
236+ */
237+function handleCharacterRename(oldAvatar, newAvatar) {
238+ if (!oldAvatar || !newAvatar) return;
239+ if (Array.isArray(extension_settings?.character_attachments?.[oldAvatar])) {
240+ extension_settings.character_attachments[newAvatar] = extension_settings.character_attachments[oldAvatar];
241+ delete extension_settings.character_attachments[oldAvatar];
242+ saveSettingsDebounced();
243+ }
244+}
245+
219246jQuery(async () => {
220247 eventSource.on(event_types.APP_READY, cleanUpAttachments);
248+ eventSource.on(event_types.CHARACTER_DELETED, cleanUpCharacterAttachments);
249+ eventSource.on(event_types.CHARACTER_RENAMED, handleCharacterRename);
221250 const manageButton = await renderExtensionTemplateAsync('attachments', 'manage-button', {});
222251 const attachButton = await renderExtensionTemplateAsync('attachments', 'attach-button', {});
223252 $('#data_bank_wand_container').append(manageButton);
public/scripts/templates/dataMaidCategory.html+68 -0
@@ -0,0 +1,68 @@
1+<div class="dataMaidCategory inline-drawer">
2+ <div class="inline-drawer-toggle inline-drawer-header">
3+ <div class="dataMaidCategoryHeader">
4+ <div class="dataMaidCategoryDetails">
5+ <div class="dataMaidCategoryName" data-i18n="{{name}}">
6+ {{name}}
7+ </div>
8+ <small>{{description}}</small>
9+ <div class="dataMaidCategoryInfo">
10+ <small>
11+ <i class="fa-solid fa-file-alt fa-sm"></i>
12+ {{totalItems}}
13+ </small>
14+ <span>&VerticalBar;</span>
15+ <small>
16+ <i class="fa-solid fa-hdd fa-sm"></i>
17+ {{totalSize}}
18+ </small>
19+ </div>
20+ </div>
21+ <div class="dataMaidDeleteAll right_menu_button" title="Delete all items in this category" data-i18n="[title]Delete all items in this category">
22+ <i class="fa-solid fa-fw fa-broom"></i>
23+ </div>
24+ </div>
25+ <div class="fa-solid fa-circle-chevron-down inline-drawer-icon down"></div>
26+ </div>
27+ <div class="dataMaidCategoryContent inline-drawer-content">
28+ <div class="flex-container">
29+ {{#each items}}
30+ {{#with this}}
31+ <div class="dataMaidItem" data-hash="{{hash}}">
32+ <div class="dataMaidItemHeader">
33+ <div class="dataMaidItemName">
34+ {{#if parent}}
35+ <span class="dataMaidItemParent">({{parent}})</span>
36+ <span>/</span>
37+ {{/if}}
38+ <b>{{name}}</b>
39+ </div>
40+ <div class="dataMaidItemActions">
41+ <button class="dataMaidItemView menu_button menu_button_icon margin0" title="View item content" data-i18n="[title]View item content">
42+ <i class="fa-solid fa-fw fa-eye"></i>
43+ </button>
44+ <button class="dataMaidItemDownload menu_button menu_button_icon margin0" title="Download item" data-i18n="[title]Download item">
45+ <i class="fa-solid fa-fw fa-download"></i>
46+ </button>
47+ <button class="dataMaidItemDelete menu_button menu_button_icon margin0" title="Delete this item" data-i18n="[title]Delete this item">
48+ <i class="fa-solid fa-fw fa-trash-alt"></i>
49+ </button>
50+ </div>
51+ </div>
52+ <div class="dataMaidItemInfo">
53+ <small>
54+ <i class="fa-solid fa-file fa-sm"></i>
55+ {{size}}
56+ </small>
57+ <span>&VerticalBar;</span>
58+ <small>
59+ <i class="fa-solid fa-calendar fa-sm"></i>
60+ {{date}}
61+ </small>
62+ </div>
63+ </div>
64+ {{/with}}
65+ {{/each}}
66+ </div>
67+ </div>
68+</div>
public/scripts/templates/dataMaidDialog.html+25 -0
@@ -0,0 +1,25 @@
1+<div class="dataMaidDialog">
2+ <div class="dataMaidDialogHeader">
3+ <div class="dataMaidHeaderInfo info-block warning margin0">
4+ <small data-i18n="Once deleted, the files will be gone forever!">
5+ Once deleted, the files will be gone forever!
6+ </small>
7+ <br>
8+ <small data-i18n="Make sure to back up your data in advance.">
9+ Make sure to back up your data in advance.
10+ </small>
11+ </div>
12+ <button class="menu_button menu_button_icon dataMaidStartButton">
13+ <i class="fa fa-cog"></i>
14+ <span data-i18n="Scan">Scan</span>
15+ </button>
16+ </div>
17+ <hr>
18+ <div class="dataMaidPlaceholder" data-i18n="No results yet. Tap 'Scan' to start scanning.">
19+ No results yet. Tap 'Scan' to start scanning.
20+ </div>
21+ <div class="displayNone dataMaidSpinner">
22+ <i class="fa-solid fa-spinner fa-spin fa-3x"></i>
23+ </div>
24+ <div class="dataMaidResultsList"></div>
25+</div>
public/style.css+1 -0
@@ -11,6 +11,7 @@
1111@import url(css/tags.css);
1212@import url(css/scrollable-button.css);
1313@import url(css/welcome.css);
14+@import url(css/data-maid.css);
1415
1516:root {
1617 --doc-height: 100%;
src/endpoints/chats.js+5 -4
@@ -23,6 +23,8 @@ const maxTotalChatBackups = Number(getConfigValue('backups.chat.maxTotalBackups'
2323const throttleInterval = Number(getConfigValue('backups.chat.throttleInterval', 10_000, 'number'));
2424const checkIntegrity = !!getConfigValue('backups.chat.checkIntegrity', true, 'boolean');
2525
26+export const CHAT_BACKUPS_PREFIX = 'chat_';
27+
2628/**
2729 * Saves a chat to the backups directory.
2830 * @param {string} directory The user's backups directory.
@@ -31,7 +33,6 @@ const checkIntegrity = !!getConfigValue('backups.chat.checkIntegrity', true, 'bo
3133 */
3234function backupChat(directory, name, chat) {
3335 try {
34-
3536 if (!isBackupEnabled) {
3637 return;
3738 }
@@ -39,16 +40,16 @@ function backupChat(directory, name, chat) {
3940 // replace non-alphanumeric characters with underscores
4041 name = sanitize(name).replace(/[^a-z0-9]/gi, '_').toLowerCase();
4142
4243 const backupFile = path.join(directory, `chat_${CHAT_BACKUPS_PREFIX}${name}_${generateTimestamp()}.jsonl`);
4344 writeFileAtomicSync(backupFile, chat, 'utf-8');
4445
4546 removeOldBackups(directory, `chat_${CHAT_BACKUPS_PREFIX}${name}_`);
4647
4748 if (isNaN(maxTotalChatBackups) || maxTotalChatBackups < 0) {
4849 return;
4950 }
5051
5152 removeOldBackups(directory, 'chat_'CHAT_BACKUPS_PREFIX, maxTotalChatBackups);
5253 } catch (err) {
5354 console.error(`Could not backup chat for ${name}`, err);
5455 }
src/endpoints/data-maid.js+735 -0
@@ -0,0 +1,735 @@
1+import crypto from 'node:crypto';
2+import fs from 'node:fs';
3+import path from 'node:path';
4+import express from 'express';
5+import mime from 'mime-types';
6+import { getSettingsBackupFilePrefix } from './settings.js';
7+import { CHAT_BACKUPS_PREFIX } from './chats.js';
8+import { isPathUnderParent, tryParse } from '../util.js';
9+import { SETTINGS_FILE } from '../constants.js';
10+
11+const sha256 = str => crypto.createHash('sha256').update(str).digest('hex');
12+
13+/**
14+ * @typedef {object} DataMaidRawReport
15+ * @property {string[]} images - List of loose user images
16+ * @property {string[]} files - List of loose user files
17+ * @property {string[]} chats - List of loose character chats
18+ * @property {string[]} groupChats - List of loose group chats
19+ * @property {string[]} avatarThumbnails - List of loose avatar thumbnails
20+ * @property {string[]} backgroundThumbnails - List of loose background thumbnails
21+ * @property {string[]} chatBackups - List of chat backups
22+ * @property {string[]} settingsBackups - List of settings backups
23+ */
24+
25+/**
26+ * @typedef {object} DataMaidSanitizedRecord - The entry excluding the sensitive paths.
27+ * @property {string} name - The name of the file.
28+ * @property {string} hash - The SHA-256 hash of the file path.
29+ * @property {string} [parent] - The name of the parent directory, if applicable.
30+ * @property {number} [size] - The size of the file in bytes, if available.
31+ * @property {number} [mtime] - The last modification time of the file, if available.
32+ */
33+
34+/**
35+ * @typedef {object} DataMaidSanitizedReport - The report containing loose user data.
36+ * @property {DataMaidSanitizedRecord[]} images - List of sanitized loose user images
37+ * @property {DataMaidSanitizedRecord[]} files - List of sanitized loose user files
38+ * @property {DataMaidSanitizedRecord[]} chats - List of sanitized loose character chats
39+ * @property {DataMaidSanitizedRecord[]} groupChats - List of sanitized loose group chats
40+ * @property {DataMaidSanitizedRecord[]} avatarThumbnails - List of sanitized loose avatar thumbnails
41+ * @property {DataMaidSanitizedRecord[]} backgroundThumbnails - List of sanitized loose background thumbnails
42+ * @property {DataMaidSanitizedRecord[]} chatBackups - List of sanitized chat backups
43+ * @property {DataMaidSanitizedRecord[]} settingsBackups - List of sanitized settings backups
44+ */
45+
46+/**
47+ * @typedef {object} DataMaidMessage - The chat message object.
48+ * @property {DataMaidMessageExtra} [extra] - The extra data object.
49+ * @property {DataMaidChatMetadata} [chat_metadata] - The chat metadata object.
50+ */
51+
52+/**
53+ * @typedef {object} DataMaidFile - The file object.
54+ * @property {string} url - The file URL
55+ */
56+
57+/**
58+ * @typedef {object} DataMaidChatMetadata - The chat metadata object.
59+ * @property {DataMaidFile[]} [attachments] - The array of attachments, if any.
60+ */
61+
62+/**
63+ * @typedef {object} DataMaidMessageExtra - The extra data object.
64+ * @property {string} [image] - The link to the image, if any.
65+ * @property {string[]} [image_swipes] - The links to the image swipes, if any.
66+ * @property {DataMaidFile} [file] - The file object, if any.
67+ */
68+
69+/**
70+ * @typedef {object} DataMaidTokenEntry
71+ * @property {string} handle - The user's handle or identifier.
72+ * @property {{path: string, hash: string}[]} paths - The list of file paths and their hashes that can be cleaned up.
73+ */
74+
75+/**
76+ * Service for detecting and managing loose user data files.
77+ * Helps identify orphaned files that are no longer referenced by the application.
78+ */
79+export class DataMaidService {
80+ /**
81+ * @type {Map<string, DataMaidTokenEntry>} Map clean-up tokens to user IDs
82+ */
83+ static TOKENS = new Map();
84+
85+ /**
86+ * Creates a new DataMaidService instance for a specific user.
87+ * @param {string} handle - The user's handle.
88+ * @param {import('../users.js').UserDirectoryList} directories - List of user directories to scan for loose data.
89+ */
90+ constructor(handle, directories) {
91+ this.handle = handle;
92+ this.directories = directories;
93+ }
94+
95+ /**
96+ * Generates a report of loose user data.
97+ * @returns {Promise<DataMaidRawReport>} A report containing lists of loose user data.
98+ */
99+ async generateReport() {
100+ /** @type {DataMaidRawReport} */
101+ const report = {
102+ images: await this.#collectImages(),
103+ files: await this.#collectFiles(),
104+ chats: await this.#collectChats(),
105+ groupChats: await this.#collectGroupChats(),
106+ avatarThumbnails: await this.#collectAvatarThumbnails(),
107+ backgroundThumbnails: await this.#collectBackgroundThumbnails(),
108+ chatBackups: await this.#collectChatBackups(),
109+ settingsBackups: await this.#collectSettingsBackups(),
110+ };
111+
112+ return report;
113+ }
114+
115+
116+ /**
117+ * Sanitizes a record by hashing the file name and removing sensitive information.
118+ * Additionally, adds metadata like size and modification time.
119+ * @param {string} name The file or directory name to sanitize.
120+ * @param {boolean} withParent If the model should include the parent directory name.
121+ * @returns {Promise<DataMaidSanitizedRecord>} A sanitized record with the file name, hash, parent directory name, size, and modification time.
122+ */
123+ async #sanitizeRecord(name, withParent) {
124+ const stat = fs.existsSync(name) ? await fs.promises.stat(name) : null;
125+ return {
126+ name: path.basename(name),
127+ hash: sha256(name),
128+ parent: withParent ? path.basename(path.dirname(name)) : void 0,
129+ size: stat?.size,
130+ mtime: stat?.mtimeMs,
131+ };
132+ }
133+
134+ /**
135+ * Sanitizes the report by hashing the file paths and removing sensitive information.
136+ * @param {DataMaidRawReport} report - The raw report containing loose user data.
137+ * @returns {Promise<DataMaidSanitizedReport>} A sanitized report with sensitive paths removed.
138+ */
139+ async sanitizeReport(report) {
140+ const sanitizedReport = {
141+ images: await Promise.all(report.images.map(i => this.#sanitizeRecord(i, true))),
142+ files: await Promise.all(report.files.map(i => this.#sanitizeRecord(i, false))),
143+ chats: await Promise.all(report.chats.map(i => this.#sanitizeRecord(i, true))),
144+ groupChats: await Promise.all(report.groupChats.map(i => this.#sanitizeRecord(i, false))),
145+ avatarThumbnails: await Promise.all(report.avatarThumbnails.map(i => this.#sanitizeRecord(i, false))),
146+ backgroundThumbnails: await Promise.all(report.backgroundThumbnails.map(i => this.#sanitizeRecord(i, false))),
147+ chatBackups: await Promise.all(report.chatBackups.map(i => this.#sanitizeRecord(i, false))),
148+ settingsBackups: await Promise.all(report.settingsBackups.map(i => this.#sanitizeRecord(i, false))),
149+ };
150+
151+ return sanitizedReport;
152+ }
153+
154+ /**
155+ * Collects loose user images from the provided directories.
156+ * Images are considered loose if they exist in the user images directory
157+ * but are not referenced in any chat messages.
158+ * @returns {Promise<string[]>} List of paths to loose user images
159+ */
160+ async #collectImages() {
161+ const result = [];
162+
163+ try {
164+ const messages = await this.#parseAllChats(x => !!x?.extra?.image || Array.isArray(x?.extra?.image_swipes));
165+ const knownImages = new Set();
166+ for (const message of messages) {
167+ if (message?.extra?.image) {
168+ knownImages.add(message.extra.image);
169+ }
170+ if (Array.isArray(message?.extra?.image_swipes)) {
171+ for (const swipe of message.extra.image_swipes) {
172+ knownImages.add(swipe);
173+ }
174+ }
175+ }
176+ const knownImageFullPaths = new Set();
177+ knownImages.forEach(image => {
178+ if (image.startsWith('http') || image.startsWith('data:')) {
179+ return; // Skip URLs and data URIs
180+ }
181+ knownImageFullPaths.add(path.normalize(path.join(this.directories.root, image)));
182+ });
183+ const images = await fs.promises.readdir(this.directories.userImages, { withFileTypes: true });
184+ for (const dirent of images) {
185+ const direntPath = path.join(dirent.parentPath, dirent.name);
186+ if (dirent.isFile() && !knownImageFullPaths.has(direntPath)) {
187+ result.push(direntPath);
188+ }
189+ if (dirent.isDirectory()) {
190+ const subdirFiles = await fs.promises.readdir(direntPath, { withFileTypes: true });
191+ for (const file of subdirFiles) {
192+ const subdirFilePath = path.join(direntPath, file.name);
193+ if (file.isFile() && !knownImageFullPaths.has(subdirFilePath)) {
194+ result.push(subdirFilePath);
195+ }
196+ }
197+ }
198+ }
199+ } catch (error) {
200+ console.error('[Data Maid] Error collecting user images:', error);
201+ }
202+
203+ return result;
204+ }
205+
206+ /**
207+ * Collects loose user files from the provided directories.
208+ * Files are considered loose if they exist in the files directory
209+ * but are not referenced in chat messages, metadata, or settings.
210+ * @returns {Promise<string[]>} List of paths to loose user files
211+ */
212+ async #collectFiles() {
213+ const result = [];
214+
215+ try {
216+ const messages = await this.#parseAllChats(x => !!x?.extra?.file?.url);
217+ const knownFiles = new Set();
218+ for (const message of messages) {
219+ if (message?.extra?.file?.url) {
220+ knownFiles.add(message.extra.file.url);
221+ }
222+ }
223+ const metadata = await this.#parseAllMetadata(x => Array.isArray(x?.attachments) && x.attachments.length > 0);
224+ for (const meta of metadata) {
225+ if (Array.isArray(meta?.attachments)) {
226+ for (const attachment of meta.attachments) {
227+ if (attachment?.url) {
228+ knownFiles.add(attachment.url);
229+ }
230+ }
231+ }
232+ }
233+ const pathToSettings = path.join(this.directories.root, SETTINGS_FILE);
234+ if (fs.existsSync(pathToSettings)) {
235+ try {
236+ const settingsContent = await fs.promises.readFile(pathToSettings, 'utf-8');
237+ const settings = tryParse(settingsContent);
238+ if (Array.isArray(settings?.extension_settings?.attachments)) {
239+ for (const file of settings.extension_settings.attachments) {
240+ if (file?.url) {
241+ knownFiles.add(file.url);
242+ }
243+ }
244+ }
245+ if (typeof settings?.extension_settings?.character_attachments === 'object') {
246+ for (const files of Object.values(settings.extension_settings.character_attachments)) {
247+ if (!Array.isArray(files)) {
248+ continue;
249+ }
250+ for (const file of files) {
251+ if (file?.url) {
252+ knownFiles.add(file.url);
253+ }
254+ }
255+ }
256+ }
257+ } catch (error) {
258+ console.error('[Data Maid] Error reading settings file:', error);
259+ }
260+ }
261+ const knownFileFullPaths = new Set();
262+ knownFiles.forEach(file => {
263+ knownFileFullPaths.add(path.normalize(path.join(this.directories.root, file)));
264+ });
265+ const files = await fs.promises.readdir(this.directories.files, { withFileTypes: true });
266+ for (const file of files) {
267+ const filePath = path.join(this.directories.files, file.name);
268+ if (file.isFile() && !knownFileFullPaths.has(filePath)) {
269+ result.push(filePath);
270+ }
271+ }
272+ } catch (error) {
273+ console.error('[Data Maid] Error collecting user files:', error);
274+ }
275+
276+ return result;
277+ }
278+
279+ /**
280+ * Collects loose character chats from the provided directories.
281+ * Chat folders are considered loose if they don't have corresponding character files.
282+ * @returns {Promise<string[]>} List of paths to loose character chats
283+ */
284+ async #collectChats() {
285+ const result = [];
286+
287+ try {
288+ const knownChatFolders = new Set();
289+ const characters = await fs.promises.readdir(this.directories.characters, { withFileTypes: true });
290+ for (const file of characters) {
291+ if (file.isFile() && path.parse(file.name).ext === '.png') {
292+ knownChatFolders.add(file.name.replace('.png', ''));
293+ }
294+ }
295+ const chatFolders = await fs.promises.readdir(this.directories.chats, { withFileTypes: true });
296+ for (const folder of chatFolders) {
297+ if (folder.isDirectory() && !knownChatFolders.has(folder.name)) {
298+ const chatFiles = await fs.promises.readdir(path.join(this.directories.chats, folder.name), { withFileTypes: true });
299+ for (const file of chatFiles) {
300+ if (file.isFile() && path.parse(file.name).ext === '.jsonl') {
301+ result.push(path.join(this.directories.chats, folder.name, file.name));
302+ }
303+ }
304+ }
305+ }
306+ } catch (error) {
307+ console.error('[Data Maid] Error collecting character chats:', error);
308+ }
309+
310+ return result;
311+ }
312+
313+ /**
314+ * Collects loose group chats from the provided directories.
315+ * Group chat files are considered loose if they're not referenced by any group definition.
316+ * @returns {Promise<string[]>} List of paths to loose group chats
317+ */
318+ async #collectGroupChats() {
319+ const result = [];
320+
321+ try {
322+ const groups = await fs.promises.readdir(this.directories.groups, { withFileTypes: true });
323+ const knownGroupChats = new Set();
324+ for (const file of groups) {
325+ if (file.isFile() && path.parse(file.name).ext === '.json') {
326+ try {
327+ const pathToFile = path.join(this.directories.groups, file.name);
328+ const fileContent = await fs.promises.readFile(pathToFile, 'utf-8');
329+ const groupData = tryParse(fileContent);
330+ if (groupData?.chat_id) {
331+ knownGroupChats.add(groupData.chat_id);
332+ }
333+ if (Array.isArray(groupData?.chats)) {
334+ for (const chat of groupData.chats) {
335+ knownGroupChats.add(chat);
336+ }
337+ }
338+ } catch (error) {
339+ console.error(`[Data Maid] Error parsing group chat file ${file.name}:`, error);
340+ }
341+ }
342+ }
343+ const groupChats = await fs.promises.readdir(this.directories.groupChats, { withFileTypes: true });
344+ for (const file of groupChats) {
345+ if (file.isFile() && path.parse(file.name).ext === '.jsonl') {
346+ if (!knownGroupChats.has(path.parse(file.name).name)) {
347+ result.push(path.join(this.directories.groupChats, file.name));
348+ }
349+ }
350+ }
351+ } catch (error) {
352+ console.error('[Data Maid] Error collecting group chats:', error);
353+ }
354+
355+ return result;
356+ }
357+
358+ /**
359+ * Collects loose avatar thumbnails from the provided directories.
360+ * @returns {Promise<string[]>} List of paths to loose avatar thumbnails
361+ */
362+ async #collectAvatarThumbnails() {
363+ const result = [];
364+
365+ try {
366+ const knownAvatars = new Set();
367+ const avatars = await fs.promises.readdir(this.directories.characters, { withFileTypes: true });
368+ for (const file of avatars) {
369+ if (file.isFile()) {
370+ knownAvatars.add(file.name);
371+ }
372+ }
373+ const avatarThumbnails = await fs.promises.readdir(this.directories.thumbnailsAvatar, { withFileTypes: true });
374+ for (const file of avatarThumbnails) {
375+ if (file.isFile() && !knownAvatars.has(file.name)) {
376+ result.push(path.join(this.directories.thumbnailsAvatar, file.name));
377+ }
378+ }
379+ } catch (error) {
380+ console.error('[Data Maid] Error collecting avatar thumbnails:', error);
381+ }
382+
383+ return result;
384+ }
385+
386+ /**
387+ * Collects loose background thumbnails from the provided directories.
388+ * @returns {Promise<string[]>} List of paths to loose background thumbnails
389+ */
390+ async #collectBackgroundThumbnails() {
391+ const result = [];
392+
393+ try {
394+ const knownBackgrounds = new Set();
395+ const backgrounds = await fs.promises.readdir(this.directories.backgrounds, { withFileTypes: true });
396+ for (const file of backgrounds) {
397+ if (file.isFile()) {
398+ knownBackgrounds.add(file.name);
399+ }
400+ }
401+ const backgroundThumbnails = await fs.promises.readdir(this.directories.thumbnailsBg, { withFileTypes: true });
402+ for (const file of backgroundThumbnails) {
403+ if (file.isFile() && !knownBackgrounds.has(file.name)) {
404+ result.push(path.join(this.directories.thumbnailsBg, file.name));
405+ }
406+ }
407+ } catch (error) {
408+ console.error('[Data Maid] Error collecting background thumbnails:', error);
409+ }
410+
411+ return result;
412+ }
413+
414+ /**
415+ * Collects chat backups from the provided directories.
416+ * @returns {Promise<string[]>} List of paths to chat backups
417+ */
418+ async #collectChatBackups() {
419+ const result = [];
420+
421+ try {
422+ const prefix = CHAT_BACKUPS_PREFIX;
423+ const backups = await fs.promises.readdir(this.directories.backups, { withFileTypes: true });
424+ for (const file of backups) {
425+ if (file.isFile() && file.name.startsWith(prefix)) {
426+ result.push(path.join(this.directories.backups, file.name));
427+ }
428+ }
429+ } catch (error) {
430+ console.error('[Data Maid] Error collecting chat backups:', error);
431+ }
432+
433+ return result;
434+ }
435+
436+ /**
437+ * Collects settings backups from the provided directories.
438+ * @returns {Promise<string[]>} List of paths to settings backups
439+ */
440+ async #collectSettingsBackups() {
441+ const result = [];
442+
443+ try {
444+ const prefix = getSettingsBackupFilePrefix(this.handle);
445+ const backups = await fs.promises.readdir(this.directories.backups, { withFileTypes: true });
446+ for (const file of backups) {
447+ if (file.isFile() && file.name.startsWith(prefix)) {
448+ result.push(path.join(this.directories.backups, file.name));
449+ }
450+ }
451+ } catch (error) {
452+ console.error('[Data Maid] Error collecting settings backups:', error);
453+ }
454+
455+ return result;
456+ }
457+
458+ /**
459+ * Parses all chat files and returns an array of chat messages.
460+ * Searches both individual character chats and group chats.
461+ * @param {function(DataMaidMessage): boolean} filterFn - Filter function to apply to each message.
462+ * @returns {Promise<DataMaidMessage[]>} Array of chat messages
463+ */
464+ async #parseAllChats(filterFn) {
465+ try {
466+ const allChats = [];
467+
468+ const groupChats = await fs.promises.readdir(this.directories.groupChats, { withFileTypes: true });
469+ for (const file of groupChats) {
470+ if (file.isFile() && path.parse(file.name).ext === '.jsonl') {
471+ const chatMessages = await this.#parseChatFile(path.join(this.directories.groupChats, file.name));
472+ allChats.push(...chatMessages.filter(filterFn));
473+ }
474+ }
475+
476+ const chatDirectories = await fs.promises.readdir(this.directories.chats, { withFileTypes: true });
477+ for (const directory of chatDirectories) {
478+ if (directory.isDirectory()) {
479+ const chatFiles = await fs.promises.readdir(path.join(this.directories.chats, directory.name), { withFileTypes: true });
480+ for (const file of chatFiles) {
481+ if (file.isFile() && path.parse(file.name).ext === '.jsonl') {
482+ const chatMessages = await this.#parseChatFile(path.join(this.directories.chats, directory.name, file.name));
483+ allChats.push(...chatMessages.filter(filterFn));
484+ }
485+ }
486+ }
487+ }
488+
489+ return allChats;
490+ } catch (error) {
491+ console.error('[Data Maid] Error parsing chats:', error);
492+ return [];
493+ }
494+ }
495+
496+ /**
497+ * Parses all metadata from chat files and group definitions.
498+ * Extracts metadata from both active and historical chat data.
499+ * @param {function(DataMaidChatMetadata): boolean} filterFn - Filter function to apply to each metadata entry.
500+ * @returns {Promise<DataMaidChatMetadata[]>} Parsed chat metadata as an array.
501+ */
502+ async #parseAllMetadata(filterFn) {
503+ try {
504+ const allMetadata = [];
505+
506+ const groups = await fs.promises.readdir(this.directories.groups, { withFileTypes: true });
507+ for (const file of groups) {
508+ if (file.isFile() && path.parse(file.name).ext === '.json') {
509+ try {
510+ const pathToFile = path.join(this.directories.groups, file.name);
511+ const fileContent = await fs.promises.readFile(pathToFile, 'utf-8');
512+ const groupData = tryParse(fileContent);
513+ if (groupData?.chat_metadata && filterFn(groupData.chat_metadata)) {
514+ allMetadata.push(groupData.chat_metadata);
515+ }
516+ if (groupData?.past_metadata) {
517+ allMetadata.push(...Object.values(groupData.past_metadata).filter(filterFn));
518+ }
519+ } catch (error) {
520+ console.error(`[Data Maid] Error parsing group chat file ${file.name}:`, error);
521+ }
522+ }
523+ }
524+
525+ const chatDirectories = await fs.promises.readdir(this.directories.chats, { withFileTypes: true });
526+ for (const directory of chatDirectories) {
527+ if (directory.isDirectory()) {
528+ const chatFiles = await fs.promises.readdir(path.join(this.directories.chats, directory.name), { withFileTypes: true });
529+ for (const file of chatFiles) {
530+ if (file.isFile() && path.parse(file.name).ext === '.jsonl') {
531+ const chatMessages = await this.#parseChatFile(path.join(this.directories.chats, directory.name, file.name));
532+ const chatMetadata = chatMessages?.[0]?.chat_metadata;
533+ if (chatMetadata && filterFn(chatMetadata)) {
534+ allMetadata.push(chatMetadata);
535+ }
536+ }
537+ }
538+ }
539+ }
540+
541+ return allMetadata;
542+ } catch (error) {
543+ console.error('[Data Maid] Error parsing chats:', error);
544+ return [];
545+ }
546+ }
547+
548+ /**
549+ * Parses a single chat file and returns an array of chat messages.
550+ * Each line in the JSONL file represents one message.
551+ * @param {string} filePath Path to the chat file to parse.
552+ * @returns {Promise<DataMaidMessage[]>} Parsed chat messages as an array.
553+ */
554+ async #parseChatFile(filePath) {
555+ try {
556+ const content = await fs.promises.readFile(filePath, 'utf-8');
557+ const chatData = content.split('\n').map(tryParse).filter(Boolean);
558+ return chatData;
559+ } catch (error) {
560+ console.error(`[Data Maid] Error reading chat file ${filePath}:`, error);
561+ return [];
562+ }
563+ }
564+
565+ /**
566+ * Generates a unique token for the user to clean up their data.
567+ * Replaces any existing token for the same user.
568+ * @param {string} handle - The user's handle or identifier.
569+ * @param {DataMaidRawReport} report - The report containing loose user data.
570+ * @returns {string} A unique token.
571+ */
572+ static generateToken(handle, report) {
573+ // Remove any existing token for this user
574+ for (const [token, entry] of this.TOKENS.entries()) {
575+ if (entry.handle === handle) {
576+ this.TOKENS.delete(token);
577+ }
578+ }
579+
580+ const token = crypto.randomBytes(32).toString('hex');
581+ const tokenEntry = {
582+ handle,
583+ paths: Object.values(report).filter(v => Array.isArray(v)).flat().map(x => ({ path: x, hash: sha256(x) })),
584+ };
585+ this.TOKENS.set(token, tokenEntry);
586+ return token;
587+ }
588+}
589+
590+export const router = express.Router();
591+
592+router.post('/report', async (req, res) => {
593+ try {
594+ if (!req.user || !req.user.directories) {
595+ return res.sendStatus(403);
596+ }
597+
598+ const dataMaid = new DataMaidService(req.user.profile.handle, req.user.directories);
599+ const rawReport = await dataMaid.generateReport();
600+
601+ const report = await dataMaid.sanitizeReport(rawReport);
602+ const token = DataMaidService.generateToken(req.user.profile.handle, rawReport);
603+
604+ return res.json({ report, token });
605+ } catch (error) {
606+ console.error('[Data Maid] Error generating data maid report:', error);
607+ return res.sendStatus(500);
608+ }
609+});
610+
611+router.post('/finalize', async (req, res) => {
612+ try {
613+ if (!req.user || !req.user.directories) {
614+ return res.sendStatus(403);
615+ }
616+
617+ if (!req.body.token) {
618+ return res.sendStatus(400);
619+ }
620+
621+ const token = req.body.token.toString();
622+ if (!DataMaidService.TOKENS.has(token)) {
623+ return res.sendStatus(403);
624+ }
625+
626+ const tokenEntry = DataMaidService.TOKENS.get(token);
627+ if (!tokenEntry || tokenEntry.handle !== req.user.profile.handle) {
628+ return res.sendStatus(403);
629+ }
630+
631+ // Remove the token after finalization
632+ DataMaidService.TOKENS.delete(token);
633+ return res.sendStatus(204);
634+ } catch (error) {
635+ console.error('[Data Maid] Error finalizing the token:', error);
636+ return res.sendStatus(500);
637+ }
638+});
639+
640+router.get('/view', async (req, res) => {
641+ try {
642+ if (!req.user || !req.user.directories) {
643+ return res.sendStatus(403);
644+ }
645+
646+ if (!req.query.token || !req.query.hash) {
647+ return res.sendStatus(400);
648+ }
649+
650+ const token = req.query.token.toString();
651+ const hash = req.query.hash.toString();
652+
653+ if (!DataMaidService.TOKENS.has(token)) {
654+ return res.sendStatus(403);
655+ }
656+
657+ const tokenEntry = DataMaidService.TOKENS.get(token);
658+ if (!tokenEntry || tokenEntry.handle !== req.user.profile.handle) {
659+ return res.sendStatus(403);
660+ }
661+
662+ const fileEntry = tokenEntry.paths.find(entry => entry.hash === hash);
663+ if (!fileEntry) {
664+ return res.sendStatus(404);
665+ }
666+
667+ if (!isPathUnderParent(req.user.directories.root, fileEntry.path)) {
668+ console.warn('[Data Maid] Attempted access to a file outside of the user directory:', fileEntry.path);
669+ return res.sendStatus(403);
670+ }
671+
672+ const pathToFile = fileEntry.path;
673+ const fileExists = fs.existsSync(pathToFile);
674+
675+ if (!fileExists) {
676+ return res.sendStatus(404);
677+ }
678+
679+ const fileBuffer = await fs.promises.readFile(pathToFile);
680+ const mimeType = mime.lookup(pathToFile) || 'text/plain';
681+ res.setHeader('Content-Type', mimeType);
682+ return res.send(fileBuffer);
683+ } catch (error) {
684+ console.error('[Data Maid] Error viewing file:', error);
685+ return res.sendStatus(500);
686+ }
687+});
688+
689+router.post('/delete', async (req, res) => {
690+ try {
691+ if (!req.user || !req.user.directories) {
692+ return res.sendStatus(403);
693+ }
694+
695+ const { token, hashes } = req.body;
696+ if (!token || !Array.isArray(hashes) || hashes.length === 0) {
697+ return res.sendStatus(400);
698+ }
699+
700+ if (!DataMaidService.TOKENS.has(token)) {
701+ return res.sendStatus(403);
702+ }
703+
704+ const tokenEntry = DataMaidService.TOKENS.get(token);
705+ if (!tokenEntry || tokenEntry.handle !== req.user.profile.handle) {
706+ return res.sendStatus(403);
707+ }
708+
709+ for (const hash of hashes) {
710+ const fileEntry = tokenEntry.paths.find(entry => entry.hash === hash);
711+ if (!fileEntry) {
712+ continue;
713+ }
714+
715+ if (!isPathUnderParent(req.user.directories.root, fileEntry.path)) {
716+ console.warn('[Data Maid] Attempted deletion of a file outside of the user directory:', fileEntry.path);
717+ continue;
718+ }
719+
720+ const pathToFile = fileEntry.path;
721+ const fileExists = fs.existsSync(pathToFile);
722+
723+ if (!fileExists) {
724+ continue;
725+ }
726+
727+ await fs.promises.unlink(pathToFile);
728+ }
729+
730+ return res.sendStatus(204);
731+ } catch (error) {
732+ console.error('[Data Maid] Error deleting files:', error);
733+ return res.sendStatus(500);
734+ }
735+});
src/endpoints/settings.js+6 -6
@@ -81,7 +81,7 @@ function sortByName(_) {
8181 * @param {string} handle User handle
8282 * @returns {string} File prefix
8383 */
8484export function getFilePrefixgetSettingsBackupFilePrefix(handle) {
8585 return `settings_${handle}_`;
8686}
8787
@@ -131,7 +131,7 @@ async function backupSettings() {
131131 */
132132function backupUserSettings(handle, preventDuplicates) {
133133 const userDirectories = getUserDirectories(handle);
134134 const backupFile = path.join(userDirectories.backups, `${getFilePrefixgetSettingsBackupFilePrefix(handle)}${generateTimestamp()}.json`);
135135 const sourceFile = path.join(userDirectories.root, SETTINGS_FILE);
136136
137137 if (preventDuplicates && isDuplicateBackup(handle, sourceFile)) {
@@ -183,7 +183,7 @@ function areFilesEqual(file1, file2) {
183183function getLatestBackup(handle) {
184184 const userDirectories = getUserDirectories(handle);
185185 const backupFiles = fs.readdirSync(userDirectories.backups)
186186 .filter(x => x.startsWith(getFilePrefixgetSettingsBackupFilePrefix(handle)))
187187 .map(x => ({ name: x, ctime: fs.statSync(path.join(userDirectories.backups, x)).ctimeMs }));
188188 const latestBackup = backupFiles.sort((a, b) => b.ctime - a.ctime)[0]?.name;
189189 if (!latestBackup) {
@@ -283,7 +283,7 @@ router.post('/get', (request, response) => {
283283router.post('/get-snapshots', async (request, response) => {
284284 try {
285285 const snapshots = fs.readdirSync(request.user.directories.backups);
286286 const userFilesPattern = getFilePrefixgetSettingsBackupFilePrefix(request.user.profile.handle);
287287 const userSnapshots = snapshots.filter(x => x.startsWith(userFilesPattern));
288288
289289 const result = userSnapshots.map(x => {
@@ -300,7 +300,7 @@ router.post('/get-snapshots', async (request, response) => {
300300
301301router.post('/load-snapshot', getFileNameValidationFunction('name'), async (request, response) => {
302302 try {
303303 const userFilesPattern = getFilePrefixgetSettingsBackupFilePrefix(request.user.profile.handle);
304304
305305 if (!request.body.name || !request.body.name.startsWith(userFilesPattern)) {
306306 return response.status(400).send({ error: 'Invalid snapshot name' });
@@ -334,7 +334,7 @@ router.post('/make-snapshot', async (request, response) => {
334334
335335router.post('/restore-snapshot', getFileNameValidationFunction('name'), async (request, response) => {
336336 try {
337337 const userFilesPattern = getFilePrefixgetSettingsBackupFilePrefix(request.user.profile.handle);
338338
339339 if (!request.body.name || !request.body.name.startsWith(userFilesPattern)) {
340340 return response.status(400).send({ error: 'Invalid snapshot name' });
src/fetch-patch.js+1 -51
@@ -3,6 +3,7 @@ import path from 'node:path';
33import { fileURLToPath } from 'node:url';
44import mime from 'mime-types';
55import { serverDirectory } from './server-directory.js';
6+import { getRequestURL, isFileURL, isPathUnderParent } from './util.js';
67
78const originalFetch = globalThis.fetch;
89
@@ -10,57 +11,6 @@ const ALLOWED_EXTENSIONS = [
1011 '.wasm',
1112];
1213
13-/**
14- * Checks if a child path is under a parent path.
15- * @param {string} parentPath Parent path
16- * @param {string} childPath Child path
17- * @returns {boolean} Returns true if the child path is under the parent path, false otherwise
18- */
19-function isPathUnderParent(parentPath, childPath) {
20- const normalizedParent = path.normalize(parentPath);
21- const normalizedChild = path.normalize(childPath);
22-
23- const relativePath = path.relative(normalizedParent, normalizedChild);
24-
25- return !relativePath.startsWith('..') && !path.isAbsolute(relativePath);
26-}
27-
28-/**
29- * Checks if the given request is a file URL.
30- * @param {string | URL | Request} request The request to check
31- * @return {boolean} Returns true if the request is a file URL, false otherwise
32- */
33-function isFileURL(request) {
34- if (typeof request === 'string') {
35- return request.startsWith('file://');
36- }
37- if (request instanceof URL) {
38- return request.protocol === 'file:';
39- }
40- if (request instanceof Request) {
41- return request.url.startsWith('file://');
42- }
43- return false;
44-}
45-
46-/**
47- * Gets the URL from the request.
48- * @param {string | URL | Request} request The request to get the URL from
49- * @return {string} The URL of the request
50- */
51-function getRequestURL(request) {
52- if (typeof request === 'string') {
53- return request;
54- }
55- if (request instanceof URL) {
56- return request.href;
57- }
58- if (request instanceof Request) {
59- return request.url;
60- }
61- throw new TypeError('Invalid request type');
62-}
63-
6414// Patched fetch function that handles file URLs
6515globalThis.fetch = async (/** @type {string | URL | Request} */ request, /** @type {RequestInit | undefined} */ options) => {
6616 if (!isFileURL(request)) {
src/server-startup.js+2 -0
@@ -46,6 +46,7 @@ import { router as textCompletionsRouter } from './endpoints/backends/text-compl
4646import { router as scaleAltRouter } from './endpoints/backends/scale-alt.js';
4747import { router as speechRouter } from './endpoints/speech.js';
4848import { router as azureRouter } from './endpoints/azure.js';
49+import { router as dataMaidRouter } from './endpoints/data-maid.js';
4950
5051/**
5152 * @typedef {object} ServerStartupResult
@@ -173,6 +174,7 @@ export function setupPrivateEndpoints(app) {
173174 app.use('/api/backends/scale-alt', scaleAltRouter);
174175 app.use('/api/speech', speechRouter);
175176 app.use('/api/azure', azureRouter);
177+ app.use('/api/data-maid', dataMaidRouter);
176178}
177179
178180/**
src/util.js+51 -0
@@ -1146,3 +1146,54 @@ export function setPermissionsSync(targetPath) {
11461146 console.error(`Error setting write permissions for ${targetPath}:`, error);
11471147 }
11481148}
1149+
1150+/**
1151+ * Checks if a child path is under a parent path.
1152+ * @param {string} parentPath Parent path
1153+ * @param {string} childPath Child path
1154+ * @returns {boolean} Returns true if the child path is under the parent path, false otherwise
1155+ */
1156+export function isPathUnderParent(parentPath, childPath) {
1157+ const normalizedParent = path.normalize(parentPath);
1158+ const normalizedChild = path.normalize(childPath);
1159+
1160+ const relativePath = path.relative(normalizedParent, normalizedChild);
1161+
1162+ return !relativePath.startsWith('..') && !path.isAbsolute(relativePath);
1163+}
1164+
1165+/**
1166+ * Checks if the given request is a file URL.
1167+ * @param {string | URL | Request} request The request to check
1168+ * @return {boolean} Returns true if the request is a file URL, false otherwise
1169+ */
1170+export function isFileURL(request) {
1171+ if (typeof request === 'string') {
1172+ return request.startsWith('file://');
1173+ }
1174+ if (request instanceof URL) {
1175+ return request.protocol === 'file:';
1176+ }
1177+ if (request instanceof Request) {
1178+ return request.url.startsWith('file://');
1179+ }
1180+ return false;
1181+}
1182+
1183+/**
1184+ * Gets the URL from the request.
1185+ * @param {string | URL | Request} request The request to get the URL from
1186+ * @return {string} The URL of the request
1187+ */
1188+export function getRequestURL(request) {
1189+ if (typeof request === 'string') {
1190+ return request;
1191+ }
1192+ if (request instanceof URL) {
1193+ return request.href;
1194+ }
1195+ if (request instanceof Request) {
1196+ return request.url;
1197+ }
1198+ throw new TypeError('Invalid request type');
1199+}