Implement data clean-up dialog (#4072) * [wip] Add user data cleanup service * Add clean-up report viewer * Fix review comments * Add function comments * Implement item actions * Fix UI styles * Add placeholder for empty results, update category description view * Add displayEmptyPlaceholder method to show message when results list is empty * Adjust menu buttons row * Delete char-scoped data bank attachments on character deletion * Data Bank: Handle character attachments on rename * Remove line breaks in description strings * Drop the category when the last item is deleted * Skip invalid hashes instead of bailing

3ec9b1a09994e33d663e47e425339df39235c973

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: 7f47f84c9f7ec29ba245aa95d96c25c264eb80b9
15 files changed, +1475 -62Ignore whitespace
public/css/data-maid.css+149 -0
@@ -0,0 +1,149 @@
1.dataMaidDialogContainer {
2 height: 100%;
3 overflow: hidden;
4}
5
6.dataMaidDialog {
7 display: flex;
8 flex-direction: column;
9 gap: 5px;
10 height: 100%;
11 overflow: hidden;
12}
13
14.dataMaidDialogHeader {
15 display: flex;
16 gap: 10px;
17 align-items: center;
18 text-align: left;
19}
20
21.dataMaidHeaderInfo {
22 flex: 1;
23 margin: 0;
24 padding: 5px 10px;
25}
26
27.dataMaidTextView {
28 width: 100%;
29 height: 100%;
30 font-family: var(--monoFontFamily);
31 resize: none;
32 font-size: 0.95em;
33}
34
35.dataMaidImageView {
36 width: 100%;
37 height: 100%;
38 object-fit: contain;
39}
40
41.dataMaidSpinner {
42 display: flex;
43 justify-content: center;
44 align-items: center;
45 height: 100%;
46}
47
48.dataMaidPlaceholder {
49 display: flex;
50 justify-content: center;
51 align-items: center;
52 font-size: 1.05em;
53}
54
55.dataMaidResultsList:empty {
56 display: none;
57}
58
59.dataMaidResultsList {
60 text-align: left;
61 display: flex;
62 flex-direction: column;
63 gap: 2px;
64 overflow-y: auto;
65 height: 100%;
66 flex-grow: 1;
67}
68
69.dataMaidCategory {
70 border: 1px solid var(--SmartThemeBorderColor);
71 border-radius: 10px;
72 padding: 0 10px;
73}
74
75.dataMaidCategoryHeader {
76 width: 100%;
77 display: flex;
78 align-items: center;
79 justify-content: space-between;
80 margin-right: 5px;
81 padding: 0 5px;
82}
83
84.dataMaidCategoryDetails {
85 display: flex;
86 flex-direction: column;
87 gap: 1px;
88}
89
90.dataMaidCategoryName {
91 flex: 3;
92 font-weight: bold;
93 font-size: 1.1em;
94}
95
96.dataMaidCategoryInfo {
97 flex: 1;
98 display: flex;
99 align-items: baseline;
100 gap: 5px;
101}
102
103.dataMaidCategoryContent {
104 border: 1px solid var(--SmartThemeBorderColor);
105 padding: 5px;
106 border-radius: 10px;
107 background-color: var(--black30a);
108 margin: 10px 0;
109}
110
111.dataMaidCategoryContent>.info-block {
112 white-space: pre-wrap;
113}
114
115.dataMaidItem {
116 display: flex;
117 flex-direction: column;
118 padding: 5px;
119 width: 100%;
120 border-bottom: 1px solid var(--SmartThemeBorderColor);
121}
122
123.dataMaidItem:last-child {
124 border-bottom: none;
125}
126
127.dataMaidItemHeader {
128 display: flex;
129 align-items: center;
130 gap: 5px;
131}
132
133.dataMaidItemName {
134 display: flex;
135 flex: 1;
136 align-items: baseline;
137 gap: 2px;
138 word-break: break-all;
139}
140
141.dataMaidItemActions {
142 display: flex;
143 align-items: center;
144 gap: 5px;
145}
146
147.dataMaidItemActions>button {
148 font-size: 0.9em;
149}
public/css/st-tailwind.css+4 -0
@@ -556,6 +556,10 @@ textarea:disabled {
556 min-width: fit-content;556 min-width: fit-content;
557}557}
558558
559.flexGap2 {
560 gap: 2px;
561}
562
559.flexGap5 {563.flexGap5 {
560 gap: 5px;564 gap: 5px;
561}565}
public/index.html+4 -1
@@ -4738,13 +4738,16 @@
47384738
4739 <div name="MiscellaneousToggles">4739 <div name="MiscellaneousToggles">
4740 <h4><span data-i18n="Miscellaneous">Miscellaneous</span></h4>4740 <h4><span data-i18n="Miscellaneous">Miscellaneous</span></h4>
4741 <div class="flex-container">4741 <div class="flex-container flexGap2">
4742 <div id="reload_chat" class="menu_button whitespacenowrap" data-i18n="[title]Reload and redraw the currently open chat" title="Reload and redraw the currently open chat.">4742 <div id="reload_chat" class="menu_button whitespacenowrap" data-i18n="[title]Reload and redraw the currently open chat" title="Reload and redraw the currently open chat.">
4743 <small data-i18n="Reload Chat">Reload Chat</small>4743 <small data-i18n="Reload Chat">Reload Chat</small>
4744 </div>4744 </div>
4745 <div id="debug_menu" class="menu_button whitespacenowrap">4745 <div id="debug_menu" class="menu_button whitespacenowrap">
4746 <small data-i18n="Debug Menu">Debug Menu</small>4746 <small data-i18n="Debug Menu">Debug Menu</small>
4747 </div>4747 </div>
4748 <div id="data_maid_button" class="menu_button whitespacenowrap" title="Find and delete backups, unused chats, files, images, etc." data-i18n="[title]Find and delete backups, unused chats, files, images, etc.">
4749 <small data-i18n="Clean-Up">Clean-Up</small>
4750 </div>
4748 </div>4751 </div>
4749 <label class="checkbox_label flexWrap" for="smooth_streaming">4752 <label class="checkbox_label flexWrap" for="smooth_streaming">
4750 <input id="smooth_streaming" type="checkbox" />4753 <input id="smooth_streaming" type="checkbox" />
public/script.js+2 -0
@@ -283,6 +283,7 @@ import { getContext } from './scripts/st-context.js';
283import { extractReasoningFromData, initReasoning, parseReasoningInSwipes, PromptReasoning, ReasoningHandler, removeReasoningFromString, updateReasoningUI } from './scripts/reasoning.js';283import { extractReasoningFromData, initReasoning, parseReasoningInSwipes, PromptReasoning, ReasoningHandler, removeReasoningFromString, updateReasoningUI } from './scripts/reasoning.js';
284import { accountStorage } from './scripts/util/AccountStorage.js';284import { accountStorage } from './scripts/util/AccountStorage.js';
285import { initWelcomeScreen, openPermanentAssistantChat, openPermanentAssistantCard, getPermanentAssistantAvatar } from './scripts/welcome-screen.js';285import { initWelcomeScreen, openPermanentAssistantChat, openPermanentAssistantCard, getPermanentAssistantAvatar } from './scripts/welcome-screen.js';
286import { initDataMaid } from './scripts/data-maid.js';
286287
287// API OBJECT FOR EXTERNAL WIRING288// API OBJECT FOR EXTERNAL WIRING
288globalThis.SillyTavern = {289globalThis.SillyTavern = {
@@ -1026,6 +1027,7 @@ async function firstLoadInit() {
1026 initWelcomeScreen();1027 initWelcomeScreen();
1027 await initScrapers();1028 await initScrapers();
1028 initCustomSelectedSamplers();1029 initCustomSelectedSamplers();
1030 initDataMaid();
1029 addDebugFunctions();1031 addDebugFunctions();
1030 doDailyExtensionUpdatesCheck();1032 doDailyExtensionUpdatesCheck();
1031 await hideLoader();1033 await hideLoader();
public/scripts/data-maid.js+393 -0
@@ -0,0 +1,393 @@
1import { getRequestHeaders } from '../script.js';
2import { t } from './i18n.js';
3import { callGenericPopup, Popup, POPUP_TYPE } from './popup.js';
4import { renderTemplateAsync } from './templates.js';
5import { humanFileSize, timestampToMoment } from './utils.js';
6
7/**
8 * @typedef {object} DataMaidReportResult
9 * @property {import('../../src/endpoints/data-maid.js').DataMaidSanitizedReport} report - The sanitized report of the Data Maid.
10 * @property {string} token - The token to use for the Data Maid report.
11 */
12
13/**
14 * Data Maid Dialog class for managing the cleanup dialog interface.
15 */
16class DataMaidDialog {
17 constructor() {
18 this.token = null;
19 this.container = null;
20 this.isScanning = false;
21
22 this.DATA_MAID_CATEGORIES = {
23 files: {
24 name: t`Files`,
25 description: t`Files that are not associated with chat messages or Data Bank. WILL DELETE MANUAL UPLOADS!`,
26 },
27 images: {
28 name: t`Images`,
29 description: t`Images that are not associated with chat messages. WILL DELETE MANUAL UPLOADS!`,
30 },
31 chats: {
32 name: t`Chats`,
33 description: t`Chat files associated with deleted characters.`,
34 },
35 groupChats: {
36 name: t`Group Chats`,
37 description: t`Chat files associated with deleted groups.`,
38 },
39 avatarThumbnails: {
40 name: t`Avatar Thumbnails`,
41 description: t`Thumbnails for avatars of missing or deleted characters.`,
42 },
43 backgroundThumbnails: {
44 name: t`Background Thumbnails`,
45 description: t`Thumbnails for missing or deleted backgrounds.`,
46 },
47 chatBackups: {
48 name: t`Chat Backups`,
49 description: t`Automatically generated chat backups.`,
50 },
51 settingsBackups: {
52 name: t`Settings Backups`,
53 description: t`Automatically generated settings backups.`,
54 },
55 };
56 }
57
58 /**
59 * Returns a promise that resolves to the Data Maid report.
60 * @returns {Promise<DataMaidReportResult>}
61 * @private
62 */
63 async getReport() {
64 const response = await fetch('/api/data-maid/report', {
65 method: 'POST',
66 headers: getRequestHeaders(),
67 });
68
69 if (!response.ok) {
70 throw new Error(`Error fetching Data Maid report: ${response.statusText}`);
71 }
72
73 return await response.json();
74 }
75
76 /**
77 * Finalizes the Data Maid process by sending a request to the server.
78 * @returns {Promise<void>}
79 * @private
80 */
81 async finalize() {
82 const response = await fetch('/api/data-maid/finalize', {
83 method: 'POST',
84 headers: getRequestHeaders(),
85 body: JSON.stringify({ token: this.token }),
86 });
87
88 if (!response.ok) {
89 throw new Error(`Error finalizing Data Maid: ${response.statusText}`);
90 }
91 }
92
93 /**
94 * Sets up the dialog UI elements and event listeners.
95 * @private
96 */
97 async setupDialogUI() {
98 const template = await renderTemplateAsync('dataMaidDialog');
99 this.container = document.createElement('div');
100 this.container.classList.add('dataMaidDialogContainer');
101 this.container.innerHTML = template;
102
103 const startButton = this.container.querySelector('.dataMaidStartButton');
104 startButton.addEventListener('click', () => this.handleScanClick());
105 }
106
107 /**
108 * Handles the scan button click event.
109 * @private
110 */
111 async handleScanClick() {
112 if (this.isScanning) {
113 toastr.warning(t`The scan is already running. Please wait for it to finish.`);
114 return;
115 }
116
117 try {
118 const resultsList = this.container.querySelector('.dataMaidResultsList');
119 resultsList.innerHTML = '';
120 this.showSpinner();
121 this.isScanning = true;
122
123 const report = await this.getReport();
124
125 this.hideSpinner();
126 await this.renderReport(report, resultsList);
127 this.token = report.token;
128 } catch (error) {
129 this.hideSpinner();
130 toastr.error(t`An error has occurred. Check the console for details.`);
131 console.error('Error generating Data Maid report:', error);
132 } finally {
133 this.isScanning = false;
134 }
135 }
136
137 /**
138 * Shows the loading spinner and hides the placeholder.
139 * @private
140 */
141 showSpinner() {
142 const spinner = this.container.querySelector('.dataMaidSpinner');
143 const placeholder = this.container.querySelector('.dataMaidPlaceholder');
144 placeholder.classList.add('displayNone');
145 spinner.classList.remove('displayNone');
146 }
147
148 /**
149 * Hides the loading spinner.
150 * @private
151 */
152 hideSpinner() {
153 const spinner = this.container.querySelector('.dataMaidSpinner');
154 spinner.classList.add('displayNone');
155 }
156
157 /**
158 * Renders the Data Maid report into the results list.
159 * @param {DataMaidReportResult} report
160 * @param {Element} resultsList
161 * @private
162 */
163 async renderReport(report, resultsList) {
164 for (const [prop, data] of Object.entries(this.DATA_MAID_CATEGORIES)) {
165 const category = await this.renderCategory(prop, data.name, data.description, report.report[prop]);
166 if (!category) {
167 continue;
168 }
169 resultsList.appendChild(category);
170 }
171 this.displayEmptyPlaceholder();
172 }
173
174 /**
175 * Displays a placeholder message if no items are found in the results list.
176 * @private
177 */
178 displayEmptyPlaceholder() {
179 const resultsList = this.container.querySelector('.dataMaidResultsList');
180 if (resultsList.children.length === 0) {
181 const placeholder = this.container.querySelector('.dataMaidPlaceholder');
182 placeholder.classList.remove('displayNone');
183 placeholder.textContent = t`No items found to clean up. Come back later!`;
184 }
185 }
186
187 /**
188 * Renders a single Data Maid category into a DOM element.
189 * @param {string} prop Property name for the category
190 * @param {string} name Name of the category
191 * @param {string} description Description of the category
192 * @param {import('../../src/endpoints/data-maid.js').DataMaidSanitizedRecord[]} items List of items in the category
193 * @return {Promise<Element|null>} A promise that resolves to a DOM element containing the rendered category
194 * @private
195 */
196 async renderCategory(prop, name, description, items) {
197 if (!Array.isArray(items) || items.length === 0) {
198 return null;
199 }
200
201 const viewModel = {
202 name: name,
203 description: description,
204 totalSize: humanFileSize(items.reduce((sum, item) => sum + item.size, 0)),
205 totalItems: items.length,
206 items: items.sort((a, b) => b.mtime - a.mtime).map(item => ({
207 ...item,
208 size: humanFileSize(item.size),
209 date: timestampToMoment(item.mtime).format('L LT'),
210 })),
211 };
212
213 const template = await renderTemplateAsync('dataMaidCategory', viewModel);
214 const categoryElement = document.createElement('div');
215 categoryElement.innerHTML = template;
216 categoryElement.querySelectorAll('.dataMaidItemView').forEach(button => {
217 button.addEventListener('click', async () => {
218 const item = button.closest('.dataMaidItem');
219 const hash = item?.getAttribute('data-hash');
220 if (hash) {
221 await this.view(prop, hash);
222 }
223 });
224 });
225 categoryElement.querySelectorAll('.dataMaidItemDownload').forEach(button => {
226 button.addEventListener('click', async () => {
227 const item = button.closest('.dataMaidItem');
228 const hash = item?.getAttribute('data-hash');
229 if (hash) {
230 await this.download(items, hash);
231 }
232 });
233 });
234 categoryElement.querySelectorAll('.dataMaidDeleteAll').forEach(button => {
235 button.addEventListener('click', async (event) => {
236 event.stopPropagation();
237 const confirm = await Popup.show.confirm(t`Are you sure?`, t`This will permanently delete all files in this category. THIS CANNOT BE UNDONE!`);
238 if (!confirm) {
239 return;
240 }
241
242 const hashes = items.map(item => item.hash).filter(hash => hash);
243 await this.delete(hashes);
244
245 categoryElement.remove();
246 this.displayEmptyPlaceholder();
247 });
248
249 });
250 categoryElement.querySelectorAll('.dataMaidItemDelete').forEach(button => {
251 button.addEventListener('click', async () => {
252 const item = button.closest('.dataMaidItem');
253 const hash = item?.getAttribute('data-hash');
254 if (hash) {
255 const confirm = await Popup.show.confirm(t`Are you sure?`, t`This will permanently delete the file. THIS CANNOT BE UNDONE!`);
256 if (!confirm) {
257 return;
258 }
259 if (await this.delete([hash])) {
260 item.remove();
261 items.splice(items.findIndex(i => i.hash === hash), 1);
262 if (items.length === 0) {
263 categoryElement.remove();
264 this.displayEmptyPlaceholder();
265 }
266 }
267 }
268 });
269 });
270 return categoryElement;
271 }
272
273 /**
274 * Constructs the URL for viewing an item by its hash.
275 * @param {string} hash Hash of the item to view
276 * @returns {string} URL to view the item
277 * @private
278 */
279 getViewUrl(hash) {
280 return `/api/data-maid/view?hash=${encodeURIComponent(hash)}&token=${encodeURIComponent(this.token)}`;
281 }
282
283 /**
284 * Downloads an item by its hash.
285 * @param {import('../../src/endpoints/data-maid.js').DataMaidSanitizedRecord[]} items List of items in the category
286 * @param {string} hash Hash of the item to download
287 * @private
288 */
289 async download(items, hash) {
290 const item = items.find(i => i.hash === hash);
291 if (!item) {
292 return;
293 }
294 const url = this.getViewUrl(hash);
295 const a = document.createElement('a');
296 a.href = url;
297 a.download = item?.name || hash;
298 document.body.appendChild(a);
299 a.click();
300 document.body.removeChild(a);
301 }
302
303 /**
304 * Opens the item view for a specific hash.
305 * @param {string} prop Property name for the category
306 * @param {string} hash Item hash to view
307 * @private
308 */
309 async view(prop, hash) {
310 const url = this.getViewUrl(hash);
311 const isImage = ['images', 'avatarThumbnails', 'backgroundThumbnails'].includes(prop);
312 const element = isImage
313 ? await this.getViewElement(url)
314 : await this.getTextViewElement(url);
315 await callGenericPopup(element, POPUP_TYPE.DISPLAY, '', { large: true, wide: true });
316 }
317
318 /**
319 * Deletes an item by its file path hash.
320 * @param {string[]} hashes Hashes of items to delete
321 * @return {Promise<boolean>} True if the deletion was successful, false otherwise
322 * @private
323 */
324 async delete(hashes) {
325 try {
326 const response = await fetch('/api/data-maid/delete', {
327 method: 'POST',
328 headers: getRequestHeaders(),
329 body: JSON.stringify({ hashes: hashes, token: this.token }),
330 });
331
332 if (!response.ok) {
333 throw new Error(`Error deleting item: ${response.statusText}`);
334 }
335
336 return true;
337 } catch (error) {
338 console.error('Error deleting item:', error);
339 return false;
340 }
341 }
342
343 /**
344 * Gets an image element for viewing images.
345 * @param {string} url View URL
346 * @returns {Promise<HTMLElement>} Image element
347 * @private
348 */
349 async getViewElement(url) {
350 const img = document.createElement('img');
351 img.src = url;
352 img.classList.add('dataMaidImageView');
353 return img;
354 }
355
356 /**
357 * Gets an iframe element for viewing text content.
358 * @param {string} url View URL
359 * @returns {Promise<HTMLTextAreaElement>} Frame element
360 * @private
361 */
362 async getTextViewElement(url) {
363 const response = await fetch(url);
364 const text = await response.text();
365 const element = document.createElement('textarea');
366 element.classList.add('dataMaidTextView');
367 element.readOnly = true;
368 element.textContent = text;
369 return element;
370 }
371
372 /**
373 * Opens the Data Maid dialog and handles the interaction.
374 */
375 async open() {
376 await this.setupDialogUI();
377 await callGenericPopup(this.container, POPUP_TYPE.TEXT, '', { wide: true, large: true });
378
379 if (this.token) {
380 await this.finalize();
381 }
382 }
383}
384
385export function initDataMaid() {
386 const dataMaidButton = document.getElementById('data_maid_button');
387 if (!dataMaidButton) {
388 console.warn('Data Maid button not found');
389 return;
390 }
391
392 dataMaidButton.addEventListener('click', () => new DataMaidDialog().open());
393}
public/scripts/extensions/attachments/index.js+29 -0
@@ -216,8 +216,37 @@ function cleanUpAttachments() {
216 }216 }
217}217}
218218
219/**
220 * Clean up character attachments when a character is deleted.
221 * @param {{character: import('../../char-data.js').v1CharData}} data Event data
222 */
223function cleanUpCharacterAttachments(data) {
224 const avatar = data?.character?.avatar;
225 if (!avatar) return;
226 if (Array.isArray(extension_settings?.character_attachments?.[avatar])) {
227 delete extension_settings.character_attachments[avatar];
228 saveSettingsDebounced();
229 }
230}
231
232/**
233 * Handle character rename event to update character attachments.
234 * @param {string} oldAvatar Old avatar name
235 * @param {string} newAvatar New avatar name
236 */
237function handleCharacterRename(oldAvatar, newAvatar) {
238 if (!oldAvatar || !newAvatar) return;
239 if (Array.isArray(extension_settings?.character_attachments?.[oldAvatar])) {
240 extension_settings.character_attachments[newAvatar] = extension_settings.character_attachments[oldAvatar];
241 delete extension_settings.character_attachments[oldAvatar];
242 saveSettingsDebounced();
243 }
244}
245
219jQuery(async () => {246jQuery(async () => {
220 eventSource.on(event_types.APP_READY, cleanUpAttachments);247 eventSource.on(event_types.APP_READY, cleanUpAttachments);
248 eventSource.on(event_types.CHARACTER_DELETED, cleanUpCharacterAttachments);
249 eventSource.on(event_types.CHARACTER_RENAMED, handleCharacterRename);
221 const manageButton = await renderExtensionTemplateAsync('attachments', 'manage-button', {});250 const manageButton = await renderExtensionTemplateAsync('attachments', 'manage-button', {});
222 const attachButton = await renderExtensionTemplateAsync('attachments', 'attach-button', {});251 const attachButton = await renderExtensionTemplateAsync('attachments', 'attach-button', {});
223 $('#data_bank_wand_container').append(manageButton);252 $('#data_bank_wand_container').append(manageButton);
public/scripts/templates/dataMaidCategory.html+68 -0
@@ -0,0 +1,68 @@
1<div class="dataMaidCategory inline-drawer">
2 <div class="inline-drawer-toggle inline-drawer-header">
3 <div class="dataMaidCategoryHeader">
4 <div class="dataMaidCategoryDetails">
5 <div class="dataMaidCategoryName" data-i18n="{{name}}">
6 {{name}}
7 </div>
8 <small>{{description}}</small>
9 <div class="dataMaidCategoryInfo">
10 <small>
11 <i class="fa-solid fa-file-alt fa-sm"></i>
12 {{totalItems}}
13 </small>
14 <span>&VerticalBar;</span>
15 <small>
16 <i class="fa-solid fa-hdd fa-sm"></i>
17 {{totalSize}}
18 </small>
19 </div>
20 </div>
21 <div class="dataMaidDeleteAll right_menu_button" title="Delete all items in this category" data-i18n="[title]Delete all items in this category">
22 <i class="fa-solid fa-fw fa-broom"></i>
23 </div>
24 </div>
25 <div class="fa-solid fa-circle-chevron-down inline-drawer-icon down"></div>
26 </div>
27 <div class="dataMaidCategoryContent inline-drawer-content">
28 <div class="flex-container">
29 {{#each items}}
30 {{#with this}}
31 <div class="dataMaidItem" data-hash="{{hash}}">
32 <div class="dataMaidItemHeader">
33 <div class="dataMaidItemName">
34 {{#if parent}}
35 <span class="dataMaidItemParent">({{parent}})</span>
36 <span>/</span>
37 {{/if}}
38 <b>{{name}}</b>
39 </div>
40 <div class="dataMaidItemActions">
41 <button class="dataMaidItemView menu_button menu_button_icon margin0" title="View item content" data-i18n="[title]View item content">
42 <i class="fa-solid fa-fw fa-eye"></i>
43 </button>
44 <button class="dataMaidItemDownload menu_button menu_button_icon margin0" title="Download item" data-i18n="[title]Download item">
45 <i class="fa-solid fa-fw fa-download"></i>
46 </button>
47 <button class="dataMaidItemDelete menu_button menu_button_icon margin0" title="Delete this item" data-i18n="[title]Delete this item">
48 <i class="fa-solid fa-fw fa-trash-alt"></i>
49 </button>
50 </div>
51 </div>
52 <div class="dataMaidItemInfo">
53 <small>
54 <i class="fa-solid fa-file fa-sm"></i>
55 {{size}}
56 </small>
57 <span>&VerticalBar;</span>
58 <small>
59 <i class="fa-solid fa-calendar fa-sm"></i>
60 {{date}}
61 </small>
62 </div>
63 </div>
64 {{/with}}
65 {{/each}}
66 </div>
67 </div>
68</div>
public/scripts/templates/dataMaidDialog.html+25 -0
@@ -0,0 +1,25 @@
1<div class="dataMaidDialog">
2 <div class="dataMaidDialogHeader">
3 <div class="dataMaidHeaderInfo info-block warning margin0">
4 <small data-i18n="Once deleted, the files will be gone forever!">
5 Once deleted, the files will be gone forever!
6 </small>
7 <br>
8 <small data-i18n="Make sure to back up your data in advance.">
9 Make sure to back up your data in advance.
10 </small>
11 </div>
12 <button class="menu_button menu_button_icon dataMaidStartButton">
13 <i class="fa fa-cog"></i>
14 <span data-i18n="Scan">Scan</span>
15 </button>
16 </div>
17 <hr>
18 <div class="dataMaidPlaceholder" data-i18n="No results yet. Tap 'Scan' to start scanning.">
19 No results yet. Tap 'Scan' to start scanning.
20 </div>
21 <div class="displayNone dataMaidSpinner">
22 <i class="fa-solid fa-spinner fa-spin fa-3x"></i>
23 </div>
24 <div class="dataMaidResultsList"></div>
25</div>
public/style.css+1 -0
@@ -11,6 +11,7 @@
11@import url(css/tags.css);11@import url(css/tags.css);
12@import url(css/scrollable-button.css);12@import url(css/scrollable-button.css);
13@import url(css/welcome.css);13@import url(css/welcome.css);
14@import url(css/data-maid.css);
1415
15:root {16:root {
16 --doc-height: 100%;17 --doc-height: 100%;
src/endpoints/chats.js+5 -4
@@ -23,6 +23,8 @@ const maxTotalChatBackups = Number(getConfigValue('backups.chat.maxTotalBackups'
23const throttleInterval = Number(getConfigValue('backups.chat.throttleInterval', 10_000, 'number'));23const throttleInterval = Number(getConfigValue('backups.chat.throttleInterval', 10_000, 'number'));
24const checkIntegrity = !!getConfigValue('backups.chat.checkIntegrity', true, 'boolean');24const checkIntegrity = !!getConfigValue('backups.chat.checkIntegrity', true, 'boolean');
2525
26export const CHAT_BACKUPS_PREFIX = 'chat_';
27
26/**28/**
27 * Saves a chat to the backups directory.29 * Saves a chat to the backups directory.
28 * @param {string} directory The user's backups directory.30 * @param {string} directory The user's backups directory.
@@ -31,7 +33,6 @@ const checkIntegrity = !!getConfigValue('backups.chat.checkIntegrity', true, 'bo
31 */33 */
32function backupChat(directory, name, chat) {34function backupChat(directory, name, chat) {
33 try {35 try {
34
35 if (!isBackupEnabled) {36 if (!isBackupEnabled) {
36 return;37 return;
37 }38 }
@@ -39,16 +40,16 @@ function backupChat(directory, name, chat) {
39 // replace non-alphanumeric characters with underscores40 // replace non-alphanumeric characters with underscores
40 name = sanitize(name).replace(/[^a-z0-9]/gi, '_').toLowerCase();41 name = sanitize(name).replace(/[^a-z0-9]/gi, '_').toLowerCase();
4142
42 const backupFile = path.join(directory, `chat_${name}_${generateTimestamp()}.jsonl`);43 const backupFile = path.join(directory, `${CHAT_BACKUPS_PREFIX}${name}_${generateTimestamp()}.jsonl`);
43 writeFileAtomicSync(backupFile, chat, 'utf-8');44 writeFileAtomicSync(backupFile, chat, 'utf-8');
4445
45 removeOldBackups(directory, `chat_${name}_`);46 removeOldBackups(directory, `${CHAT_BACKUPS_PREFIX}${name}_`);
4647
47 if (isNaN(maxTotalChatBackups) || maxTotalChatBackups < 0) {48 if (isNaN(maxTotalChatBackups) || maxTotalChatBackups < 0) {
48 return;49 return;
49 }50 }
5051
51 removeOldBackups(directory, 'chat_', maxTotalChatBackups);52 removeOldBackups(directory, CHAT_BACKUPS_PREFIX, maxTotalChatBackups);
52 } catch (err) {53 } catch (err) {
53 console.error(`Could not backup chat for ${name}`, err);54 console.error(`Could not backup chat for ${name}`, err);
54 }55 }
src/endpoints/data-maid.js+735 -0
@@ -0,0 +1,735 @@
1import crypto from 'node:crypto';
2import fs from 'node:fs';
3import path from 'node:path';
4import express from 'express';
5import mime from 'mime-types';
6import { getSettingsBackupFilePrefix } from './settings.js';
7import { CHAT_BACKUPS_PREFIX } from './chats.js';
8import { isPathUnderParent, tryParse } from '../util.js';
9import { SETTINGS_FILE } from '../constants.js';
10
11const sha256 = str => crypto.createHash('sha256').update(str).digest('hex');
12
13/**
14 * @typedef {object} DataMaidRawReport
15 * @property {string[]} images - List of loose user images
16 * @property {string[]} files - List of loose user files
17 * @property {string[]} chats - List of loose character chats
18 * @property {string[]} groupChats - List of loose group chats
19 * @property {string[]} avatarThumbnails - List of loose avatar thumbnails
20 * @property {string[]} backgroundThumbnails - List of loose background thumbnails
21 * @property {string[]} chatBackups - List of chat backups
22 * @property {string[]} settingsBackups - List of settings backups
23 */
24
25/**
26 * @typedef {object} DataMaidSanitizedRecord - The entry excluding the sensitive paths.
27 * @property {string} name - The name of the file.
28 * @property {string} hash - The SHA-256 hash of the file path.
29 * @property {string} [parent] - The name of the parent directory, if applicable.
30 * @property {number} [size] - The size of the file in bytes, if available.
31 * @property {number} [mtime] - The last modification time of the file, if available.
32 */
33
34/**
35 * @typedef {object} DataMaidSanitizedReport - The report containing loose user data.
36 * @property {DataMaidSanitizedRecord[]} images - List of sanitized loose user images
37 * @property {DataMaidSanitizedRecord[]} files - List of sanitized loose user files
38 * @property {DataMaidSanitizedRecord[]} chats - List of sanitized loose character chats
39 * @property {DataMaidSanitizedRecord[]} groupChats - List of sanitized loose group chats
40 * @property {DataMaidSanitizedRecord[]} avatarThumbnails - List of sanitized loose avatar thumbnails
41 * @property {DataMaidSanitizedRecord[]} backgroundThumbnails - List of sanitized loose background thumbnails
42 * @property {DataMaidSanitizedRecord[]} chatBackups - List of sanitized chat backups
43 * @property {DataMaidSanitizedRecord[]} settingsBackups - List of sanitized settings backups
44 */
45
46/**
47 * @typedef {object} DataMaidMessage - The chat message object.
48 * @property {DataMaidMessageExtra} [extra] - The extra data object.
49 * @property {DataMaidChatMetadata} [chat_metadata] - The chat metadata object.
50 */
51
52/**
53 * @typedef {object} DataMaidFile - The file object.
54 * @property {string} url - The file URL
55 */
56
57/**
58 * @typedef {object} DataMaidChatMetadata - The chat metadata object.
59 * @property {DataMaidFile[]} [attachments] - The array of attachments, if any.
60 */
61
62/**
63 * @typedef {object} DataMaidMessageExtra - The extra data object.
64 * @property {string} [image] - The link to the image, if any.
65 * @property {string[]} [image_swipes] - The links to the image swipes, if any.
66 * @property {DataMaidFile} [file] - The file object, if any.
67 */
68
69/**
70 * @typedef {object} DataMaidTokenEntry
71 * @property {string} handle - The user's handle or identifier.
72 * @property {{path: string, hash: string}[]} paths - The list of file paths and their hashes that can be cleaned up.
73 */
74
75/**
76 * Service for detecting and managing loose user data files.
77 * Helps identify orphaned files that are no longer referenced by the application.
78 */
79export class DataMaidService {
80 /**
81 * @type {Map<string, DataMaidTokenEntry>} Map clean-up tokens to user IDs
82 */
83 static TOKENS = new Map();
84
85 /**
86 * Creates a new DataMaidService instance for a specific user.
87 * @param {string} handle - The user's handle.
88 * @param {import('../users.js').UserDirectoryList} directories - List of user directories to scan for loose data.
89 */
90 constructor(handle, directories) {
91 this.handle = handle;
92 this.directories = directories;
93 }
94
95 /**
96 * Generates a report of loose user data.
97 * @returns {Promise<DataMaidRawReport>} A report containing lists of loose user data.
98 */
99 async generateReport() {
100 /** @type {DataMaidRawReport} */
101 const report = {
102 images: await this.#collectImages(),
103 files: await this.#collectFiles(),
104 chats: await this.#collectChats(),
105 groupChats: await this.#collectGroupChats(),
106 avatarThumbnails: await this.#collectAvatarThumbnails(),
107 backgroundThumbnails: await this.#collectBackgroundThumbnails(),
108 chatBackups: await this.#collectChatBackups(),
109 settingsBackups: await this.#collectSettingsBackups(),
110 };
111
112 return report;
113 }
114
115
116 /**
117 * Sanitizes a record by hashing the file name and removing sensitive information.
118 * Additionally, adds metadata like size and modification time.
119 * @param {string} name The file or directory name to sanitize.
120 * @param {boolean} withParent If the model should include the parent directory name.
121 * @returns {Promise<DataMaidSanitizedRecord>} A sanitized record with the file name, hash, parent directory name, size, and modification time.
122 */
123 async #sanitizeRecord(name, withParent) {
124 const stat = fs.existsSync(name) ? await fs.promises.stat(name) : null;
125 return {
126 name: path.basename(name),
127 hash: sha256(name),
128 parent: withParent ? path.basename(path.dirname(name)) : void 0,
129 size: stat?.size,
130 mtime: stat?.mtimeMs,
131 };
132 }
133
134 /**
135 * Sanitizes the report by hashing the file paths and removing sensitive information.
136 * @param {DataMaidRawReport} report - The raw report containing loose user data.
137 * @returns {Promise<DataMaidSanitizedReport>} A sanitized report with sensitive paths removed.
138 */
139 async sanitizeReport(report) {
140 const sanitizedReport = {
141 images: await Promise.all(report.images.map(i => this.#sanitizeRecord(i, true))),
142 files: await Promise.all(report.files.map(i => this.#sanitizeRecord(i, false))),
143 chats: await Promise.all(report.chats.map(i => this.#sanitizeRecord(i, true))),
144 groupChats: await Promise.all(report.groupChats.map(i => this.#sanitizeRecord(i, false))),
145 avatarThumbnails: await Promise.all(report.avatarThumbnails.map(i => this.#sanitizeRecord(i, false))),
146 backgroundThumbnails: await Promise.all(report.backgroundThumbnails.map(i => this.#sanitizeRecord(i, false))),
147 chatBackups: await Promise.all(report.chatBackups.map(i => this.#sanitizeRecord(i, false))),
148 settingsBackups: await Promise.all(report.settingsBackups.map(i => this.#sanitizeRecord(i, false))),
149 };
150
151 return sanitizedReport;
152 }
153
154 /**
155 * Collects loose user images from the provided directories.
156 * Images are considered loose if they exist in the user images directory
157 * but are not referenced in any chat messages.
158 * @returns {Promise<string[]>} List of paths to loose user images
159 */
160 async #collectImages() {
161 const result = [];
162
163 try {
164 const messages = await this.#parseAllChats(x => !!x?.extra?.image || Array.isArray(x?.extra?.image_swipes));
165 const knownImages = new Set();
166 for (const message of messages) {
167 if (message?.extra?.image) {
168 knownImages.add(message.extra.image);
169 }
170 if (Array.isArray(message?.extra?.image_swipes)) {
171 for (const swipe of message.extra.image_swipes) {
172 knownImages.add(swipe);
173 }
174 }
175 }
176 const knownImageFullPaths = new Set();
177 knownImages.forEach(image => {
178 if (image.startsWith('http') || image.startsWith('data:')) {
179 return; // Skip URLs and data URIs
180 }
181 knownImageFullPaths.add(path.normalize(path.join(this.directories.root, image)));
182 });
183 const images = await fs.promises.readdir(this.directories.userImages, { withFileTypes: true });
184 for (const dirent of images) {
185 const direntPath = path.join(dirent.parentPath, dirent.name);
186 if (dirent.isFile() && !knownImageFullPaths.has(direntPath)) {
187 result.push(direntPath);
188 }
189 if (dirent.isDirectory()) {
190 const subdirFiles = await fs.promises.readdir(direntPath, { withFileTypes: true });
191 for (const file of subdirFiles) {
192 const subdirFilePath = path.join(direntPath, file.name);
193 if (file.isFile() && !knownImageFullPaths.has(subdirFilePath)) {
194 result.push(subdirFilePath);
195 }
196 }
197 }
198 }
199 } catch (error) {
200 console.error('[Data Maid] Error collecting user images:', error);
201 }
202
203 return result;
204 }
205
206 /**
207 * Collects loose user files from the provided directories.
208 * Files are considered loose if they exist in the files directory
209 * but are not referenced in chat messages, metadata, or settings.
210 * @returns {Promise<string[]>} List of paths to loose user files
211 */
212 async #collectFiles() {
213 const result = [];
214
215 try {
216 const messages = await this.#parseAllChats(x => !!x?.extra?.file?.url);
217 const knownFiles = new Set();
218 for (const message of messages) {
219 if (message?.extra?.file?.url) {
220 knownFiles.add(message.extra.file.url);
221 }
222 }
223 const metadata = await this.#parseAllMetadata(x => Array.isArray(x?.attachments) && x.attachments.length > 0);
224 for (const meta of metadata) {
225 if (Array.isArray(meta?.attachments)) {
226 for (const attachment of meta.attachments) {
227 if (attachment?.url) {
228 knownFiles.add(attachment.url);
229 }
230 }
231 }
232 }
233 const pathToSettings = path.join(this.directories.root, SETTINGS_FILE);
234 if (fs.existsSync(pathToSettings)) {
235 try {
236 const settingsContent = await fs.promises.readFile(pathToSettings, 'utf-8');
237 const settings = tryParse(settingsContent);
238 if (Array.isArray(settings?.extension_settings?.attachments)) {
239 for (const file of settings.extension_settings.attachments) {
240 if (file?.url) {
241 knownFiles.add(file.url);
242 }
243 }
244 }
245 if (typeof settings?.extension_settings?.character_attachments === 'object') {
246 for (const files of Object.values(settings.extension_settings.character_attachments)) {
247 if (!Array.isArray(files)) {
248 continue;
249 }
250 for (const file of files) {
251 if (file?.url) {
252 knownFiles.add(file.url);
253 }
254 }
255 }
256 }
257 } catch (error) {
258 console.error('[Data Maid] Error reading settings file:', error);
259 }
260 }
261 const knownFileFullPaths = new Set();
262 knownFiles.forEach(file => {
263 knownFileFullPaths.add(path.normalize(path.join(this.directories.root, file)));
264 });
265 const files = await fs.promises.readdir(this.directories.files, { withFileTypes: true });
266 for (const file of files) {
267 const filePath = path.join(this.directories.files, file.name);
268 if (file.isFile() && !knownFileFullPaths.has(filePath)) {
269 result.push(filePath);
270 }
271 }
272 } catch (error) {
273 console.error('[Data Maid] Error collecting user files:', error);
274 }
275
276 return result;
277 }
278
279 /**
280 * Collects loose character chats from the provided directories.
281 * Chat folders are considered loose if they don't have corresponding character files.
282 * @returns {Promise<string[]>} List of paths to loose character chats
283 */
284 async #collectChats() {
285 const result = [];
286
287 try {
288 const knownChatFolders = new Set();
289 const characters = await fs.promises.readdir(this.directories.characters, { withFileTypes: true });
290 for (const file of characters) {
291 if (file.isFile() && path.parse(file.name).ext === '.png') {
292 knownChatFolders.add(file.name.replace('.png', ''));
293 }
294 }
295 const chatFolders = await fs.promises.readdir(this.directories.chats, { withFileTypes: true });
296 for (const folder of chatFolders) {
297 if (folder.isDirectory() && !knownChatFolders.has(folder.name)) {
298 const chatFiles = await fs.promises.readdir(path.join(this.directories.chats, folder.name), { withFileTypes: true });
299 for (const file of chatFiles) {
300 if (file.isFile() && path.parse(file.name).ext === '.jsonl') {
301 result.push(path.join(this.directories.chats, folder.name, file.name));
302 }
303 }
304 }
305 }
306 } catch (error) {
307 console.error('[Data Maid] Error collecting character chats:', error);
308 }
309
310 return result;
311 }
312
313 /**
314 * Collects loose group chats from the provided directories.
315 * Group chat files are considered loose if they're not referenced by any group definition.
316 * @returns {Promise<string[]>} List of paths to loose group chats
317 */
318 async #collectGroupChats() {
319 const result = [];
320
321 try {
322 const groups = await fs.promises.readdir(this.directories.groups, { withFileTypes: true });
323 const knownGroupChats = new Set();
324 for (const file of groups) {
325 if (file.isFile() && path.parse(file.name).ext === '.json') {
326 try {
327 const pathToFile = path.join(this.directories.groups, file.name);
328 const fileContent = await fs.promises.readFile(pathToFile, 'utf-8');
329 const groupData = tryParse(fileContent);
330 if (groupData?.chat_id) {
331 knownGroupChats.add(groupData.chat_id);
332 }
333 if (Array.isArray(groupData?.chats)) {
334 for (const chat of groupData.chats) {
335 knownGroupChats.add(chat);
336 }
337 }
338 } catch (error) {
339 console.error(`[Data Maid] Error parsing group chat file ${file.name}:`, error);
340 }
341 }
342 }
343 const groupChats = await fs.promises.readdir(this.directories.groupChats, { withFileTypes: true });
344 for (const file of groupChats) {
345 if (file.isFile() && path.parse(file.name).ext === '.jsonl') {
346 if (!knownGroupChats.has(path.parse(file.name).name)) {
347 result.push(path.join(this.directories.groupChats, file.name));
348 }
349 }
350 }
351 } catch (error) {
352 console.error('[Data Maid] Error collecting group chats:', error);
353 }
354
355 return result;
356 }
357
358 /**
359 * Collects loose avatar thumbnails from the provided directories.
360 * @returns {Promise<string[]>} List of paths to loose avatar thumbnails
361 */
362 async #collectAvatarThumbnails() {
363 const result = [];
364
365 try {
366 const knownAvatars = new Set();
367 const avatars = await fs.promises.readdir(this.directories.characters, { withFileTypes: true });
368 for (const file of avatars) {
369 if (file.isFile()) {
370 knownAvatars.add(file.name);
371 }
372 }
373 const avatarThumbnails = await fs.promises.readdir(this.directories.thumbnailsAvatar, { withFileTypes: true });
374 for (const file of avatarThumbnails) {
375 if (file.isFile() && !knownAvatars.has(file.name)) {
376 result.push(path.join(this.directories.thumbnailsAvatar, file.name));
377 }
378 }
379 } catch (error) {
380 console.error('[Data Maid] Error collecting avatar thumbnails:', error);
381 }
382
383 return result;
384 }
385
386 /**
387 * Collects loose background thumbnails from the provided directories.
388 * @returns {Promise<string[]>} List of paths to loose background thumbnails
389 */
390 async #collectBackgroundThumbnails() {
391 const result = [];
392
393 try {
394 const knownBackgrounds = new Set();
395 const backgrounds = await fs.promises.readdir(this.directories.backgrounds, { withFileTypes: true });
396 for (const file of backgrounds) {
397 if (file.isFile()) {
398 knownBackgrounds.add(file.name);
399 }
400 }
401 const backgroundThumbnails = await fs.promises.readdir(this.directories.thumbnailsBg, { withFileTypes: true });
402 for (const file of backgroundThumbnails) {
403 if (file.isFile() && !knownBackgrounds.has(file.name)) {
404 result.push(path.join(this.directories.thumbnailsBg, file.name));
405 }
406 }
407 } catch (error) {
408 console.error('[Data Maid] Error collecting background thumbnails:', error);
409 }
410
411 return result;
412 }
413
414 /**
415 * Collects chat backups from the provided directories.
416 * @returns {Promise<string[]>} List of paths to chat backups
417 */
418 async #collectChatBackups() {
419 const result = [];
420
421 try {
422 const prefix = CHAT_BACKUPS_PREFIX;
423 const backups = await fs.promises.readdir(this.directories.backups, { withFileTypes: true });
424 for (const file of backups) {
425 if (file.isFile() && file.name.startsWith(prefix)) {
426 result.push(path.join(this.directories.backups, file.name));
427 }
428 }
429 } catch (error) {
430 console.error('[Data Maid] Error collecting chat backups:', error);
431 }
432
433 return result;
434 }
435
436 /**
437 * Collects settings backups from the provided directories.
438 * @returns {Promise<string[]>} List of paths to settings backups
439 */
440 async #collectSettingsBackups() {
441 const result = [];
442
443 try {
444 const prefix = getSettingsBackupFilePrefix(this.handle);
445 const backups = await fs.promises.readdir(this.directories.backups, { withFileTypes: true });
446 for (const file of backups) {
447 if (file.isFile() && file.name.startsWith(prefix)) {
448 result.push(path.join(this.directories.backups, file.name));
449 }
450 }
451 } catch (error) {
452 console.error('[Data Maid] Error collecting settings backups:', error);
453 }
454
455 return result;
456 }
457
458 /**
459 * Parses all chat files and returns an array of chat messages.
460 * Searches both individual character chats and group chats.
461 * @param {function(DataMaidMessage): boolean} filterFn - Filter function to apply to each message.
462 * @returns {Promise<DataMaidMessage[]>} Array of chat messages
463 */
464 async #parseAllChats(filterFn) {
465 try {
466 const allChats = [];
467
468 const groupChats = await fs.promises.readdir(this.directories.groupChats, { withFileTypes: true });
469 for (const file of groupChats) {
470 if (file.isFile() && path.parse(file.name).ext === '.jsonl') {
471 const chatMessages = await this.#parseChatFile(path.join(this.directories.groupChats, file.name));
472 allChats.push(...chatMessages.filter(filterFn));
473 }
474 }
475
476 const chatDirectories = await fs.promises.readdir(this.directories.chats, { withFileTypes: true });
477 for (const directory of chatDirectories) {
478 if (directory.isDirectory()) {
479 const chatFiles = await fs.promises.readdir(path.join(this.directories.chats, directory.name), { withFileTypes: true });
480 for (const file of chatFiles) {
481 if (file.isFile() && path.parse(file.name).ext === '.jsonl') {
482 const chatMessages = await this.#parseChatFile(path.join(this.directories.chats, directory.name, file.name));
483 allChats.push(...chatMessages.filter(filterFn));
484 }
485 }
486 }
487 }
488
489 return allChats;
490 } catch (error) {
491 console.error('[Data Maid] Error parsing chats:', error);
492 return [];
493 }
494 }
495
496 /**
497 * Parses all metadata from chat files and group definitions.
498 * Extracts metadata from both active and historical chat data.
499 * @param {function(DataMaidChatMetadata): boolean} filterFn - Filter function to apply to each metadata entry.
500 * @returns {Promise<DataMaidChatMetadata[]>} Parsed chat metadata as an array.
501 */
502 async #parseAllMetadata(filterFn) {
503 try {
504 const allMetadata = [];
505
506 const groups = await fs.promises.readdir(this.directories.groups, { withFileTypes: true });
507 for (const file of groups) {
508 if (file.isFile() && path.parse(file.name).ext === '.json') {
509 try {
510 const pathToFile = path.join(this.directories.groups, file.name);
511 const fileContent = await fs.promises.readFile(pathToFile, 'utf-8');
512 const groupData = tryParse(fileContent);
513 if (groupData?.chat_metadata && filterFn(groupData.chat_metadata)) {
514 allMetadata.push(groupData.chat_metadata);
515 }
516 if (groupData?.past_metadata) {
517 allMetadata.push(...Object.values(groupData.past_metadata).filter(filterFn));
518 }
519 } catch (error) {
520 console.error(`[Data Maid] Error parsing group chat file ${file.name}:`, error);
521 }
522 }
523 }
524
525 const chatDirectories = await fs.promises.readdir(this.directories.chats, { withFileTypes: true });
526 for (const directory of chatDirectories) {
527 if (directory.isDirectory()) {
528 const chatFiles = await fs.promises.readdir(path.join(this.directories.chats, directory.name), { withFileTypes: true });
529 for (const file of chatFiles) {
530 if (file.isFile() && path.parse(file.name).ext === '.jsonl') {
531 const chatMessages = await this.#parseChatFile(path.join(this.directories.chats, directory.name, file.name));
532 const chatMetadata = chatMessages?.[0]?.chat_metadata;
533 if (chatMetadata && filterFn(chatMetadata)) {
534 allMetadata.push(chatMetadata);
535 }
536 }
537 }
538 }
539 }
540
541 return allMetadata;
542 } catch (error) {
543 console.error('[Data Maid] Error parsing chats:', error);
544 return [];
545 }
546 }
547
548 /**
549 * Parses a single chat file and returns an array of chat messages.
550 * Each line in the JSONL file represents one message.
551 * @param {string} filePath Path to the chat file to parse.
552 * @returns {Promise<DataMaidMessage[]>} Parsed chat messages as an array.
553 */
554 async #parseChatFile(filePath) {
555 try {
556 const content = await fs.promises.readFile(filePath, 'utf-8');
557 const chatData = content.split('\n').map(tryParse).filter(Boolean);
558 return chatData;
559 } catch (error) {
560 console.error(`[Data Maid] Error reading chat file ${filePath}:`, error);
561 return [];
562 }
563 }
564
565 /**
566 * Generates a unique token for the user to clean up their data.
567 * Replaces any existing token for the same user.
568 * @param {string} handle - The user's handle or identifier.
569 * @param {DataMaidRawReport} report - The report containing loose user data.
570 * @returns {string} A unique token.
571 */
572 static generateToken(handle, report) {
573 // Remove any existing token for this user
574 for (const [token, entry] of this.TOKENS.entries()) {
575 if (entry.handle === handle) {
576 this.TOKENS.delete(token);
577 }
578 }
579
580 const token = crypto.randomBytes(32).toString('hex');
581 const tokenEntry = {
582 handle,
583 paths: Object.values(report).filter(v => Array.isArray(v)).flat().map(x => ({ path: x, hash: sha256(x) })),
584 };
585 this.TOKENS.set(token, tokenEntry);
586 return token;
587 }
588}
589
590export const router = express.Router();
591
592router.post('/report', async (req, res) => {
593 try {
594 if (!req.user || !req.user.directories) {
595 return res.sendStatus(403);
596 }
597
598 const dataMaid = new DataMaidService(req.user.profile.handle, req.user.directories);
599 const rawReport = await dataMaid.generateReport();
600
601 const report = await dataMaid.sanitizeReport(rawReport);
602 const token = DataMaidService.generateToken(req.user.profile.handle, rawReport);
603
604 return res.json({ report, token });
605 } catch (error) {
606 console.error('[Data Maid] Error generating data maid report:', error);
607 return res.sendStatus(500);
608 }
609});
610
611router.post('/finalize', async (req, res) => {
612 try {
613 if (!req.user || !req.user.directories) {
614 return res.sendStatus(403);
615 }
616
617 if (!req.body.token) {
618 return res.sendStatus(400);
619 }
620
621 const token = req.body.token.toString();
622 if (!DataMaidService.TOKENS.has(token)) {
623 return res.sendStatus(403);
624 }
625
626 const tokenEntry = DataMaidService.TOKENS.get(token);
627 if (!tokenEntry || tokenEntry.handle !== req.user.profile.handle) {
628 return res.sendStatus(403);
629 }
630
631 // Remove the token after finalization
632 DataMaidService.TOKENS.delete(token);
633 return res.sendStatus(204);
634 } catch (error) {
635 console.error('[Data Maid] Error finalizing the token:', error);
636 return res.sendStatus(500);
637 }
638});
639
640router.get('/view', async (req, res) => {
641 try {
642 if (!req.user || !req.user.directories) {
643 return res.sendStatus(403);
644 }
645
646 if (!req.query.token || !req.query.hash) {
647 return res.sendStatus(400);
648 }
649
650 const token = req.query.token.toString();
651 const hash = req.query.hash.toString();
652
653 if (!DataMaidService.TOKENS.has(token)) {
654 return res.sendStatus(403);
655 }
656
657 const tokenEntry = DataMaidService.TOKENS.get(token);
658 if (!tokenEntry || tokenEntry.handle !== req.user.profile.handle) {
659 return res.sendStatus(403);
660 }
661
662 const fileEntry = tokenEntry.paths.find(entry => entry.hash === hash);
663 if (!fileEntry) {
664 return res.sendStatus(404);
665 }
666
667 if (!isPathUnderParent(req.user.directories.root, fileEntry.path)) {
668 console.warn('[Data Maid] Attempted access to a file outside of the user directory:', fileEntry.path);
669 return res.sendStatus(403);
670 }
671
672 const pathToFile = fileEntry.path;
673 const fileExists = fs.existsSync(pathToFile);
674
675 if (!fileExists) {
676 return res.sendStatus(404);
677 }
678
679 const fileBuffer = await fs.promises.readFile(pathToFile);
680 const mimeType = mime.lookup(pathToFile) || 'text/plain';
681 res.setHeader('Content-Type', mimeType);
682 return res.send(fileBuffer);
683 } catch (error) {
684 console.error('[Data Maid] Error viewing file:', error);
685 return res.sendStatus(500);
686 }
687});
688
689router.post('/delete', async (req, res) => {
690 try {
691 if (!req.user || !req.user.directories) {
692 return res.sendStatus(403);
693 }
694
695 const { token, hashes } = req.body;
696 if (!token || !Array.isArray(hashes) || hashes.length === 0) {
697 return res.sendStatus(400);
698 }
699
700 if (!DataMaidService.TOKENS.has(token)) {
701 return res.sendStatus(403);
702 }
703
704 const tokenEntry = DataMaidService.TOKENS.get(token);
705 if (!tokenEntry || tokenEntry.handle !== req.user.profile.handle) {
706 return res.sendStatus(403);
707 }
708
709 for (const hash of hashes) {
710 const fileEntry = tokenEntry.paths.find(entry => entry.hash === hash);
711 if (!fileEntry) {
712 continue;
713 }
714
715 if (!isPathUnderParent(req.user.directories.root, fileEntry.path)) {
716 console.warn('[Data Maid] Attempted deletion of a file outside of the user directory:', fileEntry.path);
717 continue;
718 }
719
720 const pathToFile = fileEntry.path;
721 const fileExists = fs.existsSync(pathToFile);
722
723 if (!fileExists) {
724 continue;
725 }
726
727 await fs.promises.unlink(pathToFile);
728 }
729
730 return res.sendStatus(204);
731 } catch (error) {
732 console.error('[Data Maid] Error deleting files:', error);
733 return res.sendStatus(500);
734 }
735});
src/endpoints/settings.js+6 -6
@@ -81,7 +81,7 @@ function sortByName(_) {
81 * @param {string} handle User handle81 * @param {string} handle User handle
82 * @returns {string} File prefix82 * @returns {string} File prefix
83 */83 */
84function getFilePrefix(handle) {84export function getSettingsBackupFilePrefix(handle) {
85 return `settings_${handle}_`;85 return `settings_${handle}_`;
86}86}
8787
@@ -131,7 +131,7 @@ async function backupSettings() {
131 */131 */
132function backupUserSettings(handle, preventDuplicates) {132function backupUserSettings(handle, preventDuplicates) {
133 const userDirectories = getUserDirectories(handle);133 const userDirectories = getUserDirectories(handle);
134 const backupFile = path.join(userDirectories.backups, `${getFilePrefix(handle)}${generateTimestamp()}.json`);134 const backupFile = path.join(userDirectories.backups, `${getSettingsBackupFilePrefix(handle)}${generateTimestamp()}.json`);
135 const sourceFile = path.join(userDirectories.root, SETTINGS_FILE);135 const sourceFile = path.join(userDirectories.root, SETTINGS_FILE);
136136
137 if (preventDuplicates && isDuplicateBackup(handle, sourceFile)) {137 if (preventDuplicates && isDuplicateBackup(handle, sourceFile)) {
@@ -183,7 +183,7 @@ function areFilesEqual(file1, file2) {
183function getLatestBackup(handle) {183function getLatestBackup(handle) {
184 const userDirectories = getUserDirectories(handle);184 const userDirectories = getUserDirectories(handle);
185 const backupFiles = fs.readdirSync(userDirectories.backups)185 const backupFiles = fs.readdirSync(userDirectories.backups)
186 .filter(x => x.startsWith(getFilePrefix(handle)))186 .filter(x => x.startsWith(getSettingsBackupFilePrefix(handle)))
187 .map(x => ({ name: x, ctime: fs.statSync(path.join(userDirectories.backups, x)).ctimeMs }));187 .map(x => ({ name: x, ctime: fs.statSync(path.join(userDirectories.backups, x)).ctimeMs }));
188 const latestBackup = backupFiles.sort((a, b) => b.ctime - a.ctime)[0]?.name;188 const latestBackup = backupFiles.sort((a, b) => b.ctime - a.ctime)[0]?.name;
189 if (!latestBackup) {189 if (!latestBackup) {
@@ -283,7 +283,7 @@ router.post('/get', (request, response) => {
283router.post('/get-snapshots', async (request, response) => {283router.post('/get-snapshots', async (request, response) => {
284 try {284 try {
285 const snapshots = fs.readdirSync(request.user.directories.backups);285 const snapshots = fs.readdirSync(request.user.directories.backups);
286 const userFilesPattern = getFilePrefix(request.user.profile.handle);286 const userFilesPattern = getSettingsBackupFilePrefix(request.user.profile.handle);
287 const userSnapshots = snapshots.filter(x => x.startsWith(userFilesPattern));287 const userSnapshots = snapshots.filter(x => x.startsWith(userFilesPattern));
288288
289 const result = userSnapshots.map(x => {289 const result = userSnapshots.map(x => {
@@ -300,7 +300,7 @@ router.post('/get-snapshots', async (request, response) => {
300300
301router.post('/load-snapshot', getFileNameValidationFunction('name'), async (request, response) => {301router.post('/load-snapshot', getFileNameValidationFunction('name'), async (request, response) => {
302 try {302 try {
303 const userFilesPattern = getFilePrefix(request.user.profile.handle);303 const userFilesPattern = getSettingsBackupFilePrefix(request.user.profile.handle);
304304
305 if (!request.body.name || !request.body.name.startsWith(userFilesPattern)) {305 if (!request.body.name || !request.body.name.startsWith(userFilesPattern)) {
306 return response.status(400).send({ error: 'Invalid snapshot name' });306 return response.status(400).send({ error: 'Invalid snapshot name' });
@@ -334,7 +334,7 @@ router.post('/make-snapshot', async (request, response) => {
334334
335router.post('/restore-snapshot', getFileNameValidationFunction('name'), async (request, response) => {335router.post('/restore-snapshot', getFileNameValidationFunction('name'), async (request, response) => {
336 try {336 try {
337 const userFilesPattern = getFilePrefix(request.user.profile.handle);337 const userFilesPattern = getSettingsBackupFilePrefix(request.user.profile.handle);
338338
339 if (!request.body.name || !request.body.name.startsWith(userFilesPattern)) {339 if (!request.body.name || !request.body.name.startsWith(userFilesPattern)) {
340 return response.status(400).send({ error: 'Invalid snapshot name' });340 return response.status(400).send({ error: 'Invalid snapshot name' });
src/fetch-patch.js+1 -51
@@ -3,6 +3,7 @@ import path from 'node:path';
3import { fileURLToPath } from 'node:url';3import { fileURLToPath } from 'node:url';
4import mime from 'mime-types';4import mime from 'mime-types';
5import { serverDirectory } from './server-directory.js';5import { serverDirectory } from './server-directory.js';
6import { getRequestURL, isFileURL, isPathUnderParent } from './util.js';
67
7const originalFetch = globalThis.fetch;8const originalFetch = globalThis.fetch;
89
@@ -10,57 +11,6 @@ const ALLOWED_EXTENSIONS = [
10 '.wasm',11 '.wasm',
11];12];
1213
13/**
14 * Checks if a child path is under a parent path.
15 * @param {string} parentPath Parent path
16 * @param {string} childPath Child path
17 * @returns {boolean} Returns true if the child path is under the parent path, false otherwise
18 */
19function isPathUnderParent(parentPath, childPath) {
20 const normalizedParent = path.normalize(parentPath);
21 const normalizedChild = path.normalize(childPath);
22
23 const relativePath = path.relative(normalizedParent, normalizedChild);
24
25 return !relativePath.startsWith('..') && !path.isAbsolute(relativePath);
26}
27
28/**
29 * Checks if the given request is a file URL.
30 * @param {string | URL | Request} request The request to check
31 * @return {boolean} Returns true if the request is a file URL, false otherwise
32 */
33function isFileURL(request) {
34 if (typeof request === 'string') {
35 return request.startsWith('file://');
36 }
37 if (request instanceof URL) {
38 return request.protocol === 'file:';
39 }
40 if (request instanceof Request) {
41 return request.url.startsWith('file://');
42 }
43 return false;
44}
45
46/**
47 * Gets the URL from the request.
48 * @param {string | URL | Request} request The request to get the URL from
49 * @return {string} The URL of the request
50 */
51function getRequestURL(request) {
52 if (typeof request === 'string') {
53 return request;
54 }
55 if (request instanceof URL) {
56 return request.href;
57 }
58 if (request instanceof Request) {
59 return request.url;
60 }
61 throw new TypeError('Invalid request type');
62}
63
64// Patched fetch function that handles file URLs14// Patched fetch function that handles file URLs
65globalThis.fetch = async (/** @type {string | URL | Request} */ request, /** @type {RequestInit | undefined} */ options) => {15globalThis.fetch = async (/** @type {string | URL | Request} */ request, /** @type {RequestInit | undefined} */ options) => {
66 if (!isFileURL(request)) {16 if (!isFileURL(request)) {
src/server-startup.js+2 -0
@@ -46,6 +46,7 @@ import { router as textCompletionsRouter } from './endpoints/backends/text-compl
46import { router as scaleAltRouter } from './endpoints/backends/scale-alt.js';46import { router as scaleAltRouter } from './endpoints/backends/scale-alt.js';
47import { router as speechRouter } from './endpoints/speech.js';47import { router as speechRouter } from './endpoints/speech.js';
48import { router as azureRouter } from './endpoints/azure.js';48import { router as azureRouter } from './endpoints/azure.js';
49import { router as dataMaidRouter } from './endpoints/data-maid.js';
4950
50/**51/**
51 * @typedef {object} ServerStartupResult52 * @typedef {object} ServerStartupResult
@@ -173,6 +174,7 @@ export function setupPrivateEndpoints(app) {
173 app.use('/api/backends/scale-alt', scaleAltRouter);174 app.use('/api/backends/scale-alt', scaleAltRouter);
174 app.use('/api/speech', speechRouter);175 app.use('/api/speech', speechRouter);
175 app.use('/api/azure', azureRouter);176 app.use('/api/azure', azureRouter);
177 app.use('/api/data-maid', dataMaidRouter);
176}178}
177179
178/**180/**
src/util.js+51 -0
@@ -1146,3 +1146,54 @@ export function setPermissionsSync(targetPath) {
1146 console.error(`Error setting write permissions for ${targetPath}:`, error);1146 console.error(`Error setting write permissions for ${targetPath}:`, error);
1147 }1147 }
1148}1148}
1149
1150/**
1151 * Checks if a child path is under a parent path.
1152 * @param {string} parentPath Parent path
1153 * @param {string} childPath Child path
1154 * @returns {boolean} Returns true if the child path is under the parent path, false otherwise
1155 */
1156export function isPathUnderParent(parentPath, childPath) {
1157 const normalizedParent = path.normalize(parentPath);
1158 const normalizedChild = path.normalize(childPath);
1159
1160 const relativePath = path.relative(normalizedParent, normalizedChild);
1161
1162 return !relativePath.startsWith('..') && !path.isAbsolute(relativePath);
1163}
1164
1165/**
1166 * Checks if the given request is a file URL.
1167 * @param {string | URL | Request} request The request to check
1168 * @return {boolean} Returns true if the request is a file URL, false otherwise
1169 */
1170export function isFileURL(request) {
1171 if (typeof request === 'string') {
1172 return request.startsWith('file://');
1173 }
1174 if (request instanceof URL) {
1175 return request.protocol === 'file:';
1176 }
1177 if (request instanceof Request) {
1178 return request.url.startsWith('file://');
1179 }
1180 return false;
1181}
1182
1183/**
1184 * Gets the URL from the request.
1185 * @param {string | URL | Request} request The request to get the URL from
1186 * @return {string} The URL of the request
1187 */
1188export function getRequestURL(request) {
1189 if (typeof request === 'string') {
1190 return request;
1191 }
1192 if (request instanceof URL) {
1193 return request.href;
1194 }
1195 if (request instanceof Request) {
1196 return request.url;
1197 }
1198 throw new TypeError('Invalid request type');
1199}