1
import { getRequestHeaders } from '../script.js';
2
import { renderExtensionTemplateAsync } from './extensions.js';
3
import { POPUP_RESULT, POPUP_TYPE, callGenericPopup } from './popup.js';
4
import { SlashCommand } from './slash-commands/SlashCommand.js';
5
import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from './slash-commands/SlashCommandArgument.js';
6
import { SlashCommandParser } from './slash-commands/SlashCommandParser.js';
7
import { isValidUrl } from './utils.js';
8

9
/**
10
 * @typedef {Object} Scraper
11
 * @property {string} id
12
 * @property {string} name
13
 * @property {string} description
14
 * @property {string} iconClass
15
 * @property {boolean} iconAvailable
16
 * @property {() => Promise<void>} [init=null]
17
 * @property {() => Promise<boolean>} isAvailable
18
 * @property {() => Promise<File[]>} scrape
19
 */
20

21
/**
22
 * @typedef {Object} ScraperInfo
23
 * @property {string} id
24
 * @property {string} name
25
 * @property {string} description
26
 * @property {string} iconClass
27
 * @property {boolean} iconAvailable
28
 */
29

30
export class ScraperManager {
31
    /**
32
     * @type {Scraper[]}
33
     */
34
    static #scrapers = [];
35

36
    /**
37
     * Register a scraper to be used by the Data Bank.
38
     * @param {Scraper} scraper Instance of a scraper to register
39
     */
40
    static async registerDataBankScraper(scraper) {
41
        if (ScraperManager.#scrapers.some(s => s.id === scraper.id)) {
42
            console.warn(`Scraper with ID ${scraper.id} already registered`);
43
            return;
44
        }
45

46
        if (scraper.init) {
47
            await scraper.init();
48
        }
49

50
        ScraperManager.#scrapers.push(scraper);
51
    }
52

53
    /**
54
     * Gets a list of scrapers available for the Data Bank.
55
     * @returns {ScraperInfo[]} List of scrapers available for the Data Bank
56
     */
57
    static getDataBankScrapers() {
58
        return ScraperManager.#scrapers.map(s => ({ id: s.id, name: s.name, description: s.description, iconClass: s.iconClass, iconAvailable: s.iconAvailable }));
59
    }
60

61
    /**
62
     * Run a scraper to scrape data into the Data Bank.
63
     * @param {string} scraperId ID of the scraper to run
64
     * @returns {Promise<File[]>} List of files scraped by the scraper
65
     */
66
    static runDataBankScraper(scraperId) {
67
        const scraper = ScraperManager.#scrapers.find(s => s.id === scraperId);
68
        if (!scraper) {
69
            console.warn(`Scraper with ID ${scraperId} not found`);
70
            return;
71
        }
72
        return scraper.scrape();
73
    }
74

75
    /**
76
     * Check if a scraper is available.
77
     * @param {string} scraperId ID of the scraper to check
78
     * @returns {Promise<boolean>} Whether the scraper is available
79
     */
80
    static isScraperAvailable(scraperId) {
81
        const scraper = ScraperManager.#scrapers.find(s => s.id === scraperId);
82
        if (!scraper) {
83
            console.warn(`Scraper with ID ${scraperId} not found`);
84
            return;
85
        }
86
        return scraper.isAvailable();
87
    }
88
}
89

90
/**
91
 * Create a text file from a string.
92
 * @implements {Scraper}
93
 */
94
class Notepad {
95
    constructor() {
96
        this.id = 'text';
97
        this.name = 'Notepad';
98
        this.description = 'Create a text file from scratch.';
99
        this.iconClass = 'fa-solid fa-note-sticky';
100
        this.iconAvailable = true;
101
    }
102

103
    /**
104
     * Check if the scraper is available.
105
     * @returns {Promise<boolean>}
106
     */
107
    async isAvailable() {
108
        return true;
109
    }
110

111
    /**
112
     * Create a text file from a string.
113
     * @returns {Promise<File[]>} File attachments scraped from the text
114
     */
115
    async scrape() {
116
        const template = $(await renderExtensionTemplateAsync('attachments', 'notepad', {}));
117
        let fileName = `Untitled - ${new Date().toLocaleString()}`;
118
        let text = '';
119
        template.find('input[name="notepadFileName"]').val(fileName).on('input', function () {
120
            fileName = String($(this).val()).trim();
121
        });
122
        template.find('textarea[name="notepadFileContent"]').on('input', function () {
123
            text = String($(this).val());
124
        });
125

126
        const result = await callGenericPopup(template, POPUP_TYPE.CONFIRM, '', { wide: true, large: true, okButton: 'Save', cancelButton: 'Cancel' });
127

128
        if (!result || text === '') {
129
            return;
130
        }
131

132
        const file = new File([text], `Notepad - ${fileName}.txt`, { type: 'text/plain' });
133
        return [file];
134
    }
135
}
136

137
/**
138
 * Scrape data from a webpage.
139
 * @implements {Scraper}
140
 */
141
class WebScraper {
142
    constructor() {
143
        this.id = 'web';
144
        this.name = 'Web';
145
        this.description = 'Download a page from the web.';
146
        this.iconClass = 'fa-solid fa-globe';
147
        this.iconAvailable = true;
148
    }
149

150
    /**
151
     * Check if the scraper is available.
152
     * @returns {Promise<boolean>}
153
     */
154
    async isAvailable() {
155
        return true;
156
    }
157

158
    /**
159
    * Parse the title of an HTML file from a Blob.
160
    * @param {Blob} blob Blob of the HTML file
161
    * @returns {Promise<string>} Title of the HTML file
162
    */
163
    async getTitleFromHtmlBlob(blob) {
164
        const text = await blob.text();
165
        const titleMatch = text.match(/<title>(.*?)<\/title>/i);
166
        return titleMatch ? titleMatch[1] : '';
167
    }
168

169
    /**
170
     * Scrape file attachments from a webpage.
171
     * @returns {Promise<File[]>} File attachments scraped from the webpage
172
     */
173
    async scrape() {
174
        const template = $(await renderExtensionTemplateAsync('attachments', 'web-scrape', {}));
175
        const linksString = await callGenericPopup(template, POPUP_TYPE.INPUT, '', { wide: false, large: false, okButton: 'Scrape', cancelButton: 'Cancel', rows: 4 });
176

177
        if (!linksString) {
178
            return;
179
        }
180

181
        const links = String(linksString).split('\n').map(l => l.trim()).filter(l => l).filter(l => isValidUrl(l));
182

183
        if (links.length === 0) {
184
            toastr.error('Invalid URL');
185
            return;
186
        }
187

188
        const toast = toastr.info('Working, please wait...');
189

190
        const files = [];
191

192
        for (const link of links) {
193
            const result = await fetch('/api/search/visit', {
194
                method: 'POST',
195
                headers: getRequestHeaders(),
196
                body: JSON.stringify({ url: link }),
197
            });
198

199
            const blob = await result.blob();
200
            const domain = new URL(link).hostname;
201
            const timestamp = Date.now();
202
            const title = await this.getTitleFromHtmlBlob(blob) || 'webpage';
203
            const file = new File([blob], `${title} - ${domain} - ${timestamp}.html`, { type: 'text/html' });
204
            files.push(file);
205
        }
206

207
        toastr.clear(toast);
208
        return files;
209
    }
210
}
211

212
/**
213
 * Scrape data from a file selection.
214
 * @implements {Scraper}
215
 */
216
class FileScraper {
217
    constructor() {
218
        this.id = 'file';
219
        this.name = 'File';
220
        this.description = 'Upload a file from your computer.';
221
        this.iconClass = 'fa-solid fa-upload';
222
        this.iconAvailable = true;
223
    }
224

225
    /**
226
     * Check if the scraper is available.
227
     * @returns {Promise<boolean>}
228
     */
229
    async isAvailable() {
230
        return true;
231
    }
232

233
    /**
234
     * Scrape file attachments from a file.
235
     * @returns {Promise<File[]>} File attachments scraped from the files
236
     */
237
    async scrape() {
238
        return new Promise(resolve => {
239
            const fileInput = document.createElement('input');
240
            fileInput.type = 'file';
241
            fileInput.accept = '*/*';
242
            fileInput.multiple = true;
243
            fileInput.onchange = () => resolve(Array.from(fileInput.files));
244
            fileInput.click();
245
        });
246
    }
247
}
248

249
class MediaWikiScraper {
250
    constructor() {
251
        this.id = 'mediawiki';
252
        this.name = 'MediaWiki';
253
        this.description = 'Download a page from a MediaWiki wiki.';
254
        this.iconClass = 'fa-brands fa-wikipedia-w';
255
        this.iconAvailable = true;
256
    }
257

258
    async isAvailable() {
259
        try {
260
            const result = await fetch('/api/plugins/fandom/probe-mediawiki', {
261
                method: 'POST',
262
                headers: getRequestHeaders({ omitContentType: true }),
263
            });
264

265
            return result.ok;
266
        } catch (error) {
267
            console.debug('Could not probe Fandom/MediaWiki plugin', error);
268
            return false;
269
        }
270
    }
271

272
    async scrape() {
273
        let url = '';
274
        let filter = '';
275
        let output = 'single';
276

277
        const template = $(await renderExtensionTemplateAsync('attachments', 'mediawiki-scrape', {}));
278
        template.find('input[name="scrapeInput"]').on('input', function () {
279
            url = String($(this).val()).trim();
280
        });
281
        template.find('input[name="scrapeFilter"]').on('input', function () {
282
            filter = String($(this).val());
283
        });
284
        template.find('input[name="scrapeOutput"]').on('input', function () {
285
            output = String($(this).val());
286
        });
287

288
        const confirm = await callGenericPopup(template, POPUP_TYPE.CONFIRM, '', { wide: false, large: false, okButton: 'Scrape', cancelButton: 'Cancel' });
289

290
        if (confirm !== POPUP_RESULT.AFFIRMATIVE) {
291
            return;
292
        }
293

294
        if (!url) {
295
            toastr.error('URL name is required');
296
            return;
297
        }
298

299
        const toast = toastr.info('Working, please wait...');
300

301
        const result = await fetch('/api/plugins/fandom/scrape-mediawiki', {
302
            method: 'POST',
303
            headers: getRequestHeaders(),
304
            body: JSON.stringify({ url, filter }),
305
        });
306

307
        if (!result.ok) {
308
            const error = await result.text();
309
            throw new Error(error);
310
        }
311

312
        const data = await result.json();
313
        toastr.clear(toast);
314

315
        if (output === 'multi') {
316
            const files = [];
317
            for (const attachment of data) {
318
                const file = new File([String(attachment.content).trim()], `${String(attachment.title).trim()}.txt`, { type: 'text/plain' });
319
                files.push(file);
320
            }
321
            return files;
322
        }
323

324
        if (output === 'single') {
325
            const combinedContent = data.map((a) => String(a.title).trim() + '\n\n' + String(a.content).trim()).join('\n\n\n\n');
326
            const file = new File([combinedContent], `${url}.txt`, { type: 'text/plain' });
327
            return [file];
328
        }
329

330
        return [];
331
    }
332
}
333

334
/**
335
 * Scrape data from a Fandom wiki.
336
 * @implements {Scraper}
337
 */
338
class FandomScraper {
339
    constructor() {
340
        this.id = 'fandom';
341
        this.name = 'Fandom';
342
        this.description = 'Download a page from the Fandom wiki.';
343
        this.iconClass = 'fa-solid fa-fire';
344
        this.iconAvailable = true;
345
    }
346

347
    /**
348
     * Check if the scraper is available.
349
     * @returns {Promise<boolean>}
350
     */
351
    async isAvailable() {
352
        try {
353
            const result = await fetch('/api/plugins/fandom/probe', {
354
                method: 'POST',
355
                headers: getRequestHeaders({ omitContentType: true }),
356
            });
357

358
            return result.ok;
359
        } catch (error) {
360
            console.debug('Could not probe Fandom plugin', error);
361
            return false;
362
        }
363
    }
364

365
    /**
366
     * Get the ID of a fandom from a URL or name.
367
     * @param {string} fandom URL or name of the fandom
368
     * @returns {string} ID of the fandom
369
     */
370
    getFandomId(fandom) {
371
        try {
372
            const url = new URL(fandom);
373
            return url.hostname.split('.')[0] || fandom;
374
        } catch {
375
            return fandom;
376
        }
377
    }
378

379
    async scrape() {
380
        let fandom = '';
381
        let filter = '';
382
        let output = 'single';
383

384
        const template = $(await renderExtensionTemplateAsync('attachments', 'fandom-scrape', {}));
385
        template.find('input[name="fandomScrapeInput"]').on('input', function () {
386
            fandom = String($(this).val()).trim();
387
        });
388
        template.find('input[name="fandomScrapeFilter"]').on('input', function () {
389
            filter = String($(this).val());
390
        });
391
        template.find('input[name="fandomScrapeOutput"]').on('input', function () {
392
            output = String($(this).val());
393
        });
394

395
        const confirm = await callGenericPopup(template, POPUP_TYPE.CONFIRM, '', { wide: false, large: false, okButton: 'Scrape', cancelButton: 'Cancel' });
396

397
        if (confirm !== POPUP_RESULT.AFFIRMATIVE) {
398
            return;
399
        }
400

401
        if (!fandom) {
402
            toastr.error('Fandom name is required');
403
            return;
404
        }
405

406
        const toast = toastr.info('Working, please wait...');
407

408
        const result = await fetch('/api/plugins/fandom/scrape', {
409
            method: 'POST',
410
            headers: getRequestHeaders(),
411
            body: JSON.stringify({ fandom, filter }),
412
        });
413

414
        if (!result.ok) {
415
            const error = await result.text();
416
            throw new Error(error);
417
        }
418

419
        const data = await result.json();
420
        toastr.clear(toast);
421

422
        if (output === 'multi') {
423
            const files = [];
424
            for (const attachment of data) {
425
                const file = new File([String(attachment.content).trim()], `${String(attachment.title).trim()}.txt`, { type: 'text/plain' });
426
                files.push(file);
427
            }
428
            return files;
429
        }
430

431
        if (output === 'single') {
432
            const combinedContent = data.map((a) => String(a.title).trim() + '\n\n' + String(a.content).trim()).join('\n\n\n\n');
433
            const file = new File([combinedContent], `${fandom}.txt`, { type: 'text/plain' });
434
            return [file];
435
        }
436

437
        return [];
438
    }
439
}
440

441
const iso6391Codes = [
442
    'aa', 'ab', 'ae', 'af', 'ak', 'am', 'an', 'ar', 'as', 'av', 'ay', 'az',
443
    'ba', 'be', 'bg', 'bh', 'bi', 'bm', 'bn', 'bo', 'br', 'bs', 'ca', 'ce',
444
    'ch', 'co', 'cr', 'cs', 'cu', 'cv', 'cy', 'da', 'de', 'dv', 'dz', 'ee',
445
    'el', 'en', 'eo', 'es', 'et', 'eu', 'fa', 'ff', 'fi', 'fj', 'fo', 'fr',
446
    'fy', 'ga', 'gd', 'gl', 'gn', 'gu', 'gv', 'ha', 'he', 'hi', 'ho', 'hr',
447
    'ht', 'hu', 'hy', 'hz', 'ia', 'id', 'ie', 'ig', 'ii', 'ik', 'io', 'is',
448
    'it', 'iu', 'ja', 'jv', 'ka', 'kg', 'ki', 'kj', 'kk', 'kl', 'km', 'kn',
449
    'ko', 'kr', 'ks', 'ku', 'kv', 'kw', 'ky', 'la', 'lb', 'lg', 'li', 'ln',
450
    'lo', 'lt', 'lu', 'lv', 'mg', 'mh', 'mi', 'mk', 'ml', 'mn', 'mr', 'ms',
451
    'mt', 'my', 'na', 'nb', 'nd', 'ne', 'ng', 'nl', 'nn', 'no', 'nr', 'nv',
452
    'ny', 'oc', 'oj', 'om', 'or', 'os', 'pa', 'pi', 'pl', 'ps', 'pt', 'qu',
453
    'rm', 'rn', 'ro', 'ru', 'rw', 'sa', 'sc', 'sd', 'se', 'sg', 'si', 'sk',
454
    'sl', 'sm', 'sn', 'so', 'sq', 'sr', 'ss', 'st', 'su', 'sv', 'sw', 'ta',
455
    'te', 'tg', 'th', 'ti', 'tk', 'tl', 'tn', 'to', 'tr', 'ts', 'tt', 'tw',
456
    'ty', 'ug', 'uk', 'ur', 'uz', 've', 'vi', 'vo', 'wa', 'wo', 'xh', 'yi',
457
    'yo', 'za', 'zh', 'zu'];
458

459
/**
460
 * Scrape transcript from a YouTube video.
461
 * @implements {Scraper}
462
 */
463
class YouTubeScraper {
464
    constructor() {
465
        this.id = 'youtube';
466
        this.name = 'YouTube';
467
        this.description = 'Download a transcript from a YouTube video.';
468
        this.iconClass = 'fa-brands fa-youtube';
469
        this.iconAvailable = true;
470
    }
471

472
    async init() {
473
        SlashCommandParser.addCommandObject(SlashCommand.fromProps({
474
            name: 'yt-script',
475
            callback: async (args, url) => {
476
                try {
477
                    if (!url) {
478
                        throw new Error('URL or ID of the YouTube video is required');
479
                    }
480

481
                    const lang = String(args?.lang || '');
482
                    const { transcript } = await this.getScript(String(url).trim(), lang);
483
                    return transcript;
484
                } catch (error) {
485
                    toastr.error(error.message);
486
                    return '';
487
                }
488
            },
489
            helpString: 'Scrape a transcript from a YouTube video by ID or URL.',
490
            returns: ARGUMENT_TYPE.STRING,
491
            namedArgumentList: [
492
                new SlashCommandNamedArgument('lang', 'ISO 639-1 language code of the transcript, e.g. "en"', ARGUMENT_TYPE.STRING, false, false, '', iso6391Codes),
493
            ],
494
            unnamedArgumentList: [
495
                new SlashCommandArgument('URL or ID of the YouTube video', ARGUMENT_TYPE.STRING, true, false),
496
            ],
497
        }));
498
    }
499

500
    /**
501
     * Check if the scraper is available.
502
     * @returns {Promise<boolean>}
503
     */
504
    async isAvailable() {
505
        return true;
506
    }
507

508
    /**
509
     * Parse the ID of a YouTube video from a URL.
510
     * @param {string} url URL of the YouTube video
511
     * @returns {string} ID of the YouTube video
512
     */
513
    parseId(url) {
514
        // If the URL is already an ID, return it
515
        if (/^[a-zA-Z0-9_-]{11}$/.test(url)) {
516
            return url;
517
        }
518

519
        const regex = /^.*(?:(?:youtu\.be\/|v\/|vi\/|u\/\w\/|embed\/|shorts\/)|(?:(?:watch)?\?v(?:i)?=|&v(?:i)?=))([^#&?]*).*/;
520
        const match = url.match(regex);
521
        return (match?.length && match[1] ? match[1] : url);
522
    }
523

524
    /**
525
     * Scrape transcript from a YouTube video.
526
     * @returns {Promise<File[]>} File attachments scraped from the YouTube video
527
     */
528
    async scrape() {
529
        let lang = '';
530
        const template = $(await renderExtensionTemplateAsync('attachments', 'youtube-scrape', {}));
531
        const videoUrl = await callGenericPopup(template, POPUP_TYPE.INPUT, '', { wide: false, large: false, okButton: 'Scrape', cancelButton: 'Cancel' });
532

533
        template.find('input[name="youtubeLanguageCode"]').on('input', function () {
534
            lang = String($(this).val()).trim();
535
        });
536

537
        if (!videoUrl) {
538
            return;
539
        }
540

541
        const toast = toastr.info('Working, please wait...');
542
        const { transcript, id } = await this.getScript(String(videoUrl), lang);
543
        toastr.clear(toast);
544

545
        const file = new File([transcript], `YouTube - ${id} - ${Date.now()}.txt`, { type: 'text/plain' });
546
        return [file];
547
    }
548

549
    /**
550
     * Fetches the transcript of a YouTube video.
551
     * @param {string} videoUrl Video URL or ID
552
     * @param {string} lang Video language
553
     * @returns {Promise<{ transcript: string, id: string }>} Transcript of the YouTube video with the video ID
554
     */
555
    async getScript(videoUrl, lang) {
556
        const id = this.parseId(String(videoUrl).trim());
557

558
        const result = await fetch('/api/search/transcript', {
559
            method: 'POST',
560
            headers: getRequestHeaders(),
561
            body: JSON.stringify({ id, lang }),
562
        });
563

564
        if (!result.ok) {
565
            const error = await result.text();
566
            throw new Error(error);
567
        }
568

569
        const transcript = await result.text();
570
        return { transcript, id };
571
    }
572
}
573

574
export async function initScrapers() {
575
    await ScraperManager.registerDataBankScraper(new FileScraper());
576
    await ScraperManager.registerDataBankScraper(new Notepad());
577
    await ScraperManager.registerDataBankScraper(new WebScraper());
578
    await ScraperManager.registerDataBankScraper(new MediaWikiScraper());
579
    await ScraperManager.registerDataBankScraper(new FandomScraper());
580
    await ScraperManager.registerDataBankScraper(new YouTubeScraper());
581
}