Merge pull request #2651 from SillyTavern/webgpu-summary Summarize with WebLLM extension

78bee4631d7a33550d57b02c11f91f3da58af887

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: 33916aa493399bebe2d3e327ce91c5abdae295aa 53c24174e7906b278c82630af20e1c73bbb85641
9 files changed, +310 -52Ignore whitespace
.eslintrc.js+1 -0
@@ -55,6 +55,7 @@ module.exports = {
5555 isProbablyReaderable: 'readonly',
5656 ePub: 'readonly',
5757 diff_match_patch: 'readonly',
58+ SillyTavern: 'readonly',
5859 },
5960 },
6061 ],
public/global.d.ts+5 -0
@@ -14,6 +14,11 @@ declare var isProbablyReaderable;
1414declare var ePub;
1515declare var ai;
1616
17+declare var SillyTavern: {
18+ getContext(): any;
19+ llm: any;
20+};
21+
1722// Jquery plugins
1823interface JQuery {
1924 nanogallery2(options?: any): JQuery;
public/scripts/extensions.js+23 -21
@@ -989,6 +989,28 @@ export async function writeExtensionField(characterId, key, value) {
989989 }
990990}
991991
992+/**
993+ * Prompts the user to enter the Git URL of the extension to import.
994+ * After obtaining the Git URL, makes a POST request to '/api/extensions/install' to import the extension.
995+ * If the extension is imported successfully, a success message is displayed.
996+ * If the extension import fails, an error message is displayed and the error is logged to the console.
997+ * After successfully importing the extension, the extension settings are reloaded and a 'EXTENSION_SETTINGS_LOADED' event is emitted.
998+ * @param {string} [suggestUrl] Suggested URL to install
999+ * @returns {Promise<void>}
1000+ */
1001+export async function openThirdPartyExtensionMenu(suggestUrl = '') {
1002+ const html = await renderTemplateAsync('installExtension');
1003+ const input = await callGenericPopup(html, POPUP_TYPE.INPUT, suggestUrl ?? '');
1004+
1005+ if (!input) {
1006+ console.debug('Extension install cancelled');
1007+ return;
1008+ }
1009+
1010+ const url = String(input).trim();
1011+ await installExtension(url);
1012+}
1013+
9921014jQuery(async function () {
9931015 await addExtensionsButtonAndMenu();
9941016 $('#extensionsMenuButton').css('display', 'flex');
@@ -1004,28 +1026,8 @@ jQuery(async function () {
10041026
10051027 /**
10061028 * Handles the click event for the third-party extension import button.
1007- * Prompts the user to enter the Git URL of the extension to import.
1008- * After obtaining the Git URL, makes a POST request to '/api/extensions/install' to import the extension.
1009- * If the extension is imported successfully, a success message is displayed.
1010- * If the extension import fails, an error message is displayed and the error is logged to the console.
1011- * After successfully importing the extension, the extension settings are reloaded and a 'EXTENSION_SETTINGS_LOADED' event is emitted.
10121029 *
10131030 * @listens #third_party_extension_button#click - The click event of the '#third_party_extension_button' element.
10141031 */
10151032 $('#third_party_extension_button').on('click', async () => {openThirdPartyExtensionMenu());
1016- const html = `<h3>Enter the Git URL of the extension to install</h3>
1017- <br>
1018- <p><b>Disclaimer:</b> Please be aware that using external extensions can have unintended side effects and may pose security risks. Always make sure you trust the source before importing an extension. We are not responsible for any damage caused by third-party extensions.</p>
1019- <br>
1020- <p>Example: <tt> https://github.com/author/extension-name </tt></p>`;
1021- const input = await callGenericPopup(html, POPUP_TYPE.INPUT, '');
1022-
1023- if (!input) {
1024- console.debug('Extension install cancelled');
1025- return;
1026- }
1027-
1028- const url = String(input).trim();
1029- await installExtension(url);
1030- });
10311033});
public/scripts/extensions/memory/index.js+141 -26
@@ -25,6 +25,7 @@ import { SlashCommandParser } from '../../slash-commands/SlashCommandParser.js';
2525import { SlashCommand } from '../../slash-commands/SlashCommand.js';
2626import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from '../../slash-commands/SlashCommandArgument.js';
2727import { MacrosParser } from '../../macros.js';
28+import { countWebLlmTokens, generateWebLlmChatPrompt, getWebLlmContextSize, isWebLlmSupported } from '../shared.js';
2829export { MODULE_NAME };
2930
3031const MODULE_NAME = '1_memory';
@@ -36,6 +37,41 @@ let lastMessageHash = null;
3637let lastMessageId = null;
3738let inApiCall = false;
3839
40+/**
41+ * Count the number of tokens in the provided text.
42+ * @param {string} text Text to count tokens for
43+ * @param {number} padding Number of additional tokens to add to the count
44+ * @returns {Promise<number>} Number of tokens in the text
45+ */
46+async function countSourceTokens(text, padding = 0) {
47+ if (extension_settings.memory.source === summary_sources.webllm) {
48+ const count = await countWebLlmTokens(text);
49+ return count + padding;
50+ }
51+
52+ if (extension_settings.memory.source === summary_sources.extras) {
53+ const count = getTextTokens(tokenizers.GPT2, text).length;
54+ return count + padding;
55+ }
56+
57+ return await getTokenCountAsync(text, padding);
58+}
59+
60+async function getSourceContextSize() {
61+ const overrideLength = extension_settings.memory.overrideResponseLength;
62+
63+ if (extension_settings.memory.source === summary_sources.webllm) {
64+ const maxContext = await getWebLlmContextSize();
65+ return overrideLength > 0 ? (maxContext - overrideLength) : Math.round(maxContext * 0.75);
66+ }
67+
68+ if (extension_settings.source === summary_sources.extras) {
69+ return 1024 - 64;
70+ }
71+
72+ return getMaxContextSize(overrideLength);
73+}
74+
3975const formatMemoryValue = function (value) {
4076 if (!value) {
4177 return '';
@@ -55,6 +91,7 @@ const saveChatDebounced = debounce(() => getContext().saveChat(), debounce_timeo
5591const summary_sources = {
5692 'extras': 'extras',
5793 'main': 'main',
94+ 'webllm': 'webllm',
5895};
5996
6097const prompt_builders = {
@@ -130,12 +167,12 @@ function loadSettings() {
130167
131168async function onPromptForceWordsAutoClick() {
132169 const context = getContext();
133170 const maxPromptLength = getMaxContextSizeawait getSourceContextSize(extension_settings.memory.overrideResponseLength);
134171 const chat = context.chat;
135172 const allMessages = chat.filter(m => !m.is_system && m.mes).map(m => m.mes);
136173 const messagesWordCount = allMessages.map(m => extractAllWords(m)).flat().length;
137174 const averageMessageWordCount = messagesWordCount / allMessages.length;
138175 const tokensPerWord = await getTokenCountAsynccountSourceTokens(allMessages.join('\n')) / messagesWordCount;
139176 const wordsPerToken = 1 / tokensPerWord;
140177 const maxPromptLengthWords = Math.round(maxPromptLength * wordsPerToken);
141178 // How many words should pass so that messages will start be dropped out of context;
@@ -168,15 +205,15 @@ async function onPromptForceWordsAutoClick() {
168205
169206async function onPromptIntervalAutoClick() {
170207 const context = getContext();
171208 const maxPromptLength = getMaxContextSizeawait getSourceContextSize(extension_settings.memory.overrideResponseLength);
172209 const chat = context.chat;
173210 const allMessages = chat.filter(m => !m.is_system && m.mes).map(m => m.mes);
174211 const messagesWordCount = allMessages.map(m => extractAllWords(m)).flat().length;
175212 const messagesTokenCount = await getTokenCountAsynccountSourceTokens(allMessages.join('\n'));
176213 const tokensPerWord = messagesTokenCount / messagesWordCount;
177214 const averageMessageTokenCount = messagesTokenCount / allMessages.length;
178215 const targetSummaryTokens = Math.round(extension_settings.memory.promptWords * tokensPerWord);
179216 const promptTokens = await getTokenCountAsynccountSourceTokens(extension_settings.memory.prompt);
180217 const promptAllowance = maxPromptLength - promptTokens - targetSummaryTokens;
181218 const maxMessagesPerSummary = extension_settings.memory.maxMessagesPerRequest || 0;
182219 const averageMessagesPerPrompt = Math.floor(promptAllowance / averageMessageTokenCount);
@@ -213,8 +250,8 @@ function onSummarySourceChange(event) {
213250
214251function switchSourceControls(value) {
215252 $('#memory_settings [data-summary-source]').each((_, element) => {
216253 const source = $(element).datadataset.summarySource.split('summary-source,').map(s => s.trim());
217254 $(element).toggle(source === .includes(value));
218255 });
219256}
220257
@@ -353,10 +390,13 @@ function getIndexOfLatestChatSummary(chat) {
353390
354391async function onChatEvent() {
355392 // Module not enabled
356393 if (extension_settings.memory.source === summary_sources.extras && !modules.includes('summarize')) {
357- if (!modules.includes('summarize')) {
394+ return;
358- return;
395+ }
359- }
396+
397+ // WebLLM is not supported
398+ if (extension_settings.memory.source === summary_sources.webllm && !isWebLlmSupported()) {
399+ return;
360400 }
361401
362402 const context = getContext();
@@ -431,8 +471,12 @@ async function forceSummarizeChat() {
431471 return '';
432472 }
433473
434474 const toast = toastr.info('Summarizing chat...', 'Please wait', { timeOut: 0, extendedTimeOut: 0 });
435- const value = await summarizeChatMain(context, true, skipWIAN);
475+ const value = extension_settings.memory.source === summary_sources.main
476+ ? await summarizeChatMain(context, true, skipWIAN)
477+ : await summarizeChatWebLLM(context, true);
478+
479+ toastr.clear(toast);
436480
437481 if (!value) {
438482 toastr.warning('Failed to summarize chat');
@@ -464,6 +508,11 @@ async function summarizeCallback(args, text) {
464508 return await callExtrasSummarizeAPI(text);
465509 case summary_sources.main:
466510 return await generateRaw(text, '', false, false, prompt, extension_settings.memory.overrideResponseLength);
511+ case summary_sources.webllm: {
512+ const messages = [{ role: 'system', content: prompt }, { role: 'user', content: text }].filter(m => m.content);
513+ const params = extension_settings.memory.overrideResponseLength > 0 ? { max_tokens: extension_settings.memory.overrideResponseLength } : {};
514+ return await generateWebLlmChatPrompt(messages, params);
515+ }
467516 default:
468517 toastr.warning('Invalid summarization source specified');
469518 return '';
@@ -484,16 +533,25 @@ async function summarizeChat(context) {
484533 case summary_sources.main:
485534 await summarizeChatMain(context, false, skipWIAN);
486535 break;
536+ case summary_sources.webllm:
537+ await summarizeChatWebLLM(context, false);
538+ break;
487539 default:
488540 break;
489541 }
490542}
491543
492-async function summarizeChatMain(context, force, skipWIAN) {
544+/**
493-
545+ * Check if the chat should be summarized based on the current conditions.
546+ * Return summary prompt if it should be summarized.
547+ * @param {any} context ST context
548+ * @param {boolean} force Summarize the chat regardless of the conditions
549+ * @returns {Promise<string>} Summary prompt or empty string
550+ */
551+async function getSummaryPromptForNow(context, force) {
494552 if (extension_settings.memory.promptInterval === 0 && !force) {
495553 console.debug('Prompt interval is set to 0, skipping summarization');
496554 return '';
497555 }
498556
499557 try {
@@ -505,17 +563,17 @@ async function summarizeChatMain(context, force, skipWIAN) {
505563 waitUntilCondition(() => is_send_press === false, 30000, 100);
506564 } catch {
507565 console.debug('Timeout waiting for is_send_press');
508566 return '';
509567 }
510568
511569 if (!context.chat.length) {
512570 console.debug('No messages in chat to summarize');
513571 return '';
514572 }
515573
516574 if (context.chat.length < extension_settings.memory.promptInterval && !force) {
517575 console.debug(`Not enough messages in chat to summarize (chat: ${context.chat.length}, interval: ${extension_settings.memory.promptInterval})`);
518576 return '';
519577 }
520578
521579 let messagesSinceLastSummary = 0;
@@ -539,7 +597,7 @@ async function summarizeChatMain(context, force, skipWIAN) {
539597
540598 if (!conditionSatisfied && !force) {
541599 console.debug(`Summary conditions not satisfied (messages: ${messagesSinceLastSummary}, interval: ${extension_settings.memory.promptInterval}, words: ${wordsSinceLastSummary}, force words: ${extension_settings.memory.promptForceWords})`);
542600 return '';
543601 }
544602
545603 console.log('Summarizing chat, messages since last summary: ' + messagesSinceLastSummary, 'words since last summary: ' + wordsSinceLastSummary);
@@ -547,6 +605,63 @@ async function summarizeChatMain(context, force, skipWIAN) {
547605
548606 if (!prompt) {
549607 console.debug('Summarization prompt is empty. Skipping summarization.');
608+ return '';
609+ }
610+
611+ return prompt;
612+}
613+
614+async function summarizeChatWebLLM(context, force) {
615+ if (!isWebLlmSupported()) {
616+ return;
617+ }
618+
619+ const prompt = await getSummaryPromptForNow(context, force);
620+
621+ if (!prompt) {
622+ return;
623+ }
624+
625+ const { rawPrompt, lastUsedIndex } = await getRawSummaryPrompt(context, prompt);
626+
627+ if (lastUsedIndex === null || lastUsedIndex === -1) {
628+ if (force) {
629+ toastr.info('To try again, remove the latest summary.', 'No messages found to summarize');
630+ }
631+
632+ return null;
633+ }
634+
635+ const messages = [
636+ { role: 'system', content: prompt },
637+ { role: 'user', content: rawPrompt },
638+ ];
639+
640+ const params = {};
641+
642+ if (extension_settings.memory.overrideResponseLength > 0) {
643+ params.max_tokens = extension_settings.memory.overrideResponseLength;
644+ }
645+
646+ const summary = await generateWebLlmChatPrompt(messages, params);
647+ const newContext = getContext();
648+
649+ // something changed during summarization request
650+ if (newContext.groupId !== context.groupId ||
651+ newContext.chatId !== context.chatId ||
652+ (!newContext.groupId && (newContext.characterId !== context.characterId))) {
653+ console.log('Context changed, summary discarded');
654+ return;
655+ }
656+
657+ setMemoryContext(summary, true, lastUsedIndex);
658+ return summary;
659+}
660+
661+async function summarizeChatMain(context, force, skipWIAN) {
662+ const prompt = await getSummaryPromptForNow(context, force);
663+
664+ if (!prompt) {
550665 return;
551666 }
552667
@@ -634,7 +749,7 @@ async function getRawSummaryPrompt(context, prompt) {
634749 chat.pop(); // We always exclude the last message from the buffer
635750 const chatBuffer = [];
636751 const PADDING = 64;
637752 const PROMPT_SIZE = getMaxContextSizeawait getSourceContextSize(extension_settings.memory.overrideResponseLength);
638753 let latestUsedMessage = null;
639754
640755 for (let index = latestSummaryIndex + 1; index < chat.length; index++) {
@@ -651,7 +766,7 @@ async function getRawSummaryPrompt(context, prompt) {
651766 const entry = `${message.name}:\n${message.mes}`;
652767 chatBuffer.push(entry);
653768
654769 const tokens = await getTokenCountAsynccountSourceTokens(getMemoryString(true), PADDING);
655770
656771 if (tokens > PROMPT_SIZE) {
657772 chatBuffer.pop();
@@ -680,7 +795,7 @@ async function summarizeChatExtras(context) {
680795 const reversedChat = chat.slice().reverse();
681796 reversedChat.shift();
682797 const memoryBuffer = [];
683798 const CONTEXT_SIZE = 1024 -await 64getSourceContextSize();
684799
685800 for (const message of reversedChat) {
686801 // we reached the point of latest memory
@@ -698,14 +813,14 @@ async function summarizeChatExtras(context) {
698813 memoryBuffer.push(entry);
699814
700815 // check if token limit was reached
701816 const tokens = getTextTokens(tokenizers.GPT2,await countSourceTokens(getMemoryString()).length;
702817 if (tokens >= CONTEXT_SIZE) {
703818 break;
704819 }
705820 }
706821
707822 const resultingString = getMemoryString();
708823 const resultingTokens = getTextTokens(tokenizers.GPT2,await countSourceTokens(resultingString).length;
709824
710825 if (!resultingString || resultingTokens < CONTEXT_SIZE) {
711826 console.debug('Not enough context to summarize');
@@ -933,7 +1048,7 @@ jQuery(async function () {
9331048 name: 'summarize',
9341049 callback: summarizeCallback,
9351050 namedArgumentList: [
9361051 new SlashCommandNamedArgument('source', 'API to use for summarization', [ARGUMENT_TYPE.STRING], false, false, '', ['main', 'extras']Object.values(summary_sources)),
9371052 SlashCommandNamedArgument.fromProps({
9381053 name: 'prompt',
9391054 description: 'prompt to use for summarization',
public/scripts/extensions/memory/settings.html+4 -3
@@ -13,6 +13,7 @@
1313 <select id="summary_source">
1414 <option value="main" data-i18n="ext_sum_main_api">Main API</option>
1515 <option value="extras">Extras API</option>
16+ <option value="webllm" data-i18n="ext_sum_webllm">WebLLM Extension</option>
1617 </select><br>
1718
1819 <div class="flex-container justifyspacebetween alignitemscenter">
@@ -24,7 +25,7 @@
2425
2526 <textarea id="memory_contents" class="text_pole textarea_compact" rows="6" data-i18n="[placeholder]ext_sum_memory_placeholder" placeholder="Summary will be generated here..."></textarea>
2627 <div class="memory_contents_controls">
2728 <div id="memory_force_summarize" data-summary-source="main,webllm" class="menu_button menu_button_icon" title="Trigger a summary update right now." data-i18n="[title]ext_sum_force_tip">
2829 <i class="fa-solid fa-database"></i>
2930 <span data-i18n="ext_sum_force_text">Summarize now</span>
3031 </div>
@@ -58,7 +59,7 @@
5859 <span data-i18n="ext_sum_prompt_builder_3">Classic, blocking</span>
5960 </label>
6061 </div>
6162 <div data-summary-source="main,webllm">
6263 <label for="memory_prompt" class="title_restorable">
6364 <span data-i18n="Summary Prompt">Summary Prompt</span>
6465 <div id="memory_prompt_restore" data-i18n="[title]ext_sum_restore_default_prompt_tip" title="Restore default prompt" class="right_menu_button">
@@ -74,7 +75,7 @@
7475 </label>
7576 <input id="memory_override_response_length" type="range" value="{{defaultSettings.overrideResponseLength}}" min="{{defaultSettings.overrideResponseLengthMin}}" max="{{defaultSettings.overrideResponseLengthMax}}" step="{{defaultSettings.overrideResponseLengthStep}}" />
7677 <label for="memory_max_messages_per_request">
7778 <span data-i18n="ext_sum_raw_max_msg">[Raw/WebLLM] Max messages per request</span> (<span id="memory_max_messages_per_request_value"></span>)
7879 <small class="memory_disabled_hint" data-i18n="ext_sum_0_unlimited">0 = unlimited</small>
7980 </label>
8081 <input id="memory_max_messages_per_request" type="range" value="{{defaultSettings.maxMessagesPerRequest}}" min="{{defaultSettings.maxMessagesPerRequestMin}}" max="{{defaultSettings.maxMessagesPerRequestMax}}" step="{{defaultSettings.maxMessagesPerRequestStep}}" />
public/scripts/extensions/shared.js+84 -1
@@ -1,5 +1,5 @@
11import { getRequestHeaders } from '../../script.js';
22import { extension_settings, openThirdPartyExtensionMenu } from '../extensions.js';
33import { oai_settings } from '../openai.js';
44import { SECRET_KEYS, secret_state } from '../secrets.js';
55import { textgen_types, textgenerationwebui_settings } from '../textgen-settings.js';
@@ -176,3 +176,86 @@ function throwIfInvalidModel(useReverseProxy) {
176176 throw new Error('Custom API URL is not set.');
177177 }
178178}
179+
180+/**
181+ * Check if the WebLLM extension is installed and supported.
182+ * @returns {boolean} Whether the extension is installed and supported
183+ */
184+export function isWebLlmSupported() {
185+ if (!('gpu' in navigator)) {
186+ const warningKey = 'webllm_browser_warning_shown';
187+ if (!sessionStorage.getItem(warningKey)) {
188+ toastr.error('Your browser does not support the WebGPU API. Please use a different browser.', 'WebLLM', {
189+ preventDuplicates: true,
190+ timeOut: 0,
191+ extendedTimeOut: 0,
192+ });
193+ sessionStorage.setItem(warningKey, '1');
194+ }
195+ return false;
196+ }
197+
198+ if (!('llm' in SillyTavern)) {
199+ const warningKey = 'webllm_extension_warning_shown';
200+ if (!sessionStorage.getItem(warningKey)) {
201+ toastr.error('WebLLM extension is not installed. Click here to install it.', 'WebLLM', {
202+ timeOut: 0,
203+ extendedTimeOut: 0,
204+ preventDuplicates: true,
205+ onclick: () => openThirdPartyExtensionMenu('https://github.com/SillyTavern/Extension-WebLLM'),
206+ });
207+ sessionStorage.setItem(warningKey, '1');
208+ }
209+ return false;
210+ }
211+
212+ return true;
213+}
214+
215+/**
216+ * Generates text in response to a chat prompt using WebLLM.
217+ * @param {any[]} messages Messages to use for generating
218+ * @param {object} params Additional parameters
219+ * @returns {Promise<string>} Generated response
220+ */
221+export async function generateWebLlmChatPrompt(messages, params = {}) {
222+ if (!isWebLlmSupported()) {
223+ throw new Error('WebLLM extension is not installed.');
224+ }
225+
226+ console.debug('WebLLM chat completion request:', messages, params);
227+ const engine = SillyTavern.llm;
228+ const response = await engine.generateChatPrompt(messages, params);
229+ console.debug('WebLLM chat completion response:', response);
230+ return response;
231+}
232+
233+/**
234+ * Counts the number of tokens in the provided text using WebLLM's default model.
235+ * @param {string} text Text to count tokens in
236+ * @returns {Promise<number>} Number of tokens in the text
237+ */
238+export async function countWebLlmTokens(text) {
239+ if (!isWebLlmSupported()) {
240+ throw new Error('WebLLM extension is not installed.');
241+ }
242+
243+ const engine = SillyTavern.llm;
244+ const response = await engine.countTokens(text);
245+ return response;
246+}
247+
248+/**
249+ * Gets the size of the context in the WebLLM's default model.
250+ * @returns {Promise<number>} Size of the context in the WebLLM model
251+ */
252+export async function getWebLlmContextSize() {
253+ if (!isWebLlmSupported()) {
254+ throw new Error('WebLLM extension is not installed.');
255+ }
256+
257+ const engine = SillyTavern.llm;
258+ await engine.loadModel();
259+ const model = await engine.getCurrentModelInfo();
260+ return model?.context_size;
261+}
public/scripts/extensions/vectors/index.js+43 -0
@@ -31,6 +31,12 @@ import { SlashCommandParser } from '../../slash-commands/SlashCommandParser.js';
3131import { SlashCommand } from '../../slash-commands/SlashCommand.js';
3232import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from '../../slash-commands/SlashCommandArgument.js';
3333import { callGenericPopup, POPUP_RESULT, POPUP_TYPE } from '../../popup.js';
34+import { generateWebLlmChatPrompt, isWebLlmSupported } from '../shared.js';
35+
36+/**
37+ * @typedef {object} HashedMessage
38+ * @property {string} text - The hashed message text
39+ */
3440
3541const MODULE_NAME = 'vectors';
3642
@@ -192,6 +198,11 @@ function splitByChunks(items) {
192198 return chunkedItems;
193199}
194200
201+/**
202+ * Summarizes messages using the Extras API method.
203+ * @param {HashedMessage[]} hashedMessages Array of hashed messages
204+ * @returns {Promise<HashedMessage[]>} Summarized messages
205+ */
195206async function summarizeExtra(hashedMessages) {
196207 for (const element of hashedMessages) {
197208 try {
@@ -223,6 +234,11 @@ async function summarizeExtra(hashedMessages) {
223234 return hashedMessages;
224235}
225236
237+/**
238+ * Summarizes messages using the main API method.
239+ * @param {HashedMessage[]} hashedMessages Array of hashed messages
240+ * @returns {Promise<HashedMessage[]>} Summarized messages
241+ */
226242async function summarizeMain(hashedMessages) {
227243 for (const element of hashedMessages) {
228244 element.text = await generateRaw(element.text, '', false, false, settings.summary_prompt);
@@ -231,12 +247,39 @@ async function summarizeMain(hashedMessages) {
231247 return hashedMessages;
232248}
233249
250+/**
251+ * Summarizes messages using WebLLM.
252+ * @param {HashedMessage[]} hashedMessages Array of hashed messages
253+ * @returns {Promise<HashedMessage[]>} Summarized messages
254+ */
255+async function summarizeWebLLM(hashedMessages) {
256+ if (!isWebLlmSupported()) {
257+ console.warn('Vectors: WebLLM is not supported');
258+ return hashedMessages;
259+ }
260+
261+ for (const element of hashedMessages) {
262+ const messages = [{ role:'system', content: settings.summary_prompt }, { role:'user', content: element.text }];
263+ element.text = await generateWebLlmChatPrompt(messages);
264+ }
265+
266+ return hashedMessages;
267+}
268+
269+/**
270+ * Summarizes messages using the chosen method.
271+ * @param {HashedMessage[]} hashedMessages Array of hashed messages
272+ * @param {string} endpoint Type of endpoint to use
273+ * @returns {Promise<HashedMessage[]>} Summarized messages
274+ */
234275async function summarize(hashedMessages, endpoint = 'main') {
235276 switch (endpoint) {
236277 case 'main':
237278 return await summarizeMain(hashedMessages);
238279 case 'extras':
239280 return await summarizeExtra(hashedMessages);
281+ case 'webllm':
282+ return await summarizeWebLLM(hashedMessages);
240283 default:
241284 console.error('Unsupported endpoint', endpoint);
242285 }
public/scripts/extensions/vectors/settings.html+2 -1
@@ -374,10 +374,11 @@
374374 <select id="vectors_summary_source" class="text_pole">
375375 <option value="main" data-i18n="Main API">Main API</option>
376376 <option value="extras" data-i18n="Extras API">Extras API</option>
377+ <option value="webllm" data-i18n="WebLLM Extension">WebLLM Extension</option>
377378 </select>
378379
379380 <label for="vectors_summary_prompt" title="Summary Prompt:">Summary Prompt:</label>
380381 <small data-i18n="Only used when Main API or WebLLM Extension is selected.">Only used when Main API or WebLLM Extension is selected.</small>
381382 <textarea id="vectors_summary_prompt" class="text_pole textarea_compact" rows="6" placeholder="This prompt will be sent to AI to request the summary generation."></textarea>
382383 </div>
383384 </div>
public/scripts/templates/installExtension.html+7 -0
@@ -0,0 +1,7 @@
1+<h3>Enter the Git URL of the extension to install</h3>
2+<br>
3+<p><b>Disclaimer:</b> Please be aware that using external extensions can have unintended side effects and may pose
4+ security risks. Always make sure you trust the source before importing an extension. We are not responsible for any
5+ damage caused by third-party extensions.</p>
6+<br>
7+<p>Example: <tt> https://github.com/author/extension-name </tt></p>