Merge pull request #2651 from SillyTavern/webgpu-summary Summarize with WebLLM extension

78bee4631d7a33550d57b02c11f91f3da58af887

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
9 files changed, +308 -50Showing whitespace changes
.eslintrc.js+1 -0
@@ -55,6 +55,7 @@ module.exports = {
55 isProbablyReaderable: 'readonly',55 isProbablyReaderable: 'readonly',
56 ePub: 'readonly',56 ePub: 'readonly',
57 diff_match_patch: 'readonly',57 diff_match_patch: 'readonly',
58 SillyTavern: 'readonly',
58 },59 },
59 },60 },
60 ],61 ],
public/global.d.ts+5 -0
@@ -14,6 +14,11 @@ declare var isProbablyReaderable;
14declare var ePub;14declare var ePub;
15declare var ai;15declare var ai;
1616
17declare var SillyTavern: {
18 getContext(): any;
19 llm: any;
20};
21
17// Jquery plugins22// Jquery plugins
18interface JQuery {23interface JQuery {
19 nanogallery2(options?: any): JQuery;24 nanogallery2(options?: any): JQuery;
public/scripts/extensions.js+23 -21
@@ -989,6 +989,28 @@ export async function writeExtensionField(characterId, key, value) {
989 }989 }
990}990}
991991
992/**
993 * Prompts the user to enter the Git URL of the extension to import.
994 * After obtaining the Git URL, makes a POST request to '/api/extensions/install' to import the extension.
995 * If the extension is imported successfully, a success message is displayed.
996 * If the extension import fails, an error message is displayed and the error is logged to the console.
997 * After successfully importing the extension, the extension settings are reloaded and a 'EXTENSION_SETTINGS_LOADED' event is emitted.
998 * @param {string} [suggestUrl] Suggested URL to install
999 * @returns {Promise<void>}
1000 */
1001export async function openThirdPartyExtensionMenu(suggestUrl = '') {
1002 const html = await renderTemplateAsync('installExtension');
1003 const input = await callGenericPopup(html, POPUP_TYPE.INPUT, suggestUrl ?? '');
1004
1005 if (!input) {
1006 console.debug('Extension install cancelled');
1007 return;
1008 }
1009
1010 const url = String(input).trim();
1011 await installExtension(url);
1012}
1013
992jQuery(async function () {1014jQuery(async function () {
993 await addExtensionsButtonAndMenu();1015 await addExtensionsButtonAndMenu();
994 $('#extensionsMenuButton').css('display', 'flex');1016 $('#extensionsMenuButton').css('display', 'flex');
@@ -1004,28 +1026,8 @@ jQuery(async function () {
10041026
1005 /**1027 /**
1006 * Handles the click event for the third-party extension import button.1028 * Handles the click event for the third-party extension import button.
1007 * Prompts the user to enter the Git URL of the extension to import.
1008 * After obtaining the Git URL, makes a POST request to '/api/extensions/install' to import the extension.
1009 * If the extension is imported successfully, a success message is displayed.
1010 * If the extension import fails, an error message is displayed and the error is logged to the console.
1011 * After successfully importing the extension, the extension settings are reloaded and a 'EXTENSION_SETTINGS_LOADED' event is emitted.
1012 *1029 *
1013 * @listens #third_party_extension_button#click - The click event of the '#third_party_extension_button' element.1030 * @listens #third_party_extension_button#click - The click event of the '#third_party_extension_button' element.
1014 */1031 */
1015 $('#third_party_extension_button').on('click', async () => {1032 $('#third_party_extension_button').on('click', () => openThirdPartyExtensionMenu());
1016 const html = `<h3>Enter the Git URL of the extension to install</h3>
1017 <br>
1018 <p><b>Disclaimer:</b> Please be aware that using external extensions can have unintended side effects and may pose security risks. Always make sure you trust the source before importing an extension. We are not responsible for any damage caused by third-party extensions.</p>
1019 <br>
1020 <p>Example: <tt> https://github.com/author/extension-name </tt></p>`;
1021 const input = await callGenericPopup(html, POPUP_TYPE.INPUT, '');
1022
1023 if (!input) {
1024 console.debug('Extension install cancelled');
1025 return;
1026 }
1027
1028 const url = String(input).trim();
1029 await installExtension(url);
1030 });
1031});1033});
public/scripts/extensions/memory/index.js+139 -24
@@ -25,6 +25,7 @@ import { SlashCommandParser } from '../../slash-commands/SlashCommandParser.js';
25import { SlashCommand } from '../../slash-commands/SlashCommand.js';25import { SlashCommand } from '../../slash-commands/SlashCommand.js';
26import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from '../../slash-commands/SlashCommandArgument.js';26import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from '../../slash-commands/SlashCommandArgument.js';
27import { MacrosParser } from '../../macros.js';27import { MacrosParser } from '../../macros.js';
28import { countWebLlmTokens, generateWebLlmChatPrompt, getWebLlmContextSize, isWebLlmSupported } from '../shared.js';
28export { MODULE_NAME };29export { MODULE_NAME };
2930
30const MODULE_NAME = '1_memory';31const MODULE_NAME = '1_memory';
@@ -36,6 +37,41 @@ let lastMessageHash = null;
36let lastMessageId = null;37let lastMessageId = null;
37let inApiCall = false;38let inApiCall = false;
3839
40/**
41 * Count the number of tokens in the provided text.
42 * @param {string} text Text to count tokens for
43 * @param {number} padding Number of additional tokens to add to the count
44 * @returns {Promise<number>} Number of tokens in the text
45 */
46async function countSourceTokens(text, padding = 0) {
47 if (extension_settings.memory.source === summary_sources.webllm) {
48 const count = await countWebLlmTokens(text);
49 return count + padding;
50 }
51
52 if (extension_settings.memory.source === summary_sources.extras) {
53 const count = getTextTokens(tokenizers.GPT2, text).length;
54 return count + padding;
55 }
56
57 return await getTokenCountAsync(text, padding);
58}
59
60async function getSourceContextSize() {
61 const overrideLength = extension_settings.memory.overrideResponseLength;
62
63 if (extension_settings.memory.source === summary_sources.webllm) {
64 const maxContext = await getWebLlmContextSize();
65 return overrideLength > 0 ? (maxContext - overrideLength) : Math.round(maxContext * 0.75);
66 }
67
68 if (extension_settings.source === summary_sources.extras) {
69 return 1024 - 64;
70 }
71
72 return getMaxContextSize(overrideLength);
73}
74
39const formatMemoryValue = function (value) {75const formatMemoryValue = function (value) {
40 if (!value) {76 if (!value) {
41 return '';77 return '';
@@ -55,6 +91,7 @@ const saveChatDebounced = debounce(() => getContext().saveChat(), debounce_timeo
55const summary_sources = {91const summary_sources = {
56 'extras': 'extras',92 'extras': 'extras',
57 'main': 'main',93 'main': 'main',
94 'webllm': 'webllm',
58};95};
5996
60const prompt_builders = {97const prompt_builders = {
@@ -130,12 +167,12 @@ function loadSettings() {
130167
131async function onPromptForceWordsAutoClick() {168async function onPromptForceWordsAutoClick() {
132 const context = getContext();169 const context = getContext();
133 const maxPromptLength = getMaxContextSize(extension_settings.memory.overrideResponseLength);170 const maxPromptLength = await getSourceContextSize();
134 const chat = context.chat;171 const chat = context.chat;
135 const allMessages = chat.filter(m => !m.is_system && m.mes).map(m => m.mes);172 const allMessages = chat.filter(m => !m.is_system && m.mes).map(m => m.mes);
136 const messagesWordCount = allMessages.map(m => extractAllWords(m)).flat().length;173 const messagesWordCount = allMessages.map(m => extractAllWords(m)).flat().length;
137 const averageMessageWordCount = messagesWordCount / allMessages.length;174 const averageMessageWordCount = messagesWordCount / allMessages.length;
138 const tokensPerWord = await getTokenCountAsync(allMessages.join('\n')) / messagesWordCount;175 const tokensPerWord = await countSourceTokens(allMessages.join('\n')) / messagesWordCount;
139 const wordsPerToken = 1 / tokensPerWord;176 const wordsPerToken = 1 / tokensPerWord;
140 const maxPromptLengthWords = Math.round(maxPromptLength * wordsPerToken);177 const maxPromptLengthWords = Math.round(maxPromptLength * wordsPerToken);
141 // How many words should pass so that messages will start be dropped out of context;178 // How many words should pass so that messages will start be dropped out of context;
@@ -168,15 +205,15 @@ async function onPromptForceWordsAutoClick() {
168205
169async function onPromptIntervalAutoClick() {206async function onPromptIntervalAutoClick() {
170 const context = getContext();207 const context = getContext();
171 const maxPromptLength = getMaxContextSize(extension_settings.memory.overrideResponseLength);208 const maxPromptLength = await getSourceContextSize();
172 const chat = context.chat;209 const chat = context.chat;
173 const allMessages = chat.filter(m => !m.is_system && m.mes).map(m => m.mes);210 const allMessages = chat.filter(m => !m.is_system && m.mes).map(m => m.mes);
174 const messagesWordCount = allMessages.map(m => extractAllWords(m)).flat().length;211 const messagesWordCount = allMessages.map(m => extractAllWords(m)).flat().length;
175 const messagesTokenCount = await getTokenCountAsync(allMessages.join('\n'));212 const messagesTokenCount = await countSourceTokens(allMessages.join('\n'));
176 const tokensPerWord = messagesTokenCount / messagesWordCount;213 const tokensPerWord = messagesTokenCount / messagesWordCount;
177 const averageMessageTokenCount = messagesTokenCount / allMessages.length;214 const averageMessageTokenCount = messagesTokenCount / allMessages.length;
178 const targetSummaryTokens = Math.round(extension_settings.memory.promptWords * tokensPerWord);215 const targetSummaryTokens = Math.round(extension_settings.memory.promptWords * tokensPerWord);
179 const promptTokens = await getTokenCountAsync(extension_settings.memory.prompt);216 const promptTokens = await countSourceTokens(extension_settings.memory.prompt);
180 const promptAllowance = maxPromptLength - promptTokens - targetSummaryTokens;217 const promptAllowance = maxPromptLength - promptTokens - targetSummaryTokens;
181 const maxMessagesPerSummary = extension_settings.memory.maxMessagesPerRequest || 0;218 const maxMessagesPerSummary = extension_settings.memory.maxMessagesPerRequest || 0;
182 const averageMessagesPerPrompt = Math.floor(promptAllowance / averageMessageTokenCount);219 const averageMessagesPerPrompt = Math.floor(promptAllowance / averageMessageTokenCount);
@@ -213,8 +250,8 @@ function onSummarySourceChange(event) {
213250
214function switchSourceControls(value) {251function switchSourceControls(value) {
215 $('#memory_settings [data-summary-source]').each((_, element) => {252 $('#memory_settings [data-summary-source]').each((_, element) => {
216 const source = $(element).data('summary-source');253 const source = element.dataset.summarySource.split(',').map(s => s.trim());
217 $(element).toggle(source === value);254 $(element).toggle(source.includes(value));
218 });255 });
219}256}
220257
@@ -353,10 +390,13 @@ function getIndexOfLatestChatSummary(chat) {
353390
354async function onChatEvent() {391async function onChatEvent() {
355 // Module not enabled392 // Module not enabled
356 if (extension_settings.memory.source === summary_sources.extras) {393 if (extension_settings.memory.source === summary_sources.extras && !modules.includes('summarize')) {
357 if (!modules.includes('summarize')) {
358 return;394 return;
359 }395 }
396
397 // WebLLM is not supported
398 if (extension_settings.memory.source === summary_sources.webllm && !isWebLlmSupported()) {
399 return;
360 }400 }
361401
362 const context = getContext();402 const context = getContext();
@@ -431,8 +471,12 @@ async function forceSummarizeChat() {
431 return '';471 return '';
432 }472 }
433473
434 toastr.info('Summarizing chat...', 'Please wait');474 const toast = toastr.info('Summarizing chat...', 'Please wait', { timeOut: 0, extendedTimeOut: 0 });
435 const value = await summarizeChatMain(context, true, skipWIAN);475 const value = extension_settings.memory.source === summary_sources.main
476 ? await summarizeChatMain(context, true, skipWIAN)
477 : await summarizeChatWebLLM(context, true);
478
479 toastr.clear(toast);
436480
437 if (!value) {481 if (!value) {
438 toastr.warning('Failed to summarize chat');482 toastr.warning('Failed to summarize chat');
@@ -464,6 +508,11 @@ async function summarizeCallback(args, text) {
464 return await callExtrasSummarizeAPI(text);508 return await callExtrasSummarizeAPI(text);
465 case summary_sources.main:509 case summary_sources.main:
466 return await generateRaw(text, '', false, false, prompt, extension_settings.memory.overrideResponseLength);510 return await generateRaw(text, '', false, false, prompt, extension_settings.memory.overrideResponseLength);
511 case summary_sources.webllm: {
512 const messages = [{ role: 'system', content: prompt }, { role: 'user', content: text }].filter(m => m.content);
513 const params = extension_settings.memory.overrideResponseLength > 0 ? { max_tokens: extension_settings.memory.overrideResponseLength } : {};
514 return await generateWebLlmChatPrompt(messages, params);
515 }
467 default:516 default:
468 toastr.warning('Invalid summarization source specified');517 toastr.warning('Invalid summarization source specified');
469 return '';518 return '';
@@ -484,16 +533,25 @@ async function summarizeChat(context) {
484 case summary_sources.main:533 case summary_sources.main:
485 await summarizeChatMain(context, false, skipWIAN);534 await summarizeChatMain(context, false, skipWIAN);
486 break;535 break;
536 case summary_sources.webllm:
537 await summarizeChatWebLLM(context, false);
538 break;
487 default:539 default:
488 break;540 break;
489 }541 }
490}542}
491543
492async function summarizeChatMain(context, force, skipWIAN) {544/**
493545 * Check if the chat should be summarized based on the current conditions.
546 * Return summary prompt if it should be summarized.
547 * @param {any} context ST context
548 * @param {boolean} force Summarize the chat regardless of the conditions
549 * @returns {Promise<string>} Summary prompt or empty string
550 */
551async function getSummaryPromptForNow(context, force) {
494 if (extension_settings.memory.promptInterval === 0 && !force) {552 if (extension_settings.memory.promptInterval === 0 && !force) {
495 console.debug('Prompt interval is set to 0, skipping summarization');553 console.debug('Prompt interval is set to 0, skipping summarization');
496 return;554 return '';
497 }555 }
498556
499 try {557 try {
@@ -505,17 +563,17 @@ async function summarizeChatMain(context, force, skipWIAN) {
505 waitUntilCondition(() => is_send_press === false, 30000, 100);563 waitUntilCondition(() => is_send_press === false, 30000, 100);
506 } catch {564 } catch {
507 console.debug('Timeout waiting for is_send_press');565 console.debug('Timeout waiting for is_send_press');
508 return;566 return '';
509 }567 }
510568
511 if (!context.chat.length) {569 if (!context.chat.length) {
512 console.debug('No messages in chat to summarize');570 console.debug('No messages in chat to summarize');
513 return;571 return '';
514 }572 }
515573
516 if (context.chat.length < extension_settings.memory.promptInterval && !force) {574 if (context.chat.length < extension_settings.memory.promptInterval && !force) {
517 console.debug(`Not enough messages in chat to summarize (chat: ${context.chat.length}, interval: ${extension_settings.memory.promptInterval})`);575 console.debug(`Not enough messages in chat to summarize (chat: ${context.chat.length}, interval: ${extension_settings.memory.promptInterval})`);
518 return;576 return '';
519 }577 }
520578
521 let messagesSinceLastSummary = 0;579 let messagesSinceLastSummary = 0;
@@ -539,7 +597,7 @@ async function summarizeChatMain(context, force, skipWIAN) {
539597
540 if (!conditionSatisfied && !force) {598 if (!conditionSatisfied && !force) {
541 console.debug(`Summary conditions not satisfied (messages: ${messagesSinceLastSummary}, interval: ${extension_settings.memory.promptInterval}, words: ${wordsSinceLastSummary}, force words: ${extension_settings.memory.promptForceWords})`);599 console.debug(`Summary conditions not satisfied (messages: ${messagesSinceLastSummary}, interval: ${extension_settings.memory.promptInterval}, words: ${wordsSinceLastSummary}, force words: ${extension_settings.memory.promptForceWords})`);
542 return;600 return '';
543 }601 }
544602
545 console.log('Summarizing chat, messages since last summary: ' + messagesSinceLastSummary, 'words since last summary: ' + wordsSinceLastSummary);603 console.log('Summarizing chat, messages since last summary: ' + messagesSinceLastSummary, 'words since last summary: ' + wordsSinceLastSummary);
@@ -547,6 +605,63 @@ async function summarizeChatMain(context, force, skipWIAN) {
547605
548 if (!prompt) {606 if (!prompt) {
549 console.debug('Summarization prompt is empty. Skipping summarization.');607 console.debug('Summarization prompt is empty. Skipping summarization.');
608 return '';
609 }
610
611 return prompt;
612}
613
614async function summarizeChatWebLLM(context, force) {
615 if (!isWebLlmSupported()) {
616 return;
617 }
618
619 const prompt = await getSummaryPromptForNow(context, force);
620
621 if (!prompt) {
622 return;
623 }
624
625 const { rawPrompt, lastUsedIndex } = await getRawSummaryPrompt(context, prompt);
626
627 if (lastUsedIndex === null || lastUsedIndex === -1) {
628 if (force) {
629 toastr.info('To try again, remove the latest summary.', 'No messages found to summarize');
630 }
631
632 return null;
633 }
634
635 const messages = [
636 { role: 'system', content: prompt },
637 { role: 'user', content: rawPrompt },
638 ];
639
640 const params = {};
641
642 if (extension_settings.memory.overrideResponseLength > 0) {
643 params.max_tokens = extension_settings.memory.overrideResponseLength;
644 }
645
646 const summary = await generateWebLlmChatPrompt(messages, params);
647 const newContext = getContext();
648
649 // something changed during summarization request
650 if (newContext.groupId !== context.groupId ||
651 newContext.chatId !== context.chatId ||
652 (!newContext.groupId && (newContext.characterId !== context.characterId))) {
653 console.log('Context changed, summary discarded');
654 return;
655 }
656
657 setMemoryContext(summary, true, lastUsedIndex);
658 return summary;
659}
660
661async function summarizeChatMain(context, force, skipWIAN) {
662 const prompt = await getSummaryPromptForNow(context, force);
663
664 if (!prompt) {
550 return;665 return;
551 }666 }
552667
@@ -634,7 +749,7 @@ async function getRawSummaryPrompt(context, prompt) {
634 chat.pop(); // We always exclude the last message from the buffer749 chat.pop(); // We always exclude the last message from the buffer
635 const chatBuffer = [];750 const chatBuffer = [];
636 const PADDING = 64;751 const PADDING = 64;
637 const PROMPT_SIZE = getMaxContextSize(extension_settings.memory.overrideResponseLength);752 const PROMPT_SIZE = await getSourceContextSize();
638 let latestUsedMessage = null;753 let latestUsedMessage = null;
639754
640 for (let index = latestSummaryIndex + 1; index < chat.length; index++) {755 for (let index = latestSummaryIndex + 1; index < chat.length; index++) {
@@ -651,7 +766,7 @@ async function getRawSummaryPrompt(context, prompt) {
651 const entry = `${message.name}:\n${message.mes}`;766 const entry = `${message.name}:\n${message.mes}`;
652 chatBuffer.push(entry);767 chatBuffer.push(entry);
653768
654 const tokens = await getTokenCountAsync(getMemoryString(true), PADDING);769 const tokens = await countSourceTokens(getMemoryString(true), PADDING);
655770
656 if (tokens > PROMPT_SIZE) {771 if (tokens > PROMPT_SIZE) {
657 chatBuffer.pop();772 chatBuffer.pop();
@@ -680,7 +795,7 @@ async function summarizeChatExtras(context) {
680 const reversedChat = chat.slice().reverse();795 const reversedChat = chat.slice().reverse();
681 reversedChat.shift();796 reversedChat.shift();
682 const memoryBuffer = [];797 const memoryBuffer = [];
683 const CONTEXT_SIZE = 1024 - 64;798 const CONTEXT_SIZE = await getSourceContextSize();
684799
685 for (const message of reversedChat) {800 for (const message of reversedChat) {
686 // we reached the point of latest memory801 // we reached the point of latest memory
@@ -698,14 +813,14 @@ async function summarizeChatExtras(context) {
698 memoryBuffer.push(entry);813 memoryBuffer.push(entry);
699814
700 // check if token limit was reached815 // check if token limit was reached
701 const tokens = getTextTokens(tokenizers.GPT2, getMemoryString()).length;816 const tokens = await countSourceTokens(getMemoryString());
702 if (tokens >= CONTEXT_SIZE) {817 if (tokens >= CONTEXT_SIZE) {
703 break;818 break;
704 }819 }
705 }820 }
706821
707 const resultingString = getMemoryString();822 const resultingString = getMemoryString();
708 const resultingTokens = getTextTokens(tokenizers.GPT2, resultingString).length;823 const resultingTokens = await countSourceTokens(resultingString);
709824
710 if (!resultingString || resultingTokens < CONTEXT_SIZE) {825 if (!resultingString || resultingTokens < CONTEXT_SIZE) {
711 console.debug('Not enough context to summarize');826 console.debug('Not enough context to summarize');
@@ -933,7 +1048,7 @@ jQuery(async function () {
933 name: 'summarize',1048 name: 'summarize',
934 callback: summarizeCallback,1049 callback: summarizeCallback,
935 namedArgumentList: [1050 namedArgumentList: [
936 new SlashCommandNamedArgument('source', 'API to use for summarization', [ARGUMENT_TYPE.STRING], false, false, '', ['main', 'extras']),1051 new SlashCommandNamedArgument('source', 'API to use for summarization', [ARGUMENT_TYPE.STRING], false, false, '', Object.values(summary_sources)),
937 SlashCommandNamedArgument.fromProps({1052 SlashCommandNamedArgument.fromProps({
938 name: 'prompt',1053 name: 'prompt',
939 description: 'prompt to use for summarization',1054 description: 'prompt to use for summarization',
public/scripts/extensions/memory/settings.html+4 -3
@@ -13,6 +13,7 @@
13 <select id="summary_source">13 <select id="summary_source">
14 <option value="main" data-i18n="ext_sum_main_api">Main API</option>14 <option value="main" data-i18n="ext_sum_main_api">Main API</option>
15 <option value="extras">Extras API</option>15 <option value="extras">Extras API</option>
16 <option value="webllm" data-i18n="ext_sum_webllm">WebLLM Extension</option>
16 </select><br>17 </select><br>
1718
18 <div class="flex-container justifyspacebetween alignitemscenter">19 <div class="flex-container justifyspacebetween alignitemscenter">
@@ -24,7 +25,7 @@
2425
25 <textarea id="memory_contents" class="text_pole textarea_compact" rows="6" data-i18n="[placeholder]ext_sum_memory_placeholder" placeholder="Summary will be generated here..."></textarea>26 <textarea id="memory_contents" class="text_pole textarea_compact" rows="6" data-i18n="[placeholder]ext_sum_memory_placeholder" placeholder="Summary will be generated here..."></textarea>
26 <div class="memory_contents_controls">27 <div class="memory_contents_controls">
27 <div id="memory_force_summarize" data-summary-source="main" class="menu_button menu_button_icon" title="Trigger a summary update right now." data-i18n="[title]ext_sum_force_tip">28 <div id="memory_force_summarize" data-summary-source="main,webllm" class="menu_button menu_button_icon" title="Trigger a summary update right now." data-i18n="[title]ext_sum_force_tip">
28 <i class="fa-solid fa-database"></i>29 <i class="fa-solid fa-database"></i>
29 <span data-i18n="ext_sum_force_text">Summarize now</span>30 <span data-i18n="ext_sum_force_text">Summarize now</span>
30 </div>31 </div>
@@ -58,7 +59,7 @@
58 <span data-i18n="ext_sum_prompt_builder_3">Classic, blocking</span>59 <span data-i18n="ext_sum_prompt_builder_3">Classic, blocking</span>
59 </label>60 </label>
60 </div>61 </div>
61 <div data-summary-source="main">62 <div data-summary-source="main,webllm">
62 <label for="memory_prompt" class="title_restorable">63 <label for="memory_prompt" class="title_restorable">
63 <span data-i18n="Summary Prompt">Summary Prompt</span>64 <span data-i18n="Summary Prompt">Summary Prompt</span>
64 <div id="memory_prompt_restore" data-i18n="[title]ext_sum_restore_default_prompt_tip" title="Restore default prompt" class="right_menu_button">65 <div id="memory_prompt_restore" data-i18n="[title]ext_sum_restore_default_prompt_tip" title="Restore default prompt" class="right_menu_button">
@@ -74,7 +75,7 @@
74 </label>75 </label>
75 <input id="memory_override_response_length" type="range" value="{{defaultSettings.overrideResponseLength}}" min="{{defaultSettings.overrideResponseLengthMin}}" max="{{defaultSettings.overrideResponseLengthMax}}" step="{{defaultSettings.overrideResponseLengthStep}}" />76 <input id="memory_override_response_length" type="range" value="{{defaultSettings.overrideResponseLength}}" min="{{defaultSettings.overrideResponseLengthMin}}" max="{{defaultSettings.overrideResponseLengthMax}}" step="{{defaultSettings.overrideResponseLengthStep}}" />
76 <label for="memory_max_messages_per_request">77 <label for="memory_max_messages_per_request">
77 <span data-i18n="ext_sum_raw_max_msg">[Raw] Max messages per request</span> (<span id="memory_max_messages_per_request_value"></span>)78 <span data-i18n="ext_sum_raw_max_msg">[Raw/WebLLM] Max messages per request</span> (<span id="memory_max_messages_per_request_value"></span>)
78 <small class="memory_disabled_hint" data-i18n="ext_sum_0_unlimited">0 = unlimited</small>79 <small class="memory_disabled_hint" data-i18n="ext_sum_0_unlimited">0 = unlimited</small>
79 </label>80 </label>
80 <input id="memory_max_messages_per_request" type="range" value="{{defaultSettings.maxMessagesPerRequest}}" min="{{defaultSettings.maxMessagesPerRequestMin}}" max="{{defaultSettings.maxMessagesPerRequestMax}}" step="{{defaultSettings.maxMessagesPerRequestStep}}" />81 <input id="memory_max_messages_per_request" type="range" value="{{defaultSettings.maxMessagesPerRequest}}" min="{{defaultSettings.maxMessagesPerRequestMin}}" max="{{defaultSettings.maxMessagesPerRequestMax}}" step="{{defaultSettings.maxMessagesPerRequestStep}}" />
public/scripts/extensions/shared.js+84 -1
@@ -1,5 +1,5 @@
1import { getRequestHeaders } from '../../script.js';1import { getRequestHeaders } from '../../script.js';
2import { extension_settings } from '../extensions.js';2import { extension_settings, openThirdPartyExtensionMenu } from '../extensions.js';
3import { oai_settings } from '../openai.js';3import { oai_settings } from '../openai.js';
4import { SECRET_KEYS, secret_state } from '../secrets.js';4import { SECRET_KEYS, secret_state } from '../secrets.js';
5import { textgen_types, textgenerationwebui_settings } from '../textgen-settings.js';5import { textgen_types, textgenerationwebui_settings } from '../textgen-settings.js';
@@ -176,3 +176,86 @@ function throwIfInvalidModel(useReverseProxy) {
176 throw new Error('Custom API URL is not set.');176 throw new Error('Custom API URL is not set.');
177 }177 }
178}178}
179
180/**
181 * Check if the WebLLM extension is installed and supported.
182 * @returns {boolean} Whether the extension is installed and supported
183 */
184export function isWebLlmSupported() {
185 if (!('gpu' in navigator)) {
186 const warningKey = 'webllm_browser_warning_shown';
187 if (!sessionStorage.getItem(warningKey)) {
188 toastr.error('Your browser does not support the WebGPU API. Please use a different browser.', 'WebLLM', {
189 preventDuplicates: true,
190 timeOut: 0,
191 extendedTimeOut: 0,
192 });
193 sessionStorage.setItem(warningKey, '1');
194 }
195 return false;
196 }
197
198 if (!('llm' in SillyTavern)) {
199 const warningKey = 'webllm_extension_warning_shown';
200 if (!sessionStorage.getItem(warningKey)) {
201 toastr.error('WebLLM extension is not installed. Click here to install it.', 'WebLLM', {
202 timeOut: 0,
203 extendedTimeOut: 0,
204 preventDuplicates: true,
205 onclick: () => openThirdPartyExtensionMenu('https://github.com/SillyTavern/Extension-WebLLM'),
206 });
207 sessionStorage.setItem(warningKey, '1');
208 }
209 return false;
210 }
211
212 return true;
213}
214
215/**
216 * Generates text in response to a chat prompt using WebLLM.
217 * @param {any[]} messages Messages to use for generating
218 * @param {object} params Additional parameters
219 * @returns {Promise<string>} Generated response
220 */
221export async function generateWebLlmChatPrompt(messages, params = {}) {
222 if (!isWebLlmSupported()) {
223 throw new Error('WebLLM extension is not installed.');
224 }
225
226 console.debug('WebLLM chat completion request:', messages, params);
227 const engine = SillyTavern.llm;
228 const response = await engine.generateChatPrompt(messages, params);
229 console.debug('WebLLM chat completion response:', response);
230 return response;
231}
232
233/**
234 * Counts the number of tokens in the provided text using WebLLM's default model.
235 * @param {string} text Text to count tokens in
236 * @returns {Promise<number>} Number of tokens in the text
237 */
238export async function countWebLlmTokens(text) {
239 if (!isWebLlmSupported()) {
240 throw new Error('WebLLM extension is not installed.');
241 }
242
243 const engine = SillyTavern.llm;
244 const response = await engine.countTokens(text);
245 return response;
246}
247
248/**
249 * Gets the size of the context in the WebLLM's default model.
250 * @returns {Promise<number>} Size of the context in the WebLLM model
251 */
252export async function getWebLlmContextSize() {
253 if (!isWebLlmSupported()) {
254 throw new Error('WebLLM extension is not installed.');
255 }
256
257 const engine = SillyTavern.llm;
258 await engine.loadModel();
259 const model = await engine.getCurrentModelInfo();
260 return model?.context_size;
261}
public/scripts/extensions/vectors/index.js+43 -0
@@ -31,6 +31,12 @@ import { SlashCommandParser } from '../../slash-commands/SlashCommandParser.js';
31import { SlashCommand } from '../../slash-commands/SlashCommand.js';31import { SlashCommand } from '../../slash-commands/SlashCommand.js';
32import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from '../../slash-commands/SlashCommandArgument.js';32import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from '../../slash-commands/SlashCommandArgument.js';
33import { callGenericPopup, POPUP_RESULT, POPUP_TYPE } from '../../popup.js';33import { callGenericPopup, POPUP_RESULT, POPUP_TYPE } from '../../popup.js';
34import { generateWebLlmChatPrompt, isWebLlmSupported } from '../shared.js';
35
36/**
37 * @typedef {object} HashedMessage
38 * @property {string} text - The hashed message text
39 */
3440
35const MODULE_NAME = 'vectors';41const MODULE_NAME = 'vectors';
3642
@@ -192,6 +198,11 @@ function splitByChunks(items) {
192 return chunkedItems;198 return chunkedItems;
193}199}
194200
201/**
202 * Summarizes messages using the Extras API method.
203 * @param {HashedMessage[]} hashedMessages Array of hashed messages
204 * @returns {Promise<HashedMessage[]>} Summarized messages
205 */
195async function summarizeExtra(hashedMessages) {206async function summarizeExtra(hashedMessages) {
196 for (const element of hashedMessages) {207 for (const element of hashedMessages) {
197 try {208 try {
@@ -223,6 +234,11 @@ async function summarizeExtra(hashedMessages) {
223 return hashedMessages;234 return hashedMessages;
224}235}
225236
237/**
238 * Summarizes messages using the main API method.
239 * @param {HashedMessage[]} hashedMessages Array of hashed messages
240 * @returns {Promise<HashedMessage[]>} Summarized messages
241 */
226async function summarizeMain(hashedMessages) {242async function summarizeMain(hashedMessages) {
227 for (const element of hashedMessages) {243 for (const element of hashedMessages) {
228 element.text = await generateRaw(element.text, '', false, false, settings.summary_prompt);244 element.text = await generateRaw(element.text, '', false, false, settings.summary_prompt);
@@ -231,12 +247,39 @@ async function summarizeMain(hashedMessages) {
231 return hashedMessages;247 return hashedMessages;
232}248}
233249
250/**
251 * Summarizes messages using WebLLM.
252 * @param {HashedMessage[]} hashedMessages Array of hashed messages
253 * @returns {Promise<HashedMessage[]>} Summarized messages
254 */
255async function summarizeWebLLM(hashedMessages) {
256 if (!isWebLlmSupported()) {
257 console.warn('Vectors: WebLLM is not supported');
258 return hashedMessages;
259 }
260
261 for (const element of hashedMessages) {
262 const messages = [{ role:'system', content: settings.summary_prompt }, { role:'user', content: element.text }];
263 element.text = await generateWebLlmChatPrompt(messages);
264 }
265
266 return hashedMessages;
267}
268
269/**
270 * Summarizes messages using the chosen method.
271 * @param {HashedMessage[]} hashedMessages Array of hashed messages
272 * @param {string} endpoint Type of endpoint to use
273 * @returns {Promise<HashedMessage[]>} Summarized messages
274 */
234async function summarize(hashedMessages, endpoint = 'main') {275async function summarize(hashedMessages, endpoint = 'main') {
235 switch (endpoint) {276 switch (endpoint) {
236 case 'main':277 case 'main':
237 return await summarizeMain(hashedMessages);278 return await summarizeMain(hashedMessages);
238 case 'extras':279 case 'extras':
239 return await summarizeExtra(hashedMessages);280 return await summarizeExtra(hashedMessages);
281 case 'webllm':
282 return await summarizeWebLLM(hashedMessages);
240 default:283 default:
241 console.error('Unsupported endpoint', endpoint);284 console.error('Unsupported endpoint', endpoint);
242 }285 }
public/scripts/extensions/vectors/settings.html+2 -1
@@ -374,10 +374,11 @@
374 <select id="vectors_summary_source" class="text_pole">374 <select id="vectors_summary_source" class="text_pole">
375 <option value="main" data-i18n="Main API">Main API</option>375 <option value="main" data-i18n="Main API">Main API</option>
376 <option value="extras" data-i18n="Extras API">Extras API</option>376 <option value="extras" data-i18n="Extras API">Extras API</option>
377 <option value="webllm" data-i18n="WebLLM Extension">WebLLM Extension</option>
377 </select>378 </select>
378379
379 <label for="vectors_summary_prompt" title="Summary Prompt:">Summary Prompt:</label>380 <label for="vectors_summary_prompt" title="Summary Prompt:">Summary Prompt:</label>
380 <small data-i18n="Only used when Main API is selected.">Only used when Main API is selected.</small>381 <small data-i18n="Only used when Main API or WebLLM Extension is selected.">Only used when Main API or WebLLM Extension is selected.</small>
381 <textarea id="vectors_summary_prompt" class="text_pole textarea_compact" rows="6" placeholder="This prompt will be sent to AI to request the summary generation."></textarea>382 <textarea id="vectors_summary_prompt" class="text_pole textarea_compact" rows="6" placeholder="This prompt will be sent to AI to request the summary generation."></textarea>
382 </div>383 </div>
383 </div>384 </div>
public/scripts/templates/installExtension.html+7 -0
@@ -0,0 +1,7 @@
1<h3>Enter the Git URL of the extension to install</h3>
2<br>
3<p><b>Disclaimer:</b> Please be aware that using external extensions can have unintended side effects and may pose
4 security risks. Always make sure you trust the source before importing an extension. We are not responsible for any
5 damage caused by third-party extensions.</p>
6<br>
7<p>Example: <tt> https://github.com/author/extension-name </tt></p>