Merge branch 'staging' into connection-manager

669c49ebba4de8c8a2d59ccddd01956e24fef9c6

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: 10ddf77948e0d93af1f14a8ea9801ce8a4b5d397 37b3184cda2637c558ac28cbbd06aa9fd5fb6a5c
14 files changed, +248 -44Ignore whitespace
default/config.yaml+3 -0
@@ -95,6 +95,9 @@ requestOverrides: []
95enableExtensions: true95enableExtensions: true
96# Automatically update extensions when a release version changes96# Automatically update extensions when a release version changes
97enableExtensionsAutoUpdate: true97enableExtensionsAutoUpdate: true
98# Additional model tokenizers can be downloaded on demand.
99# Disabling will fallback to another locally available tokenizer.
100enableDownloadableTokenizers: true
98# Extension settings101# Extension settings
99extras:102extras:
100 # Disables automatic model download from HuggingFace103 # Disables automatic model download from HuggingFace
public/index.html+5 -3
@@ -1820,12 +1820,12 @@
1820 <span id="claude_assistant_prefill_text" data-i18n="Assistant Prefill">Assistant Prefill</span>1820 <span id="claude_assistant_prefill_text" data-i18n="Assistant Prefill">Assistant Prefill</span>
1821 <i class="editor_maximize fa-solid fa-maximize right_menu_button" data-for="claude_assistant_prefill" title="Expand the editor" data-i18n="[title]Expand the editor"></i>1821 <i class="editor_maximize fa-solid fa-maximize right_menu_button" data-for="claude_assistant_prefill" title="Expand the editor" data-i18n="[title]Expand the editor"></i>
1822 </div>1822 </div>
1823 <textarea id="claude_assistant_prefill" class="text_pole textarea_compact" name="assistant_prefill" rows="6" data-i18n="[placeholder]Start Claude's answer with..." placeholder="Start Claude's answer with..."></textarea>1823 <textarea id="claude_assistant_prefill" class="text_pole textarea_compact autoSetHeight" name="assistant_prefill" rows="2" data-i18n="[placeholder]Start Claude's answer with..." placeholder="Start Claude's answer with..."></textarea>
1824 <div class="flex-container alignItemsCenter">1824 <div class="flex-container alignItemsCenter">
1825 <span id="claude_assistant_impersonation_text" data-i18n="Assistant Impersonation Prefill">Assistant Impersonation Prefill</span>1825 <span id="claude_assistant_impersonation_text" data-i18n="Assistant Impersonation Prefill">Assistant Impersonation Prefill</span>
1826 <i class="editor_maximize fa-solid fa-maximize right_menu_button" data-for="claude_assistant_impersonation" title="Expand the editor" data-i18n="[title]Expand the editor"></i>1826 <i class="editor_maximize fa-solid fa-maximize right_menu_button" data-for="claude_assistant_impersonation" title="Expand the editor" data-i18n="[title]Expand the editor"></i>
1827 </div>1827 </div>
1828 <textarea id="claude_assistant_impersonation" class="text_pole textarea_compact" name="assistant_impersonation" rows="6" data-i18n="[placeholder]Start Claude's answer with..." placeholder="Start Claude's answer with..."></textarea>1828 <textarea id="claude_assistant_impersonation" class="text_pole textarea_compact autoSetHeight" name="assistant_impersonation" rows="2" data-i18n="[placeholder]Start Claude's answer with..." placeholder="Start Claude's answer with..."></textarea>
1829 </div>1829 </div>
1830 <label for="claude_use_sysprompt" class="checkbox_label widthFreeExpand">1830 <label for="claude_use_sysprompt" class="checkbox_label widthFreeExpand">
1831 <input id="claude_use_sysprompt" type="checkbox" />1831 <input id="claude_use_sysprompt" type="checkbox" />
@@ -1845,7 +1845,7 @@
1845 <div class="fa-solid fa-clock-rotate-left"></div>1845 <div class="fa-solid fa-clock-rotate-left"></div>
1846 </div>1846 </div>
1847 </div>1847 </div>
1848 <textarea id="claude_human_sysprompt_textarea" class="text_pole textarea_compact" rows="4" data-i18n="[placeholder]Human message" placeholder="Human message, instruction, etc.&#10;Adds nothing when empty, i.e. requires a new prompt with the role 'user'."></textarea>1848 <textarea id="claude_human_sysprompt_textarea" class="text_pole textarea_compact autoSetHeight" rows="2" data-i18n="[placeholder]Human message" placeholder="Human message, instruction, etc.&#10;Adds nothing when empty, i.e. requires a new prompt with the role 'user'."></textarea>
1849 </div>1849 </div>
1850 </div>1850 </div>
1851 </div>1851 </div>
@@ -3289,6 +3289,8 @@
3289 <option value="12">Llama 3</option>3289 <option value="12">Llama 3</option>
3290 <option value="13">Gemma / Gemini</option>3290 <option value="13">Gemma / Gemini</option>
3291 <option value="14">Jamba</option>3291 <option value="14">Jamba</option>
3292 <option value="15">Qwen2</option>
3293 <option value="16">Command-R</option>
3292 <option value="4">NerdStash (NovelAI Clio)</option>3294 <option value="4">NerdStash (NovelAI Clio)</option>
3293 <option value="5">NerdStash v2 (NovelAI Kayra)</option>3295 <option value="5">NerdStash v2 (NovelAI Kayra)</option>
3294 <option value="7">Mistral</option>3296 <option value="7">Mistral</option>
public/script.js+15 -5
@@ -8504,22 +8504,23 @@ for (const chatCompletionSource of Object.values(chat_completion_sources)) {
8504 };8504 };
8505}8505}
85068506
8507async function selectContextCallback(_, name) {8507async function selectContextCallback(args, name) {
8508 if (!name) {8508 if (!name) {
8509 return power_user.context.preset;8509 return power_user.context.preset;
8510 }8510 }
85118511
8512 const quiet = isTrueBoolean(args?.quiet);
8512 const contextNames = context_presets.map(preset => preset.name);8513 const contextNames = context_presets.map(preset => preset.name);
8513 const fuse = new Fuse(contextNames);8514 const fuse = new Fuse(contextNames);
8514 const result = fuse.search(name);8515 const result = fuse.search(name);
85158516
8516 if (result.length === 0) {8517 if (result.length === 0) {
8517 toastr.warning(`Context template "${name}" not found`);8518 !quiet && toastr.warning(`Context template "${name}" not found`);
8518 return '';8519 return '';
8519 }8520 }
85208521
8521 const foundName = result[0].item;8522 const foundName = result[0].item;
8522 selectContextPreset(foundName);8523 selectContextPreset(foundName, quiet);
8523 return foundName;8524 return foundName;
8524}8525}
85258526
@@ -8528,16 +8529,16 @@ async function selectInstructCallback(args, name) {
8528 return power_user.instruct.preset;8529 return power_user.instruct.preset;
8529 }8530 }
85308531
8532 const quiet = isTrueBoolean(args?.quiet);
8531 const instructNames = instruct_presets.map(preset => preset.name);8533 const instructNames = instruct_presets.map(preset => preset.name);
8532 const fuse = new Fuse(instructNames);8534 const fuse = new Fuse(instructNames);
8533 const result = fuse.search(name);8535 const result = fuse.search(name);
85348536
8535 if (result.length === 0) {8537 if (result.length === 0) {
8536 toastr.warning(`Instruct template "${name}" not found`);8538 !quiet && toastr.warning(`Instruct template "${name}" not found`);
8537 return '';8539 return '';
8538 }8540 }
85398541
8540 const quiet = isTrueBoolean(args?.quiet);
8541 const foundName = result[0].item;8542 const foundName = result[0].item;
8542 selectInstructPreset(foundName, quiet);8543 selectInstructPreset(foundName, quiet);
8543 return foundName;8544 return foundName;
@@ -9283,6 +9284,15 @@ jQuery(async function () {
9283 name: 'context',9284 name: 'context',
9284 callback: selectContextCallback,9285 callback: selectContextCallback,
9285 returns: 'template name',9286 returns: 'template name',
9287 namedArgumentList: [
9288 SlashCommandNamedArgument.fromProps({
9289 name: 'quiet',
9290 description: 'Suppress the toast message on template change',
9291 typeList: [ARGUMENT_TYPE.BOOLEAN],
9292 defaultValue: 'false',
9293 enumList: commonEnumProviders.boolean('trueFalse')(),
9294 }),
9295 ],
9286 unnamedArgumentList: [9296 unnamedArgumentList: [
9287 SlashCommandArgument.fromProps({9297 SlashCommandArgument.fromProps({
9288 description: 'context template name',9298 description: 'context template name',
public/scripts/extensions/stable-diffusion/index.js+17 -11
@@ -724,7 +724,7 @@ function onChatChanged() {
724}724}
725725
726async function adjustElementScrollHeight() {726async function adjustElementScrollHeight() {
727 if (!$('.sd_settings').is(':visible')) {727 if (CSS.supports('field-sizing', 'content') || !$('.sd_settings').is(':visible')) {
728 return;728 return;
729 }729 }
730730
@@ -737,17 +737,19 @@ async function adjustElementScrollHeight() {
737async function onCharacterPromptInput() {737async function onCharacterPromptInput() {
738 const key = getCharaFilename(this_chid);738 const key = getCharaFilename(this_chid);
739 extension_settings.sd.character_prompts[key] = $('#sd_character_prompt').val();739 extension_settings.sd.character_prompts[key] = $('#sd_character_prompt').val();
740 await resetScrollHeight($(this));
741 saveSettingsDebounced();740 saveSettingsDebounced();
742 writePromptFieldsDebounced(this_chid);741 writePromptFieldsDebounced(this_chid);
742 if (CSS.supports('field-sizing', 'content')) return;
743 await resetScrollHeight($(this));
743}744}
744745
745async function onCharacterNegativePromptInput() {746async function onCharacterNegativePromptInput() {
746 const key = getCharaFilename(this_chid);747 const key = getCharaFilename(this_chid);
747 extension_settings.sd.character_negative_prompts[key] = $('#sd_character_negative_prompt').val();748 extension_settings.sd.character_negative_prompts[key] = $('#sd_character_negative_prompt').val();
748 await resetScrollHeight($(this));
749 saveSettingsDebounced();749 saveSettingsDebounced();
750 writePromptFieldsDebounced(this_chid);750 writePromptFieldsDebounced(this_chid);
751 if (CSS.supports('field-sizing', 'content')) return;
752 await resetScrollHeight($(this));
751}753}
752754
753function getCharacterPrefix() {755function getCharacterPrefix() {
@@ -856,14 +858,16 @@ function onStepsInput() {
856858
857async function onPromptPrefixInput() {859async function onPromptPrefixInput() {
858 extension_settings.sd.prompt_prefix = $('#sd_prompt_prefix').val();860 extension_settings.sd.prompt_prefix = $('#sd_prompt_prefix').val();
859 await resetScrollHeight($(this));
860 saveSettingsDebounced();861 saveSettingsDebounced();
862 if (CSS.supports('field-sizing', 'content')) return;
863 await resetScrollHeight($(this));
861}864}
862865
863async function onNegativePromptInput() {866async function onNegativePromptInput() {
864 extension_settings.sd.negative_prompt = $('#sd_negative_prompt').val();867 extension_settings.sd.negative_prompt = $('#sd_negative_prompt').val();
865 await resetScrollHeight($(this));
866 saveSettingsDebounced();868 saveSettingsDebounced();
869 if (CSS.supports('field-sizing', 'content')) return;
870 await resetScrollHeight($(this));
867}871}
868872
869function onSamplerChange() {873function onSamplerChange() {
@@ -3911,12 +3915,14 @@ jQuery(async () => {
3911 $('#sd_stability_style_preset').on('change', onStabilityStylePresetChange);3915 $('#sd_stability_style_preset').on('change', onStabilityStylePresetChange);
3912 $('#sd_huggingface_model_id').on('input', onHFModelInput);3916 $('#sd_huggingface_model_id').on('input', onHFModelInput);
39133917
3914 $('.sd_settings .inline-drawer-toggle').on('click', function () {3918 if (!CSS.supports('field-sizing', 'content')) {
3915 initScrollHeight($('#sd_prompt_prefix'));3919 $('.sd_settings .inline-drawer-toggle').on('click', function () {
3916 initScrollHeight($('#sd_negative_prompt'));3920 initScrollHeight($('#sd_prompt_prefix'));
3917 initScrollHeight($('#sd_character_prompt'));3921 initScrollHeight($('#sd_negative_prompt'));
3918 initScrollHeight($('#sd_character_negative_prompt'));3922 initScrollHeight($('#sd_character_prompt'));
3919 });3923 initScrollHeight($('#sd_character_negative_prompt'));
3924 });
3925 }
39203926
3921 for (const [key, value] of Object.entries(resolutionOptions)) {3927 for (const [key, value] of Object.entries(resolutionOptions)) {
3922 const option = document.createElement('option');3928 const option = document.createElement('option');
public/scripts/extensions/stable-diffusion/settings.html+4 -4
@@ -408,16 +408,16 @@
408 </div>408 </div>
409 </div>409 </div>
410 <label for="sd_prompt_prefix" data-i18n="Common prompt prefix">Common prompt prefix</label>410 <label for="sd_prompt_prefix" data-i18n="Common prompt prefix">Common prompt prefix</label>
411 <textarea id="sd_prompt_prefix" class="text_pole textarea_compact" data-i18n="[placeholder]sd_prompt_prefix_placeholder" placeholder="Use {prompt} to specify where the generated prompt will be inserted"></textarea>411 <textarea id="sd_prompt_prefix" class="text_pole textarea_compact autoSetHeight" data-i18n="[placeholder]sd_prompt_prefix_placeholder" placeholder="Use {prompt} to specify where the generated prompt will be inserted"></textarea>
412 <label for="sd_negative_prompt" data-i18n="Negative common prompt prefix">Negative common prompt prefix</label>412 <label for="sd_negative_prompt" data-i18n="Negative common prompt prefix">Negative common prompt prefix</label>
413 <textarea id="sd_negative_prompt" class="text_pole textarea_compact"></textarea>413 <textarea id="sd_negative_prompt" class="text_pole textarea_compact autoSetHeight"></textarea>
414 <div id="sd_character_prompt_block">414 <div id="sd_character_prompt_block">
415 <label for="sd_character_prompt" data-i18n="Character-specific prompt prefix">Character-specific prompt prefix</label>415 <label for="sd_character_prompt" data-i18n="Character-specific prompt prefix">Character-specific prompt prefix</label>
416 <small data-i18n="Won't be used in groups.">Won't be used in groups.</small>416 <small data-i18n="Won't be used in groups.">Won't be used in groups.</small>
417 <textarea id="sd_character_prompt" class="text_pole textarea_compact" data-i18n="[placeholder]sd_character_prompt_placeholder" placeholder="Any characteristics that describe the currently selected character. Will be added after a common prompt prefix.&#10;Example: female, green eyes, brown hair, pink shirt"></textarea>417 <textarea id="sd_character_prompt" class="text_pole textarea_compact autoSetHeight" data-i18n="[placeholder]sd_character_prompt_placeholder" placeholder="Any characteristics that describe the currently selected character. Will be added after a common prompt prefix.&#10;Example: female, green eyes, brown hair, pink shirt"></textarea>
418 <label for="sd_character_negative_prompt" data-i18n="Character-specific negative prompt prefix">Character-specific negative prompt prefix</label>418 <label for="sd_character_negative_prompt" data-i18n="Character-specific negative prompt prefix">Character-specific negative prompt prefix</label>
419 <small data-i18n="Won't be used in groups.">Won't be used in groups.</small>419 <small data-i18n="Won't be used in groups.">Won't be used in groups.</small>
420 <textarea id="sd_character_negative_prompt" class="text_pole textarea_compact" data-i18n="[placeholder]sd_character_negative_prompt_placeholder" placeholder="Any characteristics that should not appear for the selected character. Will be added after a negative common prompt prefix.&#10;Example: jewellery, shoes, glasses"></textarea>420 <textarea id="sd_character_negative_prompt" class="text_pole textarea_compact autoSetHeight" data-i18n="[placeholder]sd_character_negative_prompt_placeholder" placeholder="Any characteristics that should not appear for the selected character. Will be added after a negative common prompt prefix.&#10;Example: jewellery, shoes, glasses"></textarea>
421 <label for="sd_character_prompt_share" class="checkbox_label flexWrap marginTop5">421 <label for="sd_character_prompt_share" class="checkbox_label flexWrap marginTop5">
422 <input id="sd_character_prompt_share" type="checkbox" />422 <input id="sd_character_prompt_share" type="checkbox" />
423 <span data-i18n="Shareable">423 <span data-i18n="Shareable">
public/scripts/extensions/token-counter/index.js+4 -2
@@ -59,8 +59,10 @@ async function doTokenCounter() {
59 $('#tokenized_chunks_display').text('—');59 $('#tokenized_chunks_display').text('—');
60 }60 }
6161
62 await resetScrollHeight($('#token_counter_textarea'));62 if (!CSS.supports('field-sizing', 'content')) {
63 await resetScrollHeight($('#token_counter_ids'));63 await resetScrollHeight($('#token_counter_textarea'));
64 await resetScrollHeight($('#token_counter_ids'));
65 }
64 }, debounce_timeout.relaxed);66 }, debounce_timeout.relaxed);
65 dialog.find('#token_counter_textarea').on('input', () => countDebounced());67 dialog.find('#token_counter_textarea').on('input', () => countDebounced());
6668
public/scripts/extensions/token-counter/style.css+5 -0
@@ -4,3 +4,8 @@
4 padding: 2px;4 padding: 2px;
5 display: inline-block;5 display: inline-block;
6}6}
7
8#token_counter_textarea,
9#token_counter_ids {
10 field-sizing: content;
11}
public/scripts/group-chats.js+5 -2
@@ -1414,8 +1414,11 @@ function toggleHiddenControls(group, generationMode = null) {
1414 const isJoin = [group_generation_mode.APPEND, group_generation_mode.APPEND_DISABLED].includes(generationMode ?? group?.generation_mode);1414 const isJoin = [group_generation_mode.APPEND, group_generation_mode.APPEND_DISABLED].includes(generationMode ?? group?.generation_mode);
1415 $('#rm_group_generation_mode_join_prefix').parent().toggle(isJoin);1415 $('#rm_group_generation_mode_join_prefix').parent().toggle(isJoin);
1416 $('#rm_group_generation_mode_join_suffix').parent().toggle(isJoin);1416 $('#rm_group_generation_mode_join_suffix').parent().toggle(isJoin);
1417 initScrollHeight($('#rm_group_generation_mode_join_prefix'));1417
1418 initScrollHeight($('#rm_group_generation_mode_join_suffix'));1418 if (!CSS.supports('field-sizing', 'content')) {
1419 initScrollHeight($('#rm_group_generation_mode_join_prefix'));
1420 initScrollHeight($('#rm_group_generation_mode_join_suffix'));
1421 }
1419}1422}
14201423
1421function select_group_chats(groupId, skipAnimation) {1424function select_group_chats(groupId, skipAnimation) {
public/scripts/instruct-mode.js+4 -3
@@ -130,19 +130,20 @@ function highlightDefaultPreset() {
130/**130/**
131 * Select context template if not already selected.131 * Select context template if not already selected.
132 * @param {string} preset Preset name.132 * @param {string} preset Preset name.
133 * @param {boolean} quiet Suppress info message.
133 */134 */
134export function selectContextPreset(preset) {135export function selectContextPreset(preset, quiet) {
135 // If context template is not already selected, select it136 // If context template is not already selected, select it
136 if (preset !== power_user.context.preset) {137 if (preset !== power_user.context.preset) {
137 $('#context_presets').val(preset).trigger('change');138 $('#context_presets').val(preset).trigger('change');
138 toastr.info(`Context Template: preset "${preset}" auto-selected`);139 !quiet && toastr.info(`Context Template: preset "${preset}" auto-selected`);
139 }140 }
140141
141 // If instruct mode is disabled, enable it, except for default context template142 // If instruct mode is disabled, enable it, except for default context template
142 if (!power_user.instruct.enabled && preset !== power_user.default_context) {143 if (!power_user.instruct.enabled && preset !== power_user.default_context) {
143 power_user.instruct.enabled = true;144 power_user.instruct.enabled = true;
144 $('#instruct_enabled').prop('checked', true).trigger('change');145 $('#instruct_enabled').prop('checked', true).trigger('change');
145 toastr.info('Instruct Mode enabled');146 !quiet && toastr.info('Instruct Mode enabled');
146 }147 }
147148
148 saveSettingsDebounced();149 saveSettingsDebounced();
public/scripts/textgen-models.js+7 -0
@@ -590,6 +590,9 @@ function calculateOpenRouterCost() {
590export function getCurrentOpenRouterModelTokenizer() {590export function getCurrentOpenRouterModelTokenizer() {
591 const modelId = textgen_settings.openrouter_model;591 const modelId = textgen_settings.openrouter_model;
592 const model = openRouterModels.find(x => x.id === modelId);592 const model = openRouterModels.find(x => x.id === modelId);
593 if (modelId?.includes('jamba')) {
594 return tokenizers.JAMBA;
595 }
593 switch (model?.architecture?.tokenizer) {596 switch (model?.architecture?.tokenizer) {
594 case 'Llama2':597 case 'Llama2':
595 return tokenizers.LLAMA;598 return tokenizers.LLAMA;
@@ -603,6 +606,10 @@ export function getCurrentOpenRouterModelTokenizer() {
603 return tokenizers.GEMMA;606 return tokenizers.GEMMA;
604 case 'Claude':607 case 'Claude':
605 return tokenizers.CLAUDE;608 return tokenizers.CLAUDE;
609 case 'Cohere':
610 return tokenizers.COMMAND_R;
611 case 'Qwen':
612 return tokenizers.QWEN2;
606 default:613 default:
607 return tokenizers.OPENAI;614 return tokenizers.OPENAI;
608 }615 }
public/scripts/tokenizers.js+30 -0
@@ -28,6 +28,8 @@ export const tokenizers = {
28 LLAMA3: 12,28 LLAMA3: 12,
29 GEMMA: 13,29 GEMMA: 13,
30 JAMBA: 14,30 JAMBA: 14,
31 QWEN2: 15,
32 COMMAND_R: 16,
31 BEST_MATCH: 99,33 BEST_MATCH: 99,
32};34};
3335
@@ -105,6 +107,16 @@ const TOKENIZER_URLS = {
105 decode: '/api/tokenizers/jamba/decode',107 decode: '/api/tokenizers/jamba/decode',
106 count: '/api/tokenizers/jamba/encode',108 count: '/api/tokenizers/jamba/encode',
107 },109 },
110 [tokenizers.QWEN2]: {
111 encode: '/api/tokenizers/qwen2/encode',
112 decode: '/api/tokenizers/qwen2/decode',
113 count: '/api/tokenizers/qwen2/encode',
114 },
115 [tokenizers.COMMAND_R]: {
116 encode: '/api/tokenizers/command-r/encode',
117 decode: '/api/tokenizers/command-r/decode',
118 count: '/api/tokenizers/command-r/encode',
119 },
108 [tokenizers.API_TEXTGENERATIONWEBUI]: {120 [tokenizers.API_TEXTGENERATIONWEBUI]: {
109 encode: '/api/tokenizers/remote/textgenerationwebui/encode',121 encode: '/api/tokenizers/remote/textgenerationwebui/encode',
110 count: '/api/tokenizers/remote/textgenerationwebui/encode',122 count: '/api/tokenizers/remote/textgenerationwebui/encode',
@@ -293,6 +305,12 @@ export function getTokenizerBestMatch(forApi) {
293 if (model.includes('jamba')) {305 if (model.includes('jamba')) {
294 return tokenizers.JAMBA;306 return tokenizers.JAMBA;
295 }307 }
308 if (model.includes('command-r')) {
309 return tokenizers.COMMAND_R;
310 }
311 if (model.includes('qwen2')) {
312 return tokenizers.QWEN2;
313 }
296 }314 }
297315
298 return tokenizers.LLAMA;316 return tokenizers.LLAMA;
@@ -511,6 +529,8 @@ export function getTokenizerModel() {
511 const yiTokenizer = 'yi';529 const yiTokenizer = 'yi';
512 const gemmaTokenizer = 'gemma';530 const gemmaTokenizer = 'gemma';
513 const jambaTokenizer = 'jamba';531 const jambaTokenizer = 'jamba';
532 const qwen2Tokenizer = 'qwen2';
533 const commandRTokenizer = 'command-r';
514534
515 // Assuming no one would use it for different models.. right?535 // Assuming no one would use it for different models.. right?
516 if (oai_settings.chat_completion_source == chat_completion_sources.SCALE) {536 if (oai_settings.chat_completion_source == chat_completion_sources.SCALE) {
@@ -558,6 +578,12 @@ export function getTokenizerModel() {
558 else if (model?.architecture?.tokenizer === 'Gemini') {578 else if (model?.architecture?.tokenizer === 'Gemini') {
559 return gemmaTokenizer;579 return gemmaTokenizer;
560 }580 }
581 else if (model?.architecture?.tokenizer === 'Qwen') {
582 return qwen2Tokenizer;
583 }
584 else if (model?.architecture?.tokenizer === 'Cohere') {
585 return commandRTokenizer;
586 }
561 else if (oai_settings.openrouter_model.includes('gpt-4o')) {587 else if (oai_settings.openrouter_model.includes('gpt-4o')) {
562 return gpt4oTokenizer;588 return gpt4oTokenizer;
563 }589 }
@@ -581,6 +607,10 @@ export function getTokenizerModel() {
581 }607 }
582 }608 }
583609
610 if (oai_settings.chat_completion_source == chat_completion_sources.COHERE) {
611 return commandRTokenizer;
612 }
613
584 if (oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) {614 if (oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) {
585 return gemmaTokenizer;615 return gemmaTokenizer;
586 }616 }
public/style.css+4 -4
@@ -182,6 +182,10 @@ body.movingUI ::-webkit-scrollbar-thumb:vertical {
182 min-width: 40px;182 min-width: 40px;
183}183}
184184
185::-webkit-scrollbar-corner {
186 background-color: transparent;
187}
188
185table.responsiveTable {189table.responsiveTable {
186 width: 100%;190 width: 100%;
187 margin: 10px 0;191 margin: 10px 0;
@@ -4675,10 +4679,6 @@ body:not(.sd) .mes_img_swipes {
4675 width: 100%;4679 width: 100%;
4676}4680}
46774681
4678.img_enlarged_holder::-webkit-scrollbar-corner {
4679 background-color: transparent;
4680}
4681
4682.img_enlarged_container pre code {4682.img_enlarged_container pre code {
4683 position: relative;4683 position: relative;
4684 display: block;4684 display: block;
src/endpoints/tokenizers.js+130 -10
@@ -4,13 +4,12 @@ const express = require('express');
4const { SentencePieceProcessor } = require('@agnai/sentencepiece-js');4const { SentencePieceProcessor } = require('@agnai/sentencepiece-js');
5const tiktoken = require('tiktoken');5const tiktoken = require('tiktoken');
6const { Tokenizer } = require('@agnai/web-tokenizers');6const { Tokenizer } = require('@agnai/web-tokenizers');
7const { convertClaudePrompt, convertGooglePrompt } = require('../prompt-converters');7const { convertClaudePrompt } = require('../prompt-converters');
8const { readSecret, SECRET_KEYS } = require('./secrets');
9const { TEXTGEN_TYPES } = require('../constants');8const { TEXTGEN_TYPES } = require('../constants');
10const { jsonParser } = require('../express-common');9const { jsonParser } = require('../express-common');
11const { setAdditionalHeaders } = require('../additional-headers');10const { setAdditionalHeaders } = require('../additional-headers');
1211const { getConfigValue, isValidUrl } = require('../util');
13const API_MAKERSUITE = 'https://generativelanguage.googleapis.com';12const writeFileAtomicSync = require('write-file-atomic').sync;
1413
15/**14/**
16 * @typedef { (req: import('express').Request, res: import('express').Response) => Promise<any> } TokenizationHandler15 * @typedef { (req: import('express').Request, res: import('express').Response) => Promise<any> } TokenizationHandler
@@ -53,6 +52,65 @@ const TEXT_COMPLETION_MODELS = [
53];52];
5453
55const CHARS_PER_TOKEN = 3.35;54const CHARS_PER_TOKEN = 3.35;
55const IS_DOWNLOAD_ALLOWED = getConfigValue('enableDownloadableTokenizers', true);
56
57/**
58 * Gets a path to the tokenizer model. Downloads the model if it's a URL.
59 * @param {string} model Model URL or path
60 * @param {string|undefined} fallbackModel Fallback model path\
61 * @returns {Promise<string>} Path to the tokenizer model
62 */
63async function getPathToTokenizer(model, fallbackModel) {
64 if (!isValidUrl(model)) {
65 return model;
66 }
67
68 try {
69 const url = new URL(model);
70
71 if (!['https:', 'http:'].includes(url.protocol)) {
72 throw new Error('Invalid URL protocol');
73 }
74
75 const fileName = url.pathname.split('/').pop();
76
77 if (!fileName) {
78 throw new Error('Failed to extract the file name from the URL');
79 }
80
81 const CACHE_PATH = path.join(global.DATA_ROOT, '_cache');
82 if (!fs.existsSync(CACHE_PATH)) {
83 fs.mkdirSync(CACHE_PATH, { recursive: true });
84 }
85
86 const cachedFile = path.join(CACHE_PATH, fileName);
87 if (fs.existsSync(cachedFile)) {
88 return cachedFile;
89 }
90
91 if (!IS_DOWNLOAD_ALLOWED) {
92 throw new Error('Downloading tokenizers is disabled, the model is not cached');
93 }
94
95 console.log('Downloading tokenizer model:', model);
96 const response = await fetch(model);
97 if (!response.ok) {
98 throw new Error(`Failed to fetch the model: ${response.status} ${response.statusText}`);
99 }
100
101 const arrayBuffer = await response.arrayBuffer();
102 writeFileAtomicSync(cachedFile, Buffer.from(arrayBuffer));
103 return cachedFile;
104 } catch (error) {
105 const getLastSegment = str => str?.split('/')?.pop() || '';
106 if (fallbackModel) {
107 console.log(`Could not get a tokenizer from ${getLastSegment(model)}. Reason: ${error.message}. Using a fallback model: ${getLastSegment(fallbackModel)}.`);
108 return fallbackModel;
109 }
110
111 throw new Error(`Failed to instantiate a tokenizer and fallback is not provided. Reason: ${error.message}`);
112 }
113}
56114
57/**115/**
58 * Sentencepiece tokenizer for tokenizing text.116 * Sentencepiece tokenizer for tokenizing text.
@@ -66,13 +124,19 @@ class SentencePieceTokenizer {
66 * @type {string} Path to the tokenizer model124 * @type {string} Path to the tokenizer model
67 */125 */
68 #model;126 #model;
127 /**
128 * @type {string|undefined} Path to the fallback model
129 */
130 #fallbackModel;
69131
70 /**132 /**
71 * Creates a new Sentencepiece tokenizer.133 * Creates a new Sentencepiece tokenizer.
72 * @param {string} model Path to the tokenizer model134 * @param {string} model Path to the tokenizer model
135 * @param {string} [fallbackModel] Path to the fallback model
73 */136 */
74 constructor(model) {137 constructor(model, fallbackModel) {
75 this.#model = model;138 this.#model = model;
139 this.#fallbackModel = fallbackModel;
76 }140 }
77141
78 /**142 /**
@@ -85,9 +149,10 @@ class SentencePieceTokenizer {
85 }149 }
86150
87 try {151 try {
152 const pathToModel = await getPathToTokenizer(this.#model, this.#fallbackModel);
88 this.#instance = new SentencePieceProcessor();153 this.#instance = new SentencePieceProcessor();
89 await this.#instance.load(this.#model);154 await this.#instance.load(pathToModel);
90 console.log('Instantiated the tokenizer for', path.parse(this.#model).name);155 console.log('Instantiated the tokenizer for', path.parse(pathToModel).name);
91 return this.#instance;156 return this.#instance;
92 } catch (error) {157 } catch (error) {
93 console.error('Sentencepiece tokenizer failed to load: ' + this.#model, error);158 console.error('Sentencepiece tokenizer failed to load: ' + this.#model, error);
@@ -108,13 +173,19 @@ class WebTokenizer {
108 * @type {string} Path to the tokenizer model173 * @type {string} Path to the tokenizer model
109 */174 */
110 #model;175 #model;
176 /**
177 * @type {string|undefined} Path to the fallback model
178 */
179 #fallbackModel;
111180
112 /**181 /**
113 * Creates a new Web tokenizer.182 * Creates a new Web tokenizer.
114 * @param {string} model Path to the tokenizer model183 * @param {string} model Path to the tokenizer model
184 * @param {string} [fallbackModel] Path to the fallback model
115 */185 */
116 constructor(model) {186 constructor(model, fallbackModel) {
117 this.#model = model;187 this.#model = model;
188 this.#fallbackModel = fallbackModel;
118 }189 }
119190
120 /**191 /**
@@ -127,9 +198,10 @@ class WebTokenizer {
127 }198 }
128199
129 try {200 try {
130 const arrayBuffer = fs.readFileSync(this.#model).buffer;201 const pathToModel = await getPathToTokenizer(this.#model, this.#fallbackModel);
202 const arrayBuffer = fs.readFileSync(pathToModel).buffer;
131 this.#instance = await Tokenizer.fromJSON(arrayBuffer);203 this.#instance = await Tokenizer.fromJSON(arrayBuffer);
132 console.log('Instantiated the tokenizer for', path.parse(this.#model).name);204 console.log('Instantiated the tokenizer for', path.parse(pathToModel).name);
133 return this.#instance;205 return this.#instance;
134 } catch (error) {206 } catch (error) {
135 console.error('Web tokenizer failed to load: ' + this.#model, error);207 console.error('Web tokenizer failed to load: ' + this.#model, error);
@@ -147,6 +219,8 @@ const spp_gemma = new SentencePieceTokenizer('src/tokenizers/gemma.model');
147const spp_jamba = new SentencePieceTokenizer('src/tokenizers/jamba.model');219const spp_jamba = new SentencePieceTokenizer('src/tokenizers/jamba.model');
148const claude_tokenizer = new WebTokenizer('src/tokenizers/claude.json');220const claude_tokenizer = new WebTokenizer('src/tokenizers/claude.json');
149const llama3_tokenizer = new WebTokenizer('src/tokenizers/llama3.json');221const llama3_tokenizer = new WebTokenizer('src/tokenizers/llama3.json');
222const commandTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/command-r.json', 'src/tokenizers/llama3.json');
223const qwen2Tokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/qwen2.json', 'src/tokenizers/llama3.json');
150224
151const sentencepieceTokenizers = [225const sentencepieceTokenizers = [
152 'llama',226 'llama',
@@ -332,6 +406,14 @@ function getTokenizerModel(requestModel) {
332 return 'jamba';406 return 'jamba';
333 }407 }
334408
409 if (requestModel.includes('qwen2')) {
410 return 'qwen2';
411 }
412
413 if (requestModel.includes('command-r')) {
414 return 'command-r';
415 }
416
335 // default417 // default
336 return 'gpt-3.5-turbo';418 return 'gpt-3.5-turbo';
337}419}
@@ -557,6 +639,8 @@ router.post('/jamba/encode', jsonParser, createSentencepieceEncodingHandler(spp_
557router.post('/gpt2/encode', jsonParser, createTiktokenEncodingHandler('gpt2'));639router.post('/gpt2/encode', jsonParser, createTiktokenEncodingHandler('gpt2'));
558router.post('/claude/encode', jsonParser, createWebTokenizerEncodingHandler(claude_tokenizer));640router.post('/claude/encode', jsonParser, createWebTokenizerEncodingHandler(claude_tokenizer));
559router.post('/llama3/encode', jsonParser, createWebTokenizerEncodingHandler(llama3_tokenizer));641router.post('/llama3/encode', jsonParser, createWebTokenizerEncodingHandler(llama3_tokenizer));
642router.post('/qwen2/encode', jsonParser, createWebTokenizerEncodingHandler(qwen2Tokenizer));
643router.post('/command-r/encode', jsonParser, createWebTokenizerEncodingHandler(commandTokenizer));
560router.post('/llama/decode', jsonParser, createSentencepieceDecodingHandler(spp_llama));644router.post('/llama/decode', jsonParser, createSentencepieceDecodingHandler(spp_llama));
561router.post('/nerdstash/decode', jsonParser, createSentencepieceDecodingHandler(spp_nerd));645router.post('/nerdstash/decode', jsonParser, createSentencepieceDecodingHandler(spp_nerd));
562router.post('/nerdstash_v2/decode', jsonParser, createSentencepieceDecodingHandler(spp_nerd_v2));646router.post('/nerdstash_v2/decode', jsonParser, createSentencepieceDecodingHandler(spp_nerd_v2));
@@ -567,6 +651,8 @@ router.post('/jamba/decode', jsonParser, createSentencepieceDecodingHandler(spp_
567router.post('/gpt2/decode', jsonParser, createTiktokenDecodingHandler('gpt2'));651router.post('/gpt2/decode', jsonParser, createTiktokenDecodingHandler('gpt2'));
568router.post('/claude/decode', jsonParser, createWebTokenizerDecodingHandler(claude_tokenizer));652router.post('/claude/decode', jsonParser, createWebTokenizerDecodingHandler(claude_tokenizer));
569router.post('/llama3/decode', jsonParser, createWebTokenizerDecodingHandler(llama3_tokenizer));653router.post('/llama3/decode', jsonParser, createWebTokenizerDecodingHandler(llama3_tokenizer));
654router.post('/qwen2/decode', jsonParser, createWebTokenizerDecodingHandler(qwen2Tokenizer));
655router.post('/command-r/decode', jsonParser, createWebTokenizerDecodingHandler(commandTokenizer));
570656
571router.post('/openai/encode', jsonParser, async function (req, res) {657router.post('/openai/encode', jsonParser, async function (req, res) {
572 try {658 try {
@@ -607,6 +693,16 @@ router.post('/openai/encode', jsonParser, async function (req, res) {
607 return handler(req, res);693 return handler(req, res);
608 }694 }
609695
696 if (queryModel.includes('qwen2')) {
697 const handler = createWebTokenizerEncodingHandler(qwen2Tokenizer);
698 return handler(req, res);
699 }
700
701 if (queryModel.includes('command-r')) {
702 const handler = createWebTokenizerEncodingHandler(commandTokenizer);
703 return handler(req, res);
704 }
705
610 const model = getTokenizerModel(queryModel);706 const model = getTokenizerModel(queryModel);
611 const handler = createTiktokenEncodingHandler(model);707 const handler = createTiktokenEncodingHandler(model);
612 return handler(req, res);708 return handler(req, res);
@@ -655,6 +751,16 @@ router.post('/openai/decode', jsonParser, async function (req, res) {
655 return handler(req, res);751 return handler(req, res);
656 }752 }
657753
754 if (queryModel.includes('qwen2')) {
755 const handler = createWebTokenizerDecodingHandler(qwen2Tokenizer);
756 return handler(req, res);
757 }
758
759 if (queryModel.includes('command-r')) {
760 const handler = createWebTokenizerDecodingHandler(commandTokenizer);
761 return handler(req, res);
762 }
763
658 const model = getTokenizerModel(queryModel);764 const model = getTokenizerModel(queryModel);
659 const handler = createTiktokenDecodingHandler(model);765 const handler = createTiktokenDecodingHandler(model);
660 return handler(req, res);766 return handler(req, res);
@@ -711,6 +817,20 @@ router.post('/openai/count', jsonParser, async function (req, res) {
711 return res.send({ 'token_count': num_tokens });817 return res.send({ 'token_count': num_tokens });
712 }818 }
713819
820 if (model === 'qwen2') {
821 const instance = await qwen2Tokenizer.get();
822 if (!instance) throw new Error('Failed to load the Qwen2 tokenizer');
823 num_tokens = countWebTokenizerTokens(instance, req.body);
824 return res.send({ 'token_count': num_tokens });
825 }
826
827 if (model === 'command-r') {
828 const instance = await commandTokenizer.get();
829 if (!instance) throw new Error('Failed to load the Command-R tokenizer');
830 num_tokens = countWebTokenizerTokens(instance, req.body);
831 return res.send({ 'token_count': num_tokens });
832 }
833
714 const tokensPerName = queryModel.includes('gpt-3.5-turbo-0301') ? -1 : 1;834 const tokensPerName = queryModel.includes('gpt-3.5-turbo-0301') ? -1 : 1;
715 const tokensPerMessage = queryModel.includes('gpt-3.5-turbo-0301') ? 4 : 3;835 const tokensPerMessage = queryModel.includes('gpt-3.5-turbo-0301') ? 4 : 3;
716 const tokensPadding = 3;836 const tokensPadding = 3;
src/util.js+15 -0
@@ -647,6 +647,20 @@ function getSeparator(n) {
647 return '='.repeat(n);647 return '='.repeat(n);
648}648}
649649
650/**
651 * Checks if the string is a valid URL.
652 * @param {string} url String to check
653 * @returns {boolean} If the URL is valid
654 */
655function isValidUrl(url) {
656 try {
657 new URL(url);
658 return true;
659 } catch (error) {
660 return false;
661 }
662}
663
650module.exports = {664module.exports = {
651 getConfig,665 getConfig,
652 getConfigValue,666 getConfigValue,
@@ -676,4 +690,5 @@ module.exports = {
676 makeHttp2Request,690 makeHttp2Request,
677 removeColorFormatting,691 removeColorFormatting,
678 getSeparator,692 getSeparator,
693 isValidUrl,
679};694};