Merge branch 'staging' into connection-manager

669c49ebba4de8c8a2d59ccddd01956e24fef9c6

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: 10ddf77948e0d93af1f14a8ea9801ce8a4b5d397 37b3184cda2637c558ac28cbbd06aa9fd5fb6a5c
14 files changed, +238 -34Showing whitespace changes
default/config.yaml+3 -0
@@ -95,6 +95,9 @@ requestOverrides: []
9595enableExtensions: true
9696# Automatically update extensions when a release version changes
9797enableExtensionsAutoUpdate: true
98+# Additional model tokenizers can be downloaded on demand.
99+# Disabling will fallback to another locally available tokenizer.
100+enableDownloadableTokenizers: true
98101# Extension settings
99102extras:
100103 # Disables automatic model download from HuggingFace
public/index.html+5 -3
@@ -1820,12 +1820,12 @@
18201820 <span id="claude_assistant_prefill_text" data-i18n="Assistant Prefill">Assistant Prefill</span>
18211821 <i class="editor_maximize fa-solid fa-maximize right_menu_button" data-for="claude_assistant_prefill" title="Expand the editor" data-i18n="[title]Expand the editor"></i>
18221822 </div>
18231823 <textarea id="claude_assistant_prefill" class="text_pole textarea_compact autoSetHeight" name="assistant_prefill" rows="62" data-i18n="[placeholder]Start Claude's answer with..." placeholder="Start Claude's answer with..."></textarea>
18241824 <div class="flex-container alignItemsCenter">
18251825 <span id="claude_assistant_impersonation_text" data-i18n="Assistant Impersonation Prefill">Assistant Impersonation Prefill</span>
18261826 <i class="editor_maximize fa-solid fa-maximize right_menu_button" data-for="claude_assistant_impersonation" title="Expand the editor" data-i18n="[title]Expand the editor"></i>
18271827 </div>
18281828 <textarea id="claude_assistant_impersonation" class="text_pole textarea_compact autoSetHeight" name="assistant_impersonation" rows="62" data-i18n="[placeholder]Start Claude's answer with..." placeholder="Start Claude's answer with..."></textarea>
18291829 </div>
18301830 <label for="claude_use_sysprompt" class="checkbox_label widthFreeExpand">
18311831 <input id="claude_use_sysprompt" type="checkbox" />
@@ -1845,7 +1845,7 @@
18451845 <div class="fa-solid fa-clock-rotate-left"></div>
18461846 </div>
18471847 </div>
18481848 <textarea id="claude_human_sysprompt_textarea" class="text_pole textarea_compact autoSetHeight" rows="42" data-i18n="[placeholder]Human message" placeholder="Human message, instruction, etc.&#10;Adds nothing when empty, i.e. requires a new prompt with the role 'user'."></textarea>
18491849 </div>
18501850 </div>
18511851 </div>
@@ -3289,6 +3289,8 @@
32893289 <option value="12">Llama 3</option>
32903290 <option value="13">Gemma / Gemini</option>
32913291 <option value="14">Jamba</option>
3292+ <option value="15">Qwen2</option>
3293+ <option value="16">Command-R</option>
32923294 <option value="4">NerdStash (NovelAI Clio)</option>
32933295 <option value="5">NerdStash v2 (NovelAI Kayra)</option>
32943296 <option value="7">Mistral</option>
public/script.js+15 -5
@@ -8504,22 +8504,23 @@ for (const chatCompletionSource of Object.values(chat_completion_sources)) {
85048504 };
85058505}
85068506
85078507async function selectContextCallback(_args, name) {
85088508 if (!name) {
85098509 return power_user.context.preset;
85108510 }
85118511
8512+ const quiet = isTrueBoolean(args?.quiet);
85128513 const contextNames = context_presets.map(preset => preset.name);
85138514 const fuse = new Fuse(contextNames);
85148515 const result = fuse.search(name);
85158516
85168517 if (result.length === 0) {
85178518 !quiet && toastr.warning(`Context template "${name}" not found`);
85188519 return '';
85198520 }
85208521
85218522 const foundName = result[0].item;
85228523 selectContextPreset(foundName, quiet);
85238524 return foundName;
85248525}
85258526
@@ -8528,16 +8529,16 @@ async function selectInstructCallback(args, name) {
85288529 return power_user.instruct.preset;
85298530 }
85308531
8532+ const quiet = isTrueBoolean(args?.quiet);
85318533 const instructNames = instruct_presets.map(preset => preset.name);
85328534 const fuse = new Fuse(instructNames);
85338535 const result = fuse.search(name);
85348536
85358537 if (result.length === 0) {
85368538 !quiet && toastr.warning(`Instruct template "${name}" not found`);
85378539 return '';
85388540 }
85398541
8540- const quiet = isTrueBoolean(args?.quiet);
85418542 const foundName = result[0].item;
85428543 selectInstructPreset(foundName, quiet);
85438544 return foundName;
@@ -9283,6 +9284,15 @@ jQuery(async function () {
92839284 name: 'context',
92849285 callback: selectContextCallback,
92859286 returns: 'template name',
9287+ namedArgumentList: [
9288+ SlashCommandNamedArgument.fromProps({
9289+ name: 'quiet',
9290+ description: 'Suppress the toast message on template change',
9291+ typeList: [ARGUMENT_TYPE.BOOLEAN],
9292+ defaultValue: 'false',
9293+ enumList: commonEnumProviders.boolean('trueFalse')(),
9294+ }),
9295+ ],
92869296 unnamedArgumentList: [
92879297 SlashCommandArgument.fromProps({
92889298 description: 'context template name',
public/scripts/extensions/stable-diffusion/index.js+11 -5
@@ -724,7 +724,7 @@ function onChatChanged() {
724724}
725725
726726async function adjustElementScrollHeight() {
727727 if (CSS.supports('field-sizing', 'content') || !$('.sd_settings').is(':visible')) {
728728 return;
729729 }
730730
@@ -737,17 +737,19 @@ async function adjustElementScrollHeight() {
737737async function onCharacterPromptInput() {
738738 const key = getCharaFilename(this_chid);
739739 extension_settings.sd.character_prompts[key] = $('#sd_character_prompt').val();
740- await resetScrollHeight($(this));
741740 saveSettingsDebounced();
742741 writePromptFieldsDebounced(this_chid);
742+ if (CSS.supports('field-sizing', 'content')) return;
743+ await resetScrollHeight($(this));
743744}
744745
745746async function onCharacterNegativePromptInput() {
746747 const key = getCharaFilename(this_chid);
747748 extension_settings.sd.character_negative_prompts[key] = $('#sd_character_negative_prompt').val();
748- await resetScrollHeight($(this));
749749 saveSettingsDebounced();
750750 writePromptFieldsDebounced(this_chid);
751+ if (CSS.supports('field-sizing', 'content')) return;
752+ await resetScrollHeight($(this));
751753}
752754
753755function getCharacterPrefix() {
@@ -856,14 +858,16 @@ function onStepsInput() {
856858
857859async function onPromptPrefixInput() {
858860 extension_settings.sd.prompt_prefix = $('#sd_prompt_prefix').val();
859- await resetScrollHeight($(this));
860861 saveSettingsDebounced();
862+ if (CSS.supports('field-sizing', 'content')) return;
863+ await resetScrollHeight($(this));
861864}
862865
863866async function onNegativePromptInput() {
864867 extension_settings.sd.negative_prompt = $('#sd_negative_prompt').val();
865- await resetScrollHeight($(this));
866868 saveSettingsDebounced();
869+ if (CSS.supports('field-sizing', 'content')) return;
870+ await resetScrollHeight($(this));
867871}
868872
869873function onSamplerChange() {
@@ -3911,12 +3915,14 @@ jQuery(async () => {
39113915 $('#sd_stability_style_preset').on('change', onStabilityStylePresetChange);
39123916 $('#sd_huggingface_model_id').on('input', onHFModelInput);
39133917
3918+ if (!CSS.supports('field-sizing', 'content')) {
39143919 $('.sd_settings .inline-drawer-toggle').on('click', function () {
39153920 initScrollHeight($('#sd_prompt_prefix'));
39163921 initScrollHeight($('#sd_negative_prompt'));
39173922 initScrollHeight($('#sd_character_prompt'));
39183923 initScrollHeight($('#sd_character_negative_prompt'));
39193924 });
3925+ }
39203926
39213927 for (const [key, value] of Object.entries(resolutionOptions)) {
39223928 const option = document.createElement('option');
public/scripts/extensions/stable-diffusion/settings.html+4 -4
@@ -408,16 +408,16 @@
408408 </div>
409409 </div>
410410 <label for="sd_prompt_prefix" data-i18n="Common prompt prefix">Common prompt prefix</label>
411411 <textarea id="sd_prompt_prefix" class="text_pole textarea_compact autoSetHeight" data-i18n="[placeholder]sd_prompt_prefix_placeholder" placeholder="Use {prompt} to specify where the generated prompt will be inserted"></textarea>
412412 <label for="sd_negative_prompt" data-i18n="Negative common prompt prefix">Negative common prompt prefix</label>
413413 <textarea id="sd_negative_prompt" class="text_pole textarea_compact autoSetHeight"></textarea>
414414 <div id="sd_character_prompt_block">
415415 <label for="sd_character_prompt" data-i18n="Character-specific prompt prefix">Character-specific prompt prefix</label>
416416 <small data-i18n="Won't be used in groups.">Won't be used in groups.</small>
417417 <textarea id="sd_character_prompt" class="text_pole textarea_compact autoSetHeight" data-i18n="[placeholder]sd_character_prompt_placeholder" placeholder="Any characteristics that describe the currently selected character. Will be added after a common prompt prefix.&#10;Example: female, green eyes, brown hair, pink shirt"></textarea>
418418 <label for="sd_character_negative_prompt" data-i18n="Character-specific negative prompt prefix">Character-specific negative prompt prefix</label>
419419 <small data-i18n="Won't be used in groups.">Won't be used in groups.</small>
420420 <textarea id="sd_character_negative_prompt" class="text_pole textarea_compact autoSetHeight" data-i18n="[placeholder]sd_character_negative_prompt_placeholder" placeholder="Any characteristics that should not appear for the selected character. Will be added after a negative common prompt prefix.&#10;Example: jewellery, shoes, glasses"></textarea>
421421 <label for="sd_character_prompt_share" class="checkbox_label flexWrap marginTop5">
422422 <input id="sd_character_prompt_share" type="checkbox" />
423423 <span data-i18n="Shareable">
public/scripts/extensions/token-counter/index.js+2 -0
@@ -59,8 +59,10 @@ async function doTokenCounter() {
5959 $('#tokenized_chunks_display').text('—');
6060 }
6161
62+ if (!CSS.supports('field-sizing', 'content')) {
6263 await resetScrollHeight($('#token_counter_textarea'));
6364 await resetScrollHeight($('#token_counter_ids'));
65+ }
6466 }, debounce_timeout.relaxed);
6567 dialog.find('#token_counter_textarea').on('input', () => countDebounced());
6668
public/scripts/extensions/token-counter/style.css+5 -0
@@ -4,3 +4,8 @@
44 padding: 2px;
55 display: inline-block;
66}
7+
8+#token_counter_textarea,
9+#token_counter_ids {
10+ field-sizing: content;
11+}
public/scripts/group-chats.js+3 -0
@@ -1414,9 +1414,12 @@ function toggleHiddenControls(group, generationMode = null) {
14141414 const isJoin = [group_generation_mode.APPEND, group_generation_mode.APPEND_DISABLED].includes(generationMode ?? group?.generation_mode);
14151415 $('#rm_group_generation_mode_join_prefix').parent().toggle(isJoin);
14161416 $('#rm_group_generation_mode_join_suffix').parent().toggle(isJoin);
1417+
1418+ if (!CSS.supports('field-sizing', 'content')) {
14171419 initScrollHeight($('#rm_group_generation_mode_join_prefix'));
14181420 initScrollHeight($('#rm_group_generation_mode_join_suffix'));
14191421 }
1422+}
14201423
14211424function select_group_chats(groupId, skipAnimation) {
14221425 openGroupId = groupId;
public/scripts/instruct-mode.js+4 -3
@@ -130,19 +130,20 @@ function highlightDefaultPreset() {
130130/**
131131 * Select context template if not already selected.
132132 * @param {string} preset Preset name.
133+ * @param {boolean} quiet Suppress info message.
133134 */
134135export function selectContextPreset(preset, quiet) {
135136 // If context template is not already selected, select it
136137 if (preset !== power_user.context.preset) {
137138 $('#context_presets').val(preset).trigger('change');
138139 !quiet && toastr.info(`Context Template: preset "${preset}" auto-selected`);
139140 }
140141
141142 // If instruct mode is disabled, enable it, except for default context template
142143 if (!power_user.instruct.enabled && preset !== power_user.default_context) {
143144 power_user.instruct.enabled = true;
144145 $('#instruct_enabled').prop('checked', true).trigger('change');
145146 !quiet && toastr.info('Instruct Mode enabled');
146147 }
147148
148149 saveSettingsDebounced();
public/scripts/textgen-models.js+7 -0
@@ -590,6 +590,9 @@ function calculateOpenRouterCost() {
590590export function getCurrentOpenRouterModelTokenizer() {
591591 const modelId = textgen_settings.openrouter_model;
592592 const model = openRouterModels.find(x => x.id === modelId);
593+ if (modelId?.includes('jamba')) {
594+ return tokenizers.JAMBA;
595+ }
593596 switch (model?.architecture?.tokenizer) {
594597 case 'Llama2':
595598 return tokenizers.LLAMA;
@@ -603,6 +606,10 @@ export function getCurrentOpenRouterModelTokenizer() {
603606 return tokenizers.GEMMA;
604607 case 'Claude':
605608 return tokenizers.CLAUDE;
609+ case 'Cohere':
610+ return tokenizers.COMMAND_R;
611+ case 'Qwen':
612+ return tokenizers.QWEN2;
606613 default:
607614 return tokenizers.OPENAI;
608615 }
public/scripts/tokenizers.js+30 -0
@@ -28,6 +28,8 @@ export const tokenizers = {
2828 LLAMA3: 12,
2929 GEMMA: 13,
3030 JAMBA: 14,
31+ QWEN2: 15,
32+ COMMAND_R: 16,
3133 BEST_MATCH: 99,
3234};
3335
@@ -105,6 +107,16 @@ const TOKENIZER_URLS = {
105107 decode: '/api/tokenizers/jamba/decode',
106108 count: '/api/tokenizers/jamba/encode',
107109 },
110+ [tokenizers.QWEN2]: {
111+ encode: '/api/tokenizers/qwen2/encode',
112+ decode: '/api/tokenizers/qwen2/decode',
113+ count: '/api/tokenizers/qwen2/encode',
114+ },
115+ [tokenizers.COMMAND_R]: {
116+ encode: '/api/tokenizers/command-r/encode',
117+ decode: '/api/tokenizers/command-r/decode',
118+ count: '/api/tokenizers/command-r/encode',
119+ },
108120 [tokenizers.API_TEXTGENERATIONWEBUI]: {
109121 encode: '/api/tokenizers/remote/textgenerationwebui/encode',
110122 count: '/api/tokenizers/remote/textgenerationwebui/encode',
@@ -293,6 +305,12 @@ export function getTokenizerBestMatch(forApi) {
293305 if (model.includes('jamba')) {
294306 return tokenizers.JAMBA;
295307 }
308+ if (model.includes('command-r')) {
309+ return tokenizers.COMMAND_R;
310+ }
311+ if (model.includes('qwen2')) {
312+ return tokenizers.QWEN2;
313+ }
296314 }
297315
298316 return tokenizers.LLAMA;
@@ -511,6 +529,8 @@ export function getTokenizerModel() {
511529 const yiTokenizer = 'yi';
512530 const gemmaTokenizer = 'gemma';
513531 const jambaTokenizer = 'jamba';
532+ const qwen2Tokenizer = 'qwen2';
533+ const commandRTokenizer = 'command-r';
514534
515535 // Assuming no one would use it for different models.. right?
516536 if (oai_settings.chat_completion_source == chat_completion_sources.SCALE) {
@@ -558,6 +578,12 @@ export function getTokenizerModel() {
558578 else if (model?.architecture?.tokenizer === 'Gemini') {
559579 return gemmaTokenizer;
560580 }
581+ else if (model?.architecture?.tokenizer === 'Qwen') {
582+ return qwen2Tokenizer;
583+ }
584+ else if (model?.architecture?.tokenizer === 'Cohere') {
585+ return commandRTokenizer;
586+ }
561587 else if (oai_settings.openrouter_model.includes('gpt-4o')) {
562588 return gpt4oTokenizer;
563589 }
@@ -581,6 +607,10 @@ export function getTokenizerModel() {
581607 }
582608 }
583609
610+ if (oai_settings.chat_completion_source == chat_completion_sources.COHERE) {
611+ return commandRTokenizer;
612+ }
613+
584614 if (oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) {
585615 return gemmaTokenizer;
586616 }
public/style.css+4 -4
@@ -182,6 +182,10 @@ body.movingUI ::-webkit-scrollbar-thumb:vertical {
182182 min-width: 40px;
183183}
184184
185+::-webkit-scrollbar-corner {
186+ background-color: transparent;
187+}
188+
185189table.responsiveTable {
186190 width: 100%;
187191 margin: 10px 0;
@@ -4675,10 +4679,6 @@ body:not(.sd) .mes_img_swipes {
46754679 width: 100%;
46764680}
46774681
4678-.img_enlarged_holder::-webkit-scrollbar-corner {
4679- background-color: transparent;
4680-}
4681-
46824682.img_enlarged_container pre code {
46834683 position: relative;
46844684 display: block;
src/endpoints/tokenizers.js+130 -10
@@ -4,13 +4,12 @@ const express = require('express');
44const { SentencePieceProcessor } = require('@agnai/sentencepiece-js');
55const tiktoken = require('tiktoken');
66const { Tokenizer } = require('@agnai/web-tokenizers');
77const { convertClaudePrompt, convertGooglePrompt } = require('../prompt-converters');
8-const { readSecret, SECRET_KEYS } = require('./secrets');
98const { TEXTGEN_TYPES } = require('../constants');
109const { jsonParser } = require('../express-common');
1110const { setAdditionalHeaders } = require('../additional-headers');
12-
11+const { getConfigValue, isValidUrl } = require('../util');
13-const API_MAKERSUITE = 'https://generativelanguage.googleapis.com';
12+const writeFileAtomicSync = require('write-file-atomic').sync;
1413
1514/**
1615 * @typedef { (req: import('express').Request, res: import('express').Response) => Promise<any> } TokenizationHandler
@@ -53,6 +52,65 @@ const TEXT_COMPLETION_MODELS = [
5352];
5453
5554const CHARS_PER_TOKEN = 3.35;
55+const IS_DOWNLOAD_ALLOWED = getConfigValue('enableDownloadableTokenizers', true);
56+
57+/**
58+ * Gets a path to the tokenizer model. Downloads the model if it's a URL.
59+ * @param {string} model Model URL or path
60+ * @param {string|undefined} fallbackModel Fallback model path\
61+ * @returns {Promise<string>} Path to the tokenizer model
62+ */
63+async function getPathToTokenizer(model, fallbackModel) {
64+ if (!isValidUrl(model)) {
65+ return model;
66+ }
67+
68+ try {
69+ const url = new URL(model);
70+
71+ if (!['https:', 'http:'].includes(url.protocol)) {
72+ throw new Error('Invalid URL protocol');
73+ }
74+
75+ const fileName = url.pathname.split('/').pop();
76+
77+ if (!fileName) {
78+ throw new Error('Failed to extract the file name from the URL');
79+ }
80+
81+ const CACHE_PATH = path.join(global.DATA_ROOT, '_cache');
82+ if (!fs.existsSync(CACHE_PATH)) {
83+ fs.mkdirSync(CACHE_PATH, { recursive: true });
84+ }
85+
86+ const cachedFile = path.join(CACHE_PATH, fileName);
87+ if (fs.existsSync(cachedFile)) {
88+ return cachedFile;
89+ }
90+
91+ if (!IS_DOWNLOAD_ALLOWED) {
92+ throw new Error('Downloading tokenizers is disabled, the model is not cached');
93+ }
94+
95+ console.log('Downloading tokenizer model:', model);
96+ const response = await fetch(model);
97+ if (!response.ok) {
98+ throw new Error(`Failed to fetch the model: ${response.status} ${response.statusText}`);
99+ }
100+
101+ const arrayBuffer = await response.arrayBuffer();
102+ writeFileAtomicSync(cachedFile, Buffer.from(arrayBuffer));
103+ return cachedFile;
104+ } catch (error) {
105+ const getLastSegment = str => str?.split('/')?.pop() || '';
106+ if (fallbackModel) {
107+ console.log(`Could not get a tokenizer from ${getLastSegment(model)}. Reason: ${error.message}. Using a fallback model: ${getLastSegment(fallbackModel)}.`);
108+ return fallbackModel;
109+ }
110+
111+ throw new Error(`Failed to instantiate a tokenizer and fallback is not provided. Reason: ${error.message}`);
112+ }
113+}
56114
57115/**
58116 * Sentencepiece tokenizer for tokenizing text.
@@ -66,13 +124,19 @@ class SentencePieceTokenizer {
66124 * @type {string} Path to the tokenizer model
67125 */
68126 #model;
127+ /**
128+ * @type {string|undefined} Path to the fallback model
129+ */
130+ #fallbackModel;
69131
70132 /**
71133 * Creates a new Sentencepiece tokenizer.
72134 * @param {string} model Path to the tokenizer model
135+ * @param {string} [fallbackModel] Path to the fallback model
73136 */
74137 constructor(model, fallbackModel) {
75138 this.#model = model;
139+ this.#fallbackModel = fallbackModel;
76140 }
77141
78142 /**
@@ -85,9 +149,10 @@ class SentencePieceTokenizer {
85149 }
86150
87151 try {
152+ const pathToModel = await getPathToTokenizer(this.#model, this.#fallbackModel);
88153 this.#instance = new SentencePieceProcessor();
89154 await this.#instance.load(this.#modelpathToModel);
90155 console.log('Instantiated the tokenizer for', path.parse(this.#modelpathToModel).name);
91156 return this.#instance;
92157 } catch (error) {
93158 console.error('Sentencepiece tokenizer failed to load: ' + this.#model, error);
@@ -108,13 +173,19 @@ class WebTokenizer {
108173 * @type {string} Path to the tokenizer model
109174 */
110175 #model;
176+ /**
177+ * @type {string|undefined} Path to the fallback model
178+ */
179+ #fallbackModel;
111180
112181 /**
113182 * Creates a new Web tokenizer.
114183 * @param {string} model Path to the tokenizer model
184+ * @param {string} [fallbackModel] Path to the fallback model
115185 */
116186 constructor(model, fallbackModel) {
117187 this.#model = model;
188+ this.#fallbackModel = fallbackModel;
118189 }
119190
120191 /**
@@ -127,9 +198,10 @@ class WebTokenizer {
127198 }
128199
129200 try {
130201 const arrayBufferpathToModel = fs.readFileSyncawait getPathToTokenizer(this.#model), this.buffer#fallbackModel);
202+ const arrayBuffer = fs.readFileSync(pathToModel).buffer;
131203 this.#instance = await Tokenizer.fromJSON(arrayBuffer);
132204 console.log('Instantiated the tokenizer for', path.parse(this.#modelpathToModel).name);
133205 return this.#instance;
134206 } catch (error) {
135207 console.error('Web tokenizer failed to load: ' + this.#model, error);
@@ -147,6 +219,8 @@ const spp_gemma = new SentencePieceTokenizer('src/tokenizers/gemma.model');
147219const spp_jamba = new SentencePieceTokenizer('src/tokenizers/jamba.model');
148220const claude_tokenizer = new WebTokenizer('src/tokenizers/claude.json');
149221const llama3_tokenizer = new WebTokenizer('src/tokenizers/llama3.json');
222+const commandTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/command-r.json', 'src/tokenizers/llama3.json');
223+const qwen2Tokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/qwen2.json', 'src/tokenizers/llama3.json');
150224
151225const sentencepieceTokenizers = [
152226 'llama',
@@ -332,6 +406,14 @@ function getTokenizerModel(requestModel) {
332406 return 'jamba';
333407 }
334408
409+ if (requestModel.includes('qwen2')) {
410+ return 'qwen2';
411+ }
412+
413+ if (requestModel.includes('command-r')) {
414+ return 'command-r';
415+ }
416+
335417 // default
336418 return 'gpt-3.5-turbo';
337419}
@@ -557,6 +639,8 @@ router.post('/jamba/encode', jsonParser, createSentencepieceEncodingHandler(spp_
557639router.post('/gpt2/encode', jsonParser, createTiktokenEncodingHandler('gpt2'));
558640router.post('/claude/encode', jsonParser, createWebTokenizerEncodingHandler(claude_tokenizer));
559641router.post('/llama3/encode', jsonParser, createWebTokenizerEncodingHandler(llama3_tokenizer));
642+router.post('/qwen2/encode', jsonParser, createWebTokenizerEncodingHandler(qwen2Tokenizer));
643+router.post('/command-r/encode', jsonParser, createWebTokenizerEncodingHandler(commandTokenizer));
560644router.post('/llama/decode', jsonParser, createSentencepieceDecodingHandler(spp_llama));
561645router.post('/nerdstash/decode', jsonParser, createSentencepieceDecodingHandler(spp_nerd));
562646router.post('/nerdstash_v2/decode', jsonParser, createSentencepieceDecodingHandler(spp_nerd_v2));
@@ -567,6 +651,8 @@ router.post('/jamba/decode', jsonParser, createSentencepieceDecodingHandler(spp_
567651router.post('/gpt2/decode', jsonParser, createTiktokenDecodingHandler('gpt2'));
568652router.post('/claude/decode', jsonParser, createWebTokenizerDecodingHandler(claude_tokenizer));
569653router.post('/llama3/decode', jsonParser, createWebTokenizerDecodingHandler(llama3_tokenizer));
654+router.post('/qwen2/decode', jsonParser, createWebTokenizerDecodingHandler(qwen2Tokenizer));
655+router.post('/command-r/decode', jsonParser, createWebTokenizerDecodingHandler(commandTokenizer));
570656
571657router.post('/openai/encode', jsonParser, async function (req, res) {
572658 try {
@@ -607,6 +693,16 @@ router.post('/openai/encode', jsonParser, async function (req, res) {
607693 return handler(req, res);
608694 }
609695
696+ if (queryModel.includes('qwen2')) {
697+ const handler = createWebTokenizerEncodingHandler(qwen2Tokenizer);
698+ return handler(req, res);
699+ }
700+
701+ if (queryModel.includes('command-r')) {
702+ const handler = createWebTokenizerEncodingHandler(commandTokenizer);
703+ return handler(req, res);
704+ }
705+
610706 const model = getTokenizerModel(queryModel);
611707 const handler = createTiktokenEncodingHandler(model);
612708 return handler(req, res);
@@ -655,6 +751,16 @@ router.post('/openai/decode', jsonParser, async function (req, res) {
655751 return handler(req, res);
656752 }
657753
754+ if (queryModel.includes('qwen2')) {
755+ const handler = createWebTokenizerDecodingHandler(qwen2Tokenizer);
756+ return handler(req, res);
757+ }
758+
759+ if (queryModel.includes('command-r')) {
760+ const handler = createWebTokenizerDecodingHandler(commandTokenizer);
761+ return handler(req, res);
762+ }
763+
658764 const model = getTokenizerModel(queryModel);
659765 const handler = createTiktokenDecodingHandler(model);
660766 return handler(req, res);
@@ -711,6 +817,20 @@ router.post('/openai/count', jsonParser, async function (req, res) {
711817 return res.send({ 'token_count': num_tokens });
712818 }
713819
820+ if (model === 'qwen2') {
821+ const instance = await qwen2Tokenizer.get();
822+ if (!instance) throw new Error('Failed to load the Qwen2 tokenizer');
823+ num_tokens = countWebTokenizerTokens(instance, req.body);
824+ return res.send({ 'token_count': num_tokens });
825+ }
826+
827+ if (model === 'command-r') {
828+ const instance = await commandTokenizer.get();
829+ if (!instance) throw new Error('Failed to load the Command-R tokenizer');
830+ num_tokens = countWebTokenizerTokens(instance, req.body);
831+ return res.send({ 'token_count': num_tokens });
832+ }
833+
714834 const tokensPerName = queryModel.includes('gpt-3.5-turbo-0301') ? -1 : 1;
715835 const tokensPerMessage = queryModel.includes('gpt-3.5-turbo-0301') ? 4 : 3;
716836 const tokensPadding = 3;
src/util.js+15 -0
@@ -647,6 +647,20 @@ function getSeparator(n) {
647647 return '='.repeat(n);
648648}
649649
650+/**
651+ * Checks if the string is a valid URL.
652+ * @param {string} url String to check
653+ * @returns {boolean} If the URL is valid
654+ */
655+function isValidUrl(url) {
656+ try {
657+ new URL(url);
658+ return true;
659+ } catch (error) {
660+ return false;
661+ }
662+}
663+
650664module.exports = {
651665 getConfig,
652666 getConfigValue,
@@ -676,4 +690,5 @@ module.exports = {
676690 makeHttp2Request,
677691 removeColorFormatting,
678692 getSeparator,
693+ isValidUrl,
679694};