Added command-a-03-2025 and command-a tokenizer

a77f4045f818677e6aa1077aefb7a8f046a4d207

bmen25124 <bmen25124@gmail.com>

Parents: 79a8080b7d11f5d905590f659f268b5de6c79051
4 files changed, +57 -7Showing whitespace changes
public/index.html+1 -0
@@ -3333,6 +3333,7 @@
3333 <option value="command-r-08-2024">command-r-08-2024</option>3333 <option value="command-r-08-2024">command-r-08-2024</option>
3334 <option value="command-r-plus-08-2024">command-r-plus-08-2024</option>3334 <option value="command-r-plus-08-2024">command-r-plus-08-2024</option>
3335 <option value="command-r7b-12-2024">command-r7b-12-2024</option>3335 <option value="command-r7b-12-2024">command-r7b-12-2024</option>
3336 <option value="command-a-03-2025">command-a-03-2025</option>
3336 </optgroup>3337 </optgroup>
3337 <optgroup label="Nightly">3338 <optgroup label="Nightly">
3338 <option value="command-light-nightly">command-light-nightly</option>3339 <option value="command-light-nightly">command-light-nightly</option>
public/scripts/openai.js+3 -0
@@ -4448,6 +4448,9 @@ async function onModelChange() {
4448 else if (oai_settings.cohere_model.includes('command-r') || ['c4ai-aya-23', 'c4ai-aya-expanse-32b', 'command-nightly'].includes(oai_settings.cohere_model)) {4448 else if (oai_settings.cohere_model.includes('command-r') || ['c4ai-aya-23', 'c4ai-aya-expanse-32b', 'command-nightly'].includes(oai_settings.cohere_model)) {
4449 $('#openai_max_context').attr('max', max_128k);4449 $('#openai_max_context').attr('max', max_128k);
4450 }4450 }
4451 else if (['command-a-03-2025'].includes(oai_settings.cohere_model)) {
4452 $('#openai_max_context').attr('max', max_256k);
4453 }
4451 else if (['c4ai-aya-23-8b', 'c4ai-aya-expanse-8b'].includes(oai_settings.cohere_model)) {4454 else if (['c4ai-aya-23-8b', 'c4ai-aya-expanse-8b'].includes(oai_settings.cohere_model)) {
4452 $('#openai_max_context').attr('max', max_8k);4455 $('#openai_max_context').attr('max', max_8k);
4453 }4456 }
public/scripts/tokenizers.js+17 -0
@@ -32,6 +32,7 @@ export const tokenizers = {
32 COMMAND_R: 16,32 COMMAND_R: 16,
33 NEMO: 17,33 NEMO: 17,
34 DEEPSEEK: 18,34 DEEPSEEK: 18,
35 COMMAND_A: 19,
35 BEST_MATCH: 99,36 BEST_MATCH: 99,
36};37};
3738
@@ -45,6 +46,7 @@ export const ENCODE_TOKENIZERS = [
45 tokenizers.JAMBA,46 tokenizers.JAMBA,
46 tokenizers.QWEN2,47 tokenizers.QWEN2,
47 tokenizers.COMMAND_R,48 tokenizers.COMMAND_R,
49 tokenizers.COMMAND_A,
48 tokenizers.NEMO,50 tokenizers.NEMO,
49 tokenizers.DEEPSEEK,51 tokenizers.DEEPSEEK,
50 // uncomment when NovelAI releases Kayra and Clio weights, lol52 // uncomment when NovelAI releases Kayra and Clio weights, lol
@@ -129,6 +131,11 @@ const TOKENIZER_URLS = {
129 decode: '/api/tokenizers/command-r/decode',131 decode: '/api/tokenizers/command-r/decode',
130 count: '/api/tokenizers/command-r/encode',132 count: '/api/tokenizers/command-r/encode',
131 },133 },
134 [tokenizers.COMMAND_A]: {
135 encode: '/api/tokenizers/command-a/encode',
136 decode: '/api/tokenizers/command-a/decode',
137 count: '/api/tokenizers/command-a/encode',
138 },
132 [tokenizers.NEMO]: {139 [tokenizers.NEMO]: {
133 encode: '/api/tokenizers/nemo/encode',140 encode: '/api/tokenizers/nemo/encode',
134 decode: '/api/tokenizers/nemo/decode',141 decode: '/api/tokenizers/nemo/decode',
@@ -340,6 +347,9 @@ export function getTokenizerBestMatch(forApi) {
340 if (model.includes('command-r')) {347 if (model.includes('command-r')) {
341 return tokenizers.COMMAND_R;348 return tokenizers.COMMAND_R;
342 }349 }
350 if (model.includes('command-a')) {
351 return tokenizers.COMMAND_A;
352 }
343 if (model.includes('qwen2')) {353 if (model.includes('qwen2')) {
344 return tokenizers.QWEN2;354 return tokenizers.QWEN2;
345 }355 }
@@ -572,6 +582,7 @@ export function getTokenizerModel() {
572 const jambaTokenizer = 'jamba';582 const jambaTokenizer = 'jamba';
573 const qwen2Tokenizer = 'qwen2';583 const qwen2Tokenizer = 'qwen2';
574 const commandRTokenizer = 'command-r';584 const commandRTokenizer = 'command-r';
585 const commandATokenizer = 'command-a';
575 const nemoTokenizer = 'nemo';586 const nemoTokenizer = 'nemo';
576 const deepseekTokenizer = 'deepseek';587 const deepseekTokenizer = 'deepseek';
577588
@@ -626,6 +637,9 @@ export function getTokenizerModel() {
626 return qwen2Tokenizer;637 return qwen2Tokenizer;
627 }638 }
628 else if (model?.architecture?.tokenizer === 'Cohere') {639 else if (model?.architecture?.tokenizer === 'Cohere') {
640 if (oai_settings.openrouter_model.includes('command-a')) {
641 return commandATokenizer;
642 }
629 return commandRTokenizer;643 return commandRTokenizer;
630 }644 }
631 else if (oai_settings.openrouter_model.includes('gpt-4o')) {645 else if (oai_settings.openrouter_model.includes('gpt-4o')) {
@@ -652,6 +666,9 @@ export function getTokenizerModel() {
652 }666 }
653667
654 if (oai_settings.chat_completion_source == chat_completion_sources.COHERE) {668 if (oai_settings.chat_completion_source == chat_completion_sources.COHERE) {
669 if (oai_settings.cohere_model.includes('command-a')) {
670 return commandATokenizer;
671 }
655 return commandRTokenizer;672 return commandRTokenizer;
656 }673 }
657674
src/endpoints/tokenizers.js+36 -7
@@ -222,7 +222,8 @@ const spp_gemma = new SentencePieceTokenizer('src/tokenizers/gemma.model');
222const spp_jamba = new SentencePieceTokenizer('src/tokenizers/jamba.model');222const spp_jamba = new SentencePieceTokenizer('src/tokenizers/jamba.model');
223const claude_tokenizer = new WebTokenizer('src/tokenizers/claude.json');223const claude_tokenizer = new WebTokenizer('src/tokenizers/claude.json');
224const llama3_tokenizer = new WebTokenizer('src/tokenizers/llama3.json');224const llama3_tokenizer = new WebTokenizer('src/tokenizers/llama3.json');
225const commandTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/command-r.json', 'src/tokenizers/llama3.json');225const commandRTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/command-r.json', 'src/tokenizers/llama3.json');
226const commandATokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/command-a.json', 'src/tokenizers/llama3.json');
226const qwen2Tokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/qwen2.json', 'src/tokenizers/llama3.json');227const qwen2Tokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/qwen2.json', 'src/tokenizers/llama3.json');
227const nemoTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/nemo.json', 'src/tokenizers/llama3.json');228const nemoTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/nemo.json', 'src/tokenizers/llama3.json');
228const deepseekTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/deepseek.json', 'src/tokenizers/llama3.json');229const deepseekTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/deepseek.json', 'src/tokenizers/llama3.json');
@@ -241,6 +242,7 @@ export const webTokenizers = [
241 'claude',242 'claude',
242 'llama3',243 'llama3',
243 'command-r',244 'command-r',
245 'command-a',
244 'qwen2',246 'qwen2',
245 'nemo',247 'nemo',
246 'deepseek',248 'deepseek',
@@ -298,7 +300,11 @@ export function getWebTokenizer(model) {
298 }300 }
299301
300 if (model.includes('command-r')) {302 if (model.includes('command-r')) {
301 return commandTokenizer;303 return commandRTokenizer;
304 }
305
306 if (model.includes('command-a')) {
307 return commandATokenizer;
302 }308 }
303309
304 if (model.includes('qwen2')) {310 if (model.includes('qwen2')) {
@@ -469,6 +475,10 @@ export function getTokenizerModel(requestModel) {
469 return 'command-r';475 return 'command-r';
470 }476 }
471477
478 if (requestModel.includes('command-a')) {
479 return 'command-a';
480 }
481
472 if (requestModel.includes('nemo')) {482 if (requestModel.includes('nemo')) {
473 return 'nemo';483 return 'nemo';
474 }484 }
@@ -699,7 +709,8 @@ router.post('/gpt2/encode', createTiktokenEncodingHandler('gpt2'));
699router.post('/claude/encode', createWebTokenizerEncodingHandler(claude_tokenizer));709router.post('/claude/encode', createWebTokenizerEncodingHandler(claude_tokenizer));
700router.post('/llama3/encode', createWebTokenizerEncodingHandler(llama3_tokenizer));710router.post('/llama3/encode', createWebTokenizerEncodingHandler(llama3_tokenizer));
701router.post('/qwen2/encode', createWebTokenizerEncodingHandler(qwen2Tokenizer));711router.post('/qwen2/encode', createWebTokenizerEncodingHandler(qwen2Tokenizer));
702router.post('/command-r/encode', createWebTokenizerEncodingHandler(commandTokenizer));712router.post('/command-r/encode', createWebTokenizerEncodingHandler(commandRTokenizer));
713router.post('/command-a/encode', createWebTokenizerEncodingHandler(commandATokenizer));
703router.post('/nemo/encode', createWebTokenizerEncodingHandler(nemoTokenizer));714router.post('/nemo/encode', createWebTokenizerEncodingHandler(nemoTokenizer));
704router.post('/deepseek/encode', createWebTokenizerEncodingHandler(deepseekTokenizer));715router.post('/deepseek/encode', createWebTokenizerEncodingHandler(deepseekTokenizer));
705router.post('/llama/decode', createSentencepieceDecodingHandler(spp_llama));716router.post('/llama/decode', createSentencepieceDecodingHandler(spp_llama));
@@ -713,7 +724,8 @@ router.post('/gpt2/decode', createTiktokenDecodingHandler('gpt2'));
713router.post('/claude/decode', createWebTokenizerDecodingHandler(claude_tokenizer));724router.post('/claude/decode', createWebTokenizerDecodingHandler(claude_tokenizer));
714router.post('/llama3/decode', createWebTokenizerDecodingHandler(llama3_tokenizer));725router.post('/llama3/decode', createWebTokenizerDecodingHandler(llama3_tokenizer));
715router.post('/qwen2/decode', createWebTokenizerDecodingHandler(qwen2Tokenizer));726router.post('/qwen2/decode', createWebTokenizerDecodingHandler(qwen2Tokenizer));
716router.post('/command-r/decode', createWebTokenizerDecodingHandler(commandTokenizer));727router.post('/command-r/decode', createWebTokenizerDecodingHandler(commandRTokenizer));
728router.post('/command-a/decode', createWebTokenizerDecodingHandler(commandATokenizer));
717router.post('/nemo/decode', createWebTokenizerDecodingHandler(nemoTokenizer));729router.post('/nemo/decode', createWebTokenizerDecodingHandler(nemoTokenizer));
718router.post('/deepseek/decode', createWebTokenizerDecodingHandler(deepseekTokenizer));730router.post('/deepseek/decode', createWebTokenizerDecodingHandler(deepseekTokenizer));
719731
@@ -762,7 +774,12 @@ router.post('/openai/encode', async function (req, res) {
762 }774 }
763775
764 if (queryModel.includes('command-r')) {776 if (queryModel.includes('command-r')) {
765 const handler = createWebTokenizerEncodingHandler(commandTokenizer);777 const handler = createWebTokenizerEncodingHandler(commandRTokenizer);
778 return handler(req, res);
779 }
780
781 if (queryModel.includes('command-a')) {
782 const handler = createWebTokenizerEncodingHandler(commandATokenizer);
766 return handler(req, res);783 return handler(req, res);
767 }784 }
768785
@@ -830,7 +847,12 @@ router.post('/openai/decode', async function (req, res) {
830 }847 }
831848
832 if (queryModel.includes('command-r')) {849 if (queryModel.includes('command-r')) {
833 const handler = createWebTokenizerDecodingHandler(commandTokenizer);850 const handler = createWebTokenizerDecodingHandler(commandRTokenizer);
851 return handler(req, res);
852 }
853
854 if (queryModel.includes('command-a')) {
855 const handler = createWebTokenizerDecodingHandler(commandATokenizer);
834 return handler(req, res);856 return handler(req, res);
835 }857 }
836858
@@ -908,12 +930,19 @@ router.post('/openai/count', async function (req, res) {
908 }930 }
909931
910 if (model === 'command-r') {932 if (model === 'command-r') {
911 const instance = await commandTokenizer.get();933 const instance = await commandRTokenizer.get();
912 if (!instance) throw new Error('Failed to load the Command-R tokenizer');934 if (!instance) throw new Error('Failed to load the Command-R tokenizer');
913 num_tokens = countWebTokenizerTokens(instance, req.body);935 num_tokens = countWebTokenizerTokens(instance, req.body);
914 return res.send({ 'token_count': num_tokens });936 return res.send({ 'token_count': num_tokens });
915 }937 }
916938
939 if (model === 'command-a') {
940 const instance = await commandATokenizer.get();
941 if (!instance) throw new Error('Failed to load the Command-A tokenizer');
942 num_tokens = countWebTokenizerTokens(instance, req.body);
943 return res.send({ 'token_count': num_tokens });
944 }
945
917 if (model === 'nemo') {946 if (model === 'nemo') {
918 const instance = await nemoTokenizer.get();947 const instance = await nemoTokenizer.get();
919 if (!instance) throw new Error('Failed to load the Nemo tokenizer');948 if (!instance) throw new Error('Failed to load the Nemo tokenizer');