Added command-a-03-2025 and command-a tokenizer

a77f4045f818677e6aa1077aefb7a8f046a4d207

bmen25124 <bmen25124@gmail.com>

Parents: 79a8080b7d11f5d905590f659f268b5de6c79051
4 files changed, +57 -7Ignore whitespace
public/index.html+1 -0
@@ -3333,6 +3333,7 @@
33333333 <option value="command-r-08-2024">command-r-08-2024</option>
33343334 <option value="command-r-plus-08-2024">command-r-plus-08-2024</option>
33353335 <option value="command-r7b-12-2024">command-r7b-12-2024</option>
3336+ <option value="command-a-03-2025">command-a-03-2025</option>
33363337 </optgroup>
33373338 <optgroup label="Nightly">
33383339 <option value="command-light-nightly">command-light-nightly</option>
public/scripts/openai.js+3 -0
@@ -4448,6 +4448,9 @@ async function onModelChange() {
44484448 else if (oai_settings.cohere_model.includes('command-r') || ['c4ai-aya-23', 'c4ai-aya-expanse-32b', 'command-nightly'].includes(oai_settings.cohere_model)) {
44494449 $('#openai_max_context').attr('max', max_128k);
44504450 }
4451+ else if (['command-a-03-2025'].includes(oai_settings.cohere_model)) {
4452+ $('#openai_max_context').attr('max', max_256k);
4453+ }
44514454 else if (['c4ai-aya-23-8b', 'c4ai-aya-expanse-8b'].includes(oai_settings.cohere_model)) {
44524455 $('#openai_max_context').attr('max', max_8k);
44534456 }
public/scripts/tokenizers.js+17 -0
@@ -32,6 +32,7 @@ export const tokenizers = {
3232 COMMAND_R: 16,
3333 NEMO: 17,
3434 DEEPSEEK: 18,
35+ COMMAND_A: 19,
3536 BEST_MATCH: 99,
3637};
3738
@@ -45,6 +46,7 @@ export const ENCODE_TOKENIZERS = [
4546 tokenizers.JAMBA,
4647 tokenizers.QWEN2,
4748 tokenizers.COMMAND_R,
49+ tokenizers.COMMAND_A,
4850 tokenizers.NEMO,
4951 tokenizers.DEEPSEEK,
5052 // uncomment when NovelAI releases Kayra and Clio weights, lol
@@ -129,6 +131,11 @@ const TOKENIZER_URLS = {
129131 decode: '/api/tokenizers/command-r/decode',
130132 count: '/api/tokenizers/command-r/encode',
131133 },
134+ [tokenizers.COMMAND_A]: {
135+ encode: '/api/tokenizers/command-a/encode',
136+ decode: '/api/tokenizers/command-a/decode',
137+ count: '/api/tokenizers/command-a/encode',
138+ },
132139 [tokenizers.NEMO]: {
133140 encode: '/api/tokenizers/nemo/encode',
134141 decode: '/api/tokenizers/nemo/decode',
@@ -340,6 +347,9 @@ export function getTokenizerBestMatch(forApi) {
340347 if (model.includes('command-r')) {
341348 return tokenizers.COMMAND_R;
342349 }
350+ if (model.includes('command-a')) {
351+ return tokenizers.COMMAND_A;
352+ }
343353 if (model.includes('qwen2')) {
344354 return tokenizers.QWEN2;
345355 }
@@ -572,6 +582,7 @@ export function getTokenizerModel() {
572582 const jambaTokenizer = 'jamba';
573583 const qwen2Tokenizer = 'qwen2';
574584 const commandRTokenizer = 'command-r';
585+ const commandATokenizer = 'command-a';
575586 const nemoTokenizer = 'nemo';
576587 const deepseekTokenizer = 'deepseek';
577588
@@ -626,6 +637,9 @@ export function getTokenizerModel() {
626637 return qwen2Tokenizer;
627638 }
628639 else if (model?.architecture?.tokenizer === 'Cohere') {
640+ if (oai_settings.openrouter_model.includes('command-a')) {
641+ return commandATokenizer;
642+ }
629643 return commandRTokenizer;
630644 }
631645 else if (oai_settings.openrouter_model.includes('gpt-4o')) {
@@ -652,6 +666,9 @@ export function getTokenizerModel() {
652666 }
653667
654668 if (oai_settings.chat_completion_source == chat_completion_sources.COHERE) {
669+ if (oai_settings.cohere_model.includes('command-a')) {
670+ return commandATokenizer;
671+ }
655672 return commandRTokenizer;
656673 }
657674
src/endpoints/tokenizers.js+36 -7
@@ -222,7 +222,8 @@ const spp_gemma = new SentencePieceTokenizer('src/tokenizers/gemma.model');
222222const spp_jamba = new SentencePieceTokenizer('src/tokenizers/jamba.model');
223223const claude_tokenizer = new WebTokenizer('src/tokenizers/claude.json');
224224const llama3_tokenizer = new WebTokenizer('src/tokenizers/llama3.json');
225225const commandTokenizercommandRTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/command-r.json', 'src/tokenizers/llama3.json');
226+const commandATokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/command-a.json', 'src/tokenizers/llama3.json');
226227const qwen2Tokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/qwen2.json', 'src/tokenizers/llama3.json');
227228const nemoTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/nemo.json', 'src/tokenizers/llama3.json');
228229const deepseekTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/deepseek.json', 'src/tokenizers/llama3.json');
@@ -241,6 +242,7 @@ export const webTokenizers = [
241242 'claude',
242243 'llama3',
243244 'command-r',
245+ 'command-a',
244246 'qwen2',
245247 'nemo',
246248 'deepseek',
@@ -298,7 +300,11 @@ export function getWebTokenizer(model) {
298300 }
299301
300302 if (model.includes('command-r')) {
301303 return commandTokenizercommandRTokenizer;
304+ }
305+
306+ if (model.includes('command-a')) {
307+ return commandATokenizer;
302308 }
303309
304310 if (model.includes('qwen2')) {
@@ -469,6 +475,10 @@ export function getTokenizerModel(requestModel) {
469475 return 'command-r';
470476 }
471477
478+ if (requestModel.includes('command-a')) {
479+ return 'command-a';
480+ }
481+
472482 if (requestModel.includes('nemo')) {
473483 return 'nemo';
474484 }
@@ -699,7 +709,8 @@ router.post('/gpt2/encode', createTiktokenEncodingHandler('gpt2'));
699709router.post('/claude/encode', createWebTokenizerEncodingHandler(claude_tokenizer));
700710router.post('/llama3/encode', createWebTokenizerEncodingHandler(llama3_tokenizer));
701711router.post('/qwen2/encode', createWebTokenizerEncodingHandler(qwen2Tokenizer));
702712router.post('/command-r/encode', createWebTokenizerEncodingHandler(commandTokenizercommandRTokenizer));
713+router.post('/command-a/encode', createWebTokenizerEncodingHandler(commandATokenizer));
703714router.post('/nemo/encode', createWebTokenizerEncodingHandler(nemoTokenizer));
704715router.post('/deepseek/encode', createWebTokenizerEncodingHandler(deepseekTokenizer));
705716router.post('/llama/decode', createSentencepieceDecodingHandler(spp_llama));
@@ -713,7 +724,8 @@ router.post('/gpt2/decode', createTiktokenDecodingHandler('gpt2'));
713724router.post('/claude/decode', createWebTokenizerDecodingHandler(claude_tokenizer));
714725router.post('/llama3/decode', createWebTokenizerDecodingHandler(llama3_tokenizer));
715726router.post('/qwen2/decode', createWebTokenizerDecodingHandler(qwen2Tokenizer));
716727router.post('/command-r/decode', createWebTokenizerDecodingHandler(commandTokenizercommandRTokenizer));
728+router.post('/command-a/decode', createWebTokenizerDecodingHandler(commandATokenizer));
717729router.post('/nemo/decode', createWebTokenizerDecodingHandler(nemoTokenizer));
718730router.post('/deepseek/decode', createWebTokenizerDecodingHandler(deepseekTokenizer));
719731
@@ -762,7 +774,12 @@ router.post('/openai/encode', async function (req, res) {
762774 }
763775
764776 if (queryModel.includes('command-r')) {
765777 const handler = createWebTokenizerEncodingHandler(commandTokenizercommandRTokenizer);
778+ return handler(req, res);
779+ }
780+
781+ if (queryModel.includes('command-a')) {
782+ const handler = createWebTokenizerEncodingHandler(commandATokenizer);
766783 return handler(req, res);
767784 }
768785
@@ -830,7 +847,12 @@ router.post('/openai/decode', async function (req, res) {
830847 }
831848
832849 if (queryModel.includes('command-r')) {
833850 const handler = createWebTokenizerDecodingHandler(commandTokenizercommandRTokenizer);
851+ return handler(req, res);
852+ }
853+
854+ if (queryModel.includes('command-a')) {
855+ const handler = createWebTokenizerDecodingHandler(commandATokenizer);
834856 return handler(req, res);
835857 }
836858
@@ -908,12 +930,19 @@ router.post('/openai/count', async function (req, res) {
908930 }
909931
910932 if (model === 'command-r') {
911933 const instance = await commandTokenizercommandRTokenizer.get();
912934 if (!instance) throw new Error('Failed to load the Command-R tokenizer');
913935 num_tokens = countWebTokenizerTokens(instance, req.body);
914936 return res.send({ 'token_count': num_tokens });
915937 }
916938
939+ if (model === 'command-a') {
940+ const instance = await commandATokenizer.get();
941+ if (!instance) throw new Error('Failed to load the Command-A tokenizer');
942+ num_tokens = countWebTokenizerTokens(instance, req.body);
943+ return res.send({ 'token_count': num_tokens });
944+ }
945+
917946 if (model === 'nemo') {
918947 const instance = await nemoTokenizer.get();
919948 if (!instance) throw new Error('Failed to load the Nemo tokenizer');