Merge pull request #2873 from SillyTavern/nemo-tokenizer Add Mistral Nemo downloadable tokenizer

83c3f6d1bfcb240eaeda4c0f109160352de91175

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

public/index.html+2 -1 Viewed

                                     <option value="16">Command-R</option>
                                     <option value="4">NerdStash (NovelAI Clio)</option>
                                     <option value="5">NerdStash v2 (NovelAI Kayra)</option>
-                                    <option value="7">Mistral</option>
+                                    <option value="7">Mistral V1</option>
+                                    <option value="17">Mistral Nemo</option>
                                     <option value="8">Yi</option>
                                     <option value="11">Claude 1/2</option>
                                     <option value="6">API (WebUI / koboldcpp)</option>

public/scripts/tokenizers.js+11 -0 Viewed

     JAMBA: 14,
     QWEN2: 15,
     COMMAND_R: 16,
+    NEMO: 17,
     BEST_MATCH: 99,
 };

     tokenizers.JAMBA,
     tokenizers.QWEN2,
     tokenizers.COMMAND_R,
+    tokenizers.NEMO,
     // uncomment when NovelAI releases Kayra and Clio weights, lol
     //tokenizers.NERD,
     //tokenizers.NERD2,

         decode: '/api/tokenizers/command-r/decode',
         count: '/api/tokenizers/command-r/encode',
     },
+    [tokenizers.NEMO]: {
+        encode: '/api/tokenizers/nemo/encode',
+        decode: '/api/tokenizers/nemo/decode',
+        count: '/api/tokenizers/nemo/encode',
+    },
     [tokenizers.API_TEXTGENERATIONWEBUI]: {
         encode: '/api/tokenizers/remote/textgenerationwebui/encode',
         count: '/api/tokenizers/remote/textgenerationwebui/encode',

     const jambaTokenizer = 'jamba';
     const qwen2Tokenizer = 'qwen2';
     const commandRTokenizer = 'command-r';
+    const nemoTokenizer = 'nemo';
     // Assuming no one would use it for different models.. right?
     if (oai_settings.chat_completion_source == chat_completion_sources.SCALE) {

@@ -628,6 +636,9 @@ export function getTokenizerModel() {
628	}	636	}
629		637
630	if (oai_settings.chat_completion_source == chat_completion_sources.MISTRALAI) {	638	if (oai_settings.chat_completion_source == chat_completion_sources.MISTRALAI) {
		639	if (oai_settings.mistralai_model.includes('nemo') \|\| oai_settings.mistralai_model.includes('pixtral')) {
		640	return nemoTokenizer;
		641	}
631	return mistralTokenizer;	642	return mistralTokenizer;
632	}	643	}
633		644

src/endpoints/tokenizers.js+24 -0 Viewed

 const llama3_tokenizer = new WebTokenizer('src/tokenizers/llama3.json');
 const commandTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/command-r.json', 'src/tokenizers/llama3.json');
 const qwen2Tokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/qwen2.json', 'src/tokenizers/llama3.json');
+const nemoTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/nemo.json', 'src/tokenizers/llama3.json');
 const sentencepieceTokenizers = [
     'llama',

@@ -418,6 +419,10 @@ function getTokenizerModel(requestModel) {
418	return 'command-r';	419	return 'command-r';
419	}	420	}
420		421
		422	if (requestModel.includes('nemo')) {
		423	return 'nemo';
		424	}
		425
421	// default	426	// default
422	return 'gpt-3.5-turbo';	427	return 'gpt-3.5-turbo';
423	}	428	}

 router.post('/llama3/encode', jsonParser, createWebTokenizerEncodingHandler(llama3_tokenizer));
 router.post('/qwen2/encode', jsonParser, createWebTokenizerEncodingHandler(qwen2Tokenizer));
 router.post('/command-r/encode', jsonParser, createWebTokenizerEncodingHandler(commandTokenizer));
+router.post('/nemo/encode', jsonParser, createWebTokenizerEncodingHandler(nemoTokenizer));
 router.post('/llama/decode', jsonParser, createSentencepieceDecodingHandler(spp_llama));
 router.post('/nerdstash/decode', jsonParser, createSentencepieceDecodingHandler(spp_nerd));
 router.post('/nerdstash_v2/decode', jsonParser, createSentencepieceDecodingHandler(spp_nerd_v2));

 router.post('/llama3/decode', jsonParser, createWebTokenizerDecodingHandler(llama3_tokenizer));
 router.post('/qwen2/decode', jsonParser, createWebTokenizerDecodingHandler(qwen2Tokenizer));
 router.post('/command-r/decode', jsonParser, createWebTokenizerDecodingHandler(commandTokenizer));
+router.post('/nemo/decode', jsonParser, createWebTokenizerDecodingHandler(nemoTokenizer));
 router.post('/openai/encode', jsonParser, async function (req, res) {
     try {

             return handler(req, res);
+        }
+        if (queryModel.includes('nemo')) {
+            const handler = createWebTokenizerEncodingHandler(nemoTokenizer);
+            return handler(req, res);
+        }
         const model = getTokenizerModel(queryModel);
         const handler = createTiktokenEncodingHandler(model);
         return handler(req, res);

             return handler(req, res);
+        }
+        if (queryModel.includes('nemo')) {
+            const handler = createWebTokenizerDecodingHandler(nemoTokenizer);
+            return handler(req, res);
+        }
         const model = getTokenizerModel(queryModel);
         const handler = createTiktokenDecodingHandler(model);
         return handler(req, res);

             return res.send({ 'token_count': num_tokens });
+        }
+        if (model === 'nemo') {
+            const instance = await nemoTokenizer.get();
+            if (!instance) throw new Error('Failed to load the Nemo tokenizer');
+            num_tokens = countWebTokenizerTokens(instance, req.body);
+            return res.send({ 'token_count': num_tokens });
+        }
         const tokensPerName = queryModel.includes('gpt-3.5-turbo-0301') ? -1 : 1;
         const tokensPerMessage = queryModel.includes('gpt-3.5-turbo-0301') ? 4 : 3;
         const tokensPadding = 3;