Add Mistral Nemo downloadable tokenizer

a782dcec336cabe2695b3661ee98cb945aacb55a

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: 6d731bbefe171fdb73264aae6c20cbffc83b214a
3 files changed, +37 -1Ignore whitespace
public/index.html+2 -1
@@ -3339,7 +3339,8 @@
33393339 <option value="16">Command-R</option>
33403340 <option value="4">NerdStash (NovelAI Clio)</option>
33413341 <option value="5">NerdStash v2 (NovelAI Kayra)</option>
33423342 <option value="7">Mistral V1</option>
3343+ <option value="17">Mistral Nemo</option>
33433344 <option value="8">Yi</option>
33443345 <option value="11">Claude 1/2</option>
33453346 <option value="6">API (WebUI / koboldcpp)</option>
public/scripts/tokenizers.js+11 -0
@@ -30,6 +30,7 @@ export const tokenizers = {
3030 JAMBA: 14,
3131 QWEN2: 15,
3232 COMMAND_R: 16,
33+ NEMO: 17,
3334 BEST_MATCH: 99,
3435};
3536
@@ -43,6 +44,7 @@ export const ENCODE_TOKENIZERS = [
4344 tokenizers.JAMBA,
4445 tokenizers.QWEN2,
4546 tokenizers.COMMAND_R,
47+ tokenizers.NEMO,
4648 // uncomment when NovelAI releases Kayra and Clio weights, lol
4749 //tokenizers.NERD,
4850 //tokenizers.NERD2,
@@ -121,6 +123,11 @@ const TOKENIZER_URLS = {
121123 decode: '/api/tokenizers/command-r/decode',
122124 count: '/api/tokenizers/command-r/encode',
123125 },
126+ [tokenizers.NEMO]: {
127+ encode: '/api/tokenizers/nemo/encode',
128+ decode: '/api/tokenizers/nemo/decode',
129+ count: '/api/tokenizers/nemo/encode',
130+ },
124131 [tokenizers.API_TEXTGENERATIONWEBUI]: {
125132 encode: '/api/tokenizers/remote/textgenerationwebui/encode',
126133 count: '/api/tokenizers/remote/textgenerationwebui/encode',
@@ -535,6 +542,7 @@ export function getTokenizerModel() {
535542 const jambaTokenizer = 'jamba';
536543 const qwen2Tokenizer = 'qwen2';
537544 const commandRTokenizer = 'command-r';
545+ const nemoTokenizer = 'nemo';
538546
539547 // Assuming no one would use it for different models.. right?
540548 if (oai_settings.chat_completion_source == chat_completion_sources.SCALE) {
@@ -628,6 +636,9 @@ export function getTokenizerModel() {
628636 }
629637
630638 if (oai_settings.chat_completion_source == chat_completion_sources.MISTRALAI) {
639+ if (oai_settings.mistralai_model.includes('nemo') || oai_settings.mistralai_model.includes('pixtral')) {
640+ return nemoTokenizer;
641+ }
631642 return mistralTokenizer;
632643 }
633644
src/endpoints/tokenizers.js+24 -0
@@ -221,6 +221,7 @@ const claude_tokenizer = new WebTokenizer('src/tokenizers/claude.json');
221221const llama3_tokenizer = new WebTokenizer('src/tokenizers/llama3.json');
222222const commandTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/command-r.json', 'src/tokenizers/llama3.json');
223223const qwen2Tokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/qwen2.json', 'src/tokenizers/llama3.json');
224+const nemoTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/nemo.json', 'src/tokenizers/llama3.json');
224225
225226const sentencepieceTokenizers = [
226227 'llama',
@@ -418,6 +419,10 @@ function getTokenizerModel(requestModel) {
418419 return 'command-r';
419420 }
420421
422+ if (requestModel.includes('nemo')) {
423+ return 'nemo';
424+ }
425+
421426 // default
422427 return 'gpt-3.5-turbo';
423428}
@@ -645,6 +650,7 @@ router.post('/claude/encode', jsonParser, createWebTokenizerEncodingHandler(clau
645650router.post('/llama3/encode', jsonParser, createWebTokenizerEncodingHandler(llama3_tokenizer));
646651router.post('/qwen2/encode', jsonParser, createWebTokenizerEncodingHandler(qwen2Tokenizer));
647652router.post('/command-r/encode', jsonParser, createWebTokenizerEncodingHandler(commandTokenizer));
653+router.post('/nemo/encode', jsonParser, createWebTokenizerEncodingHandler(nemoTokenizer));
648654router.post('/llama/decode', jsonParser, createSentencepieceDecodingHandler(spp_llama));
649655router.post('/nerdstash/decode', jsonParser, createSentencepieceDecodingHandler(spp_nerd));
650656router.post('/nerdstash_v2/decode', jsonParser, createSentencepieceDecodingHandler(spp_nerd_v2));
@@ -657,6 +663,7 @@ router.post('/claude/decode', jsonParser, createWebTokenizerDecodingHandler(clau
657663router.post('/llama3/decode', jsonParser, createWebTokenizerDecodingHandler(llama3_tokenizer));
658664router.post('/qwen2/decode', jsonParser, createWebTokenizerDecodingHandler(qwen2Tokenizer));
659665router.post('/command-r/decode', jsonParser, createWebTokenizerDecodingHandler(commandTokenizer));
666+router.post('/nemo/decode', jsonParser, createWebTokenizerDecodingHandler(nemoTokenizer));
660667
661668router.post('/openai/encode', jsonParser, async function (req, res) {
662669 try {
@@ -707,6 +714,11 @@ router.post('/openai/encode', jsonParser, async function (req, res) {
707714 return handler(req, res);
708715 }
709716
717+ if (queryModel.includes('nemo')) {
718+ const handler = createWebTokenizerEncodingHandler(nemoTokenizer);
719+ return handler(req, res);
720+ }
721+
710722 const model = getTokenizerModel(queryModel);
711723 const handler = createTiktokenEncodingHandler(model);
712724 return handler(req, res);
@@ -765,6 +777,11 @@ router.post('/openai/decode', jsonParser, async function (req, res) {
765777 return handler(req, res);
766778 }
767779
780+ if (queryModel.includes('nemo')) {
781+ const handler = createWebTokenizerDecodingHandler(nemoTokenizer);
782+ return handler(req, res);
783+ }
784+
768785 const model = getTokenizerModel(queryModel);
769786 const handler = createTiktokenDecodingHandler(model);
770787 return handler(req, res);
@@ -835,6 +852,13 @@ router.post('/openai/count', jsonParser, async function (req, res) {
835852 return res.send({ 'token_count': num_tokens });
836853 }
837854
855+ if (model === 'nemo') {
856+ const instance = await nemoTokenizer.get();
857+ if (!instance) throw new Error('Failed to load the Nemo tokenizer');
858+ num_tokens = countWebTokenizerTokens(instance, req.body);
859+ return res.send({ 'token_count': num_tokens });
860+ }
861+
838862 const tokensPerName = queryModel.includes('gpt-3.5-turbo-0301') ? -1 : 1;
839863 const tokensPerMessage = queryModel.includes('gpt-3.5-turbo-0301') ? 4 : 3;
840864 const tokensPadding = 3;