Add tokenizer for Gemma/Gemini

e707def7dddb00ed9d72b04df9d7fb07136901d0

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: ce8b0aae960ee00f3b50a85e3b2628e05b2f8148
5 files changed, +52 -3Showing whitespace changes
public/index.html+1 -0
@@ -3449,6 +3449,7 @@
3449 <!-- Option #2 was a legacy GPT-2/3 tokenizer -->3449 <!-- Option #2 was a legacy GPT-2/3 tokenizer -->
3450 <option value="3">Llama 1/2</option>3450 <option value="3">Llama 1/2</option>
3451 <option value="12">Llama 3</option>3451 <option value="12">Llama 3</option>
3452 <option value="13">Gemma / Gemini</option>
3452 <option value="4">NerdStash (NovelAI Clio)</option>3453 <option value="4">NerdStash (NovelAI Clio)</option>
3453 <option value="5">NerdStash v2 (NovelAI Kayra)</option>3454 <option value="5">NerdStash v2 (NovelAI Kayra)</option>
3454 <option value="7">Mistral</option>3455 <option value="7">Mistral</option>
public/scripts/textgen-models.js+4 -0
@@ -599,6 +599,10 @@ export function getCurrentOpenRouterModelTokenizer() {
599 return tokenizers.YI;599 return tokenizers.YI;
600 case 'Mistral':600 case 'Mistral':
601 return tokenizers.MISTRAL;601 return tokenizers.MISTRAL;
602 case 'Gemini':
603 return tokenizers.GEMMA;
604 case 'Claude':
605 return tokenizers.CLAUDE;
602 default:606 default:
603 return tokenizers.OPENAI;607 return tokenizers.OPENAI;
604 }608 }
public/scripts/tokenizers.js+22 -1
@@ -26,6 +26,7 @@ export const tokenizers = {
26 API_KOBOLD: 10,26 API_KOBOLD: 10,
27 CLAUDE: 11,27 CLAUDE: 11,
28 LLAMA3: 12,28 LLAMA3: 12,
29 GEMMA: 13,
29 BEST_MATCH: 99,30 BEST_MATCH: 99,
30};31};
3132
@@ -34,6 +35,7 @@ export const SENTENCEPIECE_TOKENIZERS = [
34 tokenizers.MISTRAL,35 tokenizers.MISTRAL,
35 tokenizers.YI,36 tokenizers.YI,
36 tokenizers.LLAMA3,37 tokenizers.LLAMA3,
38 tokenizers.GEMMA,
37 // uncomment when NovelAI releases Kayra and Clio weights, lol39 // uncomment when NovelAI releases Kayra and Clio weights, lol
38 //tokenizers.NERD,40 //tokenizers.NERD,
39 //tokenizers.NERD2,41 //tokenizers.NERD2,
@@ -91,6 +93,11 @@ const TOKENIZER_URLS = {
91 decode: '/api/tokenizers/llama3/decode',93 decode: '/api/tokenizers/llama3/decode',
92 count: '/api/tokenizers/llama3/encode',94 count: '/api/tokenizers/llama3/encode',
93 },95 },
96 [tokenizers.GEMMA]: {
97 encode: '/api/tokenizers/gemma/encode',
98 decode: '/api/tokenizers/gemma/decode',
99 count: '/api/tokenizers/gemma/encode',
100 },
94 [tokenizers.API_TEXTGENERATIONWEBUI]: {101 [tokenizers.API_TEXTGENERATIONWEBUI]: {
95 encode: '/api/tokenizers/remote/textgenerationwebui/encode',102 encode: '/api/tokenizers/remote/textgenerationwebui/encode',
96 count: '/api/tokenizers/remote/textgenerationwebui/encode',103 count: '/api/tokenizers/remote/textgenerationwebui/encode',
@@ -232,6 +239,9 @@ export function getTokenizerBestMatch(forApi) {
232 if (model.includes('mistral') || model.includes('mixtral')) {239 if (model.includes('mistral') || model.includes('mixtral')) {
233 return tokenizers.MISTRAL;240 return tokenizers.MISTRAL;
234 }241 }
242 if (model.includes('gemma')) {
243 return tokenizers.GEMMA;
244 }
235 }245 }
236246
237 return tokenizers.LLAMA;247 return tokenizers.LLAMA;
@@ -441,12 +451,14 @@ export function getTokenizerModel() {
441 const turbo0301Tokenizer = 'gpt-3.5-turbo-0301';451 const turbo0301Tokenizer = 'gpt-3.5-turbo-0301';
442 const turboTokenizer = 'gpt-3.5-turbo';452 const turboTokenizer = 'gpt-3.5-turbo';
443 const gpt4Tokenizer = 'gpt-4';453 const gpt4Tokenizer = 'gpt-4';
454 const gpt4oTokenizer = 'gpt-4o';
444 const gpt2Tokenizer = 'gpt2';455 const gpt2Tokenizer = 'gpt2';
445 const claudeTokenizer = 'claude';456 const claudeTokenizer = 'claude';
446 const llamaTokenizer = 'llama';457 const llamaTokenizer = 'llama';
447 const llama3Tokenizer = 'llama3';458 const llama3Tokenizer = 'llama3';
448 const mistralTokenizer = 'mistral';459 const mistralTokenizer = 'mistral';
449 const yiTokenizer = 'yi';460 const yiTokenizer = 'yi';
461 const gemmaTokenizer = 'gemma';
450462
451 // Assuming no one would use it for different models.. right?463 // Assuming no one would use it for different models.. right?
452 if (oai_settings.chat_completion_source == chat_completion_sources.SCALE) {464 if (oai_settings.chat_completion_source == chat_completion_sources.SCALE) {
@@ -491,6 +503,12 @@ export function getTokenizerModel() {
491 else if (model?.architecture?.tokenizer === 'Yi') {503 else if (model?.architecture?.tokenizer === 'Yi') {
492 return yiTokenizer;504 return yiTokenizer;
493 }505 }
506 else if (model?.architecture?.tokenizer === 'Gemini') {
507 return gemmaTokenizer;
508 }
509 else if (oai_settings.openrouter_model.includes('gpt-4o')) {
510 return gpt4oTokenizer;
511 }
494 else if (oai_settings.openrouter_model.includes('gpt-4')) {512 else if (oai_settings.openrouter_model.includes('gpt-4')) {
495 return gpt4Tokenizer;513 return gpt4Tokenizer;
496 }514 }
@@ -509,7 +527,7 @@ export function getTokenizerModel() {
509 }527 }
510528
511 if (oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) {529 if (oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) {
512 return oai_settings.google_model;530 return gemmaTokenizer;
513 }531 }
514532
515 if (oai_settings.chat_completion_source == chat_completion_sources.CLAUDE) {533 if (oai_settings.chat_completion_source == chat_completion_sources.CLAUDE) {
@@ -543,6 +561,9 @@ export function getTokenizerModel() {
543 if (oai_settings.groq_model.includes('mistral') || oai_settings.groq_model.includes('mixtral')) {561 if (oai_settings.groq_model.includes('mistral') || oai_settings.groq_model.includes('mixtral')) {
544 return mistralTokenizer;562 return mistralTokenizer;
545 }563 }
564 if (oai_settings.groq_model.includes('gemma')) {
565 return gemmaTokenizer;
566 }
546 }567 }
547568
548 if (oai_settings.chat_completion_source === chat_completion_sources.ZEROONEAI) {569 if (oai_settings.chat_completion_source === chat_completion_sources.ZEROONEAI) {
src/endpoints/tokenizers.js+25 -2
@@ -143,6 +143,7 @@ const spp_nerd = new SentencePieceTokenizer('src/tokenizers/nerdstash.model');
143const spp_nerd_v2 = new SentencePieceTokenizer('src/tokenizers/nerdstash_v2.model');143const spp_nerd_v2 = new SentencePieceTokenizer('src/tokenizers/nerdstash_v2.model');
144const spp_mistral = new SentencePieceTokenizer('src/tokenizers/mistral.model');144const spp_mistral = new SentencePieceTokenizer('src/tokenizers/mistral.model');
145const spp_yi = new SentencePieceTokenizer('src/tokenizers/yi.model');145const spp_yi = new SentencePieceTokenizer('src/tokenizers/yi.model');
146const spp_gemma = new SentencePieceTokenizer('src/tokenizers/gemma.model');
146const claude_tokenizer = new WebTokenizer('src/tokenizers/claude.json');147const claude_tokenizer = new WebTokenizer('src/tokenizers/claude.json');
147const llama3_tokenizer = new WebTokenizer('src/tokenizers/llama3.json');148const llama3_tokenizer = new WebTokenizer('src/tokenizers/llama3.json');
148149
@@ -152,6 +153,7 @@ const sentencepieceTokenizers = [
152 'nerdstash_v2',153 'nerdstash_v2',
153 'mistral',154 'mistral',
154 'yi',155 'yi',
156 'gemma',
155];157];
156158
157/**159/**
@@ -180,6 +182,10 @@ function getSentencepiceTokenizer(model) {
180 return spp_yi;182 return spp_yi;
181 }183 }
182184
185 if (model.includes('gemma')) {
186 return spp_gemma;
187 }
188
183 return null;189 return null;
184}190}
185191
@@ -312,8 +318,8 @@ function getTokenizerModel(requestModel) {
312 return 'yi';318 return 'yi';
313 }319 }
314320
315 if (requestModel.includes('gemini')) {321 if (requestModel.includes('gemma') || requestModel.includes('gemini')) {
316 return 'gpt-4o';322 return 'gemma';
317 }323 }
318324
319 // default325 // default
@@ -583,6 +589,7 @@ router.post('/nerdstash/encode', jsonParser, createSentencepieceEncodingHandler(
583router.post('/nerdstash_v2/encode', jsonParser, createSentencepieceEncodingHandler(spp_nerd_v2));589router.post('/nerdstash_v2/encode', jsonParser, createSentencepieceEncodingHandler(spp_nerd_v2));
584router.post('/mistral/encode', jsonParser, createSentencepieceEncodingHandler(spp_mistral));590router.post('/mistral/encode', jsonParser, createSentencepieceEncodingHandler(spp_mistral));
585router.post('/yi/encode', jsonParser, createSentencepieceEncodingHandler(spp_yi));591router.post('/yi/encode', jsonParser, createSentencepieceEncodingHandler(spp_yi));
592router.post('/gemma/encode', jsonParser, createSentencepieceEncodingHandler(spp_gemma));
586router.post('/gpt2/encode', jsonParser, createTiktokenEncodingHandler('gpt2'));593router.post('/gpt2/encode', jsonParser, createTiktokenEncodingHandler('gpt2'));
587router.post('/claude/encode', jsonParser, createWebTokenizerEncodingHandler(claude_tokenizer));594router.post('/claude/encode', jsonParser, createWebTokenizerEncodingHandler(claude_tokenizer));
588router.post('/llama3/encode', jsonParser, createWebTokenizerEncodingHandler(llama3_tokenizer));595router.post('/llama3/encode', jsonParser, createWebTokenizerEncodingHandler(llama3_tokenizer));
@@ -591,6 +598,7 @@ router.post('/nerdstash/decode', jsonParser, createSentencepieceDecodingHandler(
591router.post('/nerdstash_v2/decode', jsonParser, createSentencepieceDecodingHandler(spp_nerd_v2));598router.post('/nerdstash_v2/decode', jsonParser, createSentencepieceDecodingHandler(spp_nerd_v2));
592router.post('/mistral/decode', jsonParser, createSentencepieceDecodingHandler(spp_mistral));599router.post('/mistral/decode', jsonParser, createSentencepieceDecodingHandler(spp_mistral));
593router.post('/yi/decode', jsonParser, createSentencepieceDecodingHandler(spp_yi));600router.post('/yi/decode', jsonParser, createSentencepieceDecodingHandler(spp_yi));
601router.post('/gemma/decode', jsonParser, createSentencepieceDecodingHandler(spp_gemma));
594router.post('/gpt2/decode', jsonParser, createTiktokenDecodingHandler('gpt2'));602router.post('/gpt2/decode', jsonParser, createTiktokenDecodingHandler('gpt2'));
595router.post('/claude/decode', jsonParser, createWebTokenizerDecodingHandler(claude_tokenizer));603router.post('/claude/decode', jsonParser, createWebTokenizerDecodingHandler(claude_tokenizer));
596router.post('/llama3/decode', jsonParser, createWebTokenizerDecodingHandler(llama3_tokenizer));604router.post('/llama3/decode', jsonParser, createWebTokenizerDecodingHandler(llama3_tokenizer));
@@ -624,6 +632,11 @@ router.post('/openai/encode', jsonParser, async function (req, res) {
624 return handler(req, res);632 return handler(req, res);
625 }633 }
626634
635 if (queryModel.includes('gemma') || queryModel.includes('gemini')) {
636 const handler = createSentencepieceEncodingHandler(spp_gemma);
637 return handler(req, res);
638 }
639
627 const model = getTokenizerModel(queryModel);640 const model = getTokenizerModel(queryModel);
628 const handler = createTiktokenEncodingHandler(model);641 const handler = createTiktokenEncodingHandler(model);
629 return handler(req, res);642 return handler(req, res);
@@ -662,6 +675,11 @@ router.post('/openai/decode', jsonParser, async function (req, res) {
662 return handler(req, res);675 return handler(req, res);
663 }676 }
664677
678 if (queryModel.includes('gemma') || queryModel.includes('gemini')) {
679 const handler = createSentencepieceDecodingHandler(spp_gemma);
680 return handler(req, res);
681 }
682
665 const model = getTokenizerModel(queryModel);683 const model = getTokenizerModel(queryModel);
666 const handler = createTiktokenDecodingHandler(model);684 const handler = createTiktokenDecodingHandler(model);
667 return handler(req, res);685 return handler(req, res);
@@ -708,6 +726,11 @@ router.post('/openai/count', jsonParser, async function (req, res) {
708 return res.send({ 'token_count': num_tokens });726 return res.send({ 'token_count': num_tokens });
709 }727 }
710728
729 if (model === 'gemma' || model === 'gemini') {
730 num_tokens = await countSentencepieceArrayTokens(spp_gemma, req.body);
731 return res.send({ 'token_count': num_tokens });
732 }
733
711 const tokensPerName = queryModel.includes('gpt-3.5-turbo-0301') ? -1 : 1;734 const tokensPerName = queryModel.includes('gpt-3.5-turbo-0301') ? -1 : 1;
712 const tokensPerMessage = queryModel.includes('gpt-3.5-turbo-0301') ? 4 : 3;735 const tokensPerMessage = queryModel.includes('gpt-3.5-turbo-0301') ? 4 : 3;
713 const tokensPadding = 3;736 const tokensPadding = 3;
src/tokenizers/gemma.model+0 -0

Binary file