Add best match tokenizer logic for DeepSeek and Nemo
| @@ -325,6 +325,12 @@ export function getTokenizerBestMatch(forApi) { | ||
| 325 | 325 | if (model.includes('gemma')) { |
| 326 | 326 | return tokenizers.GEMMA; |
| 327 | 327 | } |
| 328 | + if (model.includes('nemo') || model.includes('pixtral')) { | |
| 329 | + return tokenizers.NEMO; | |
| 330 | + } | |
| 331 | + if (model.includes('deepseek')) { | |
| 332 | + return tokenizers.DEEPSEEK; | |
| 333 | + } | |
| 328 | 334 | if (model.includes('yi')) { |
| 329 | 335 | return tokenizers.YI; |
| 330 | 336 | } |