Add best match tokenizer logic for DeepSeek and Nemo

40b05bb6b15d91743838acbfc6ac4fd8bdac0229

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: 2d4bae17ccc71a9515e32d9b8c4ff259cfb14d4f
1 files changed, +6 -0Showing whitespace changes
public/scripts/tokenizers.js+6 -0
@@ -325,6 +325,12 @@ export function getTokenizerBestMatch(forApi) {
325 if (model.includes('gemma')) {325 if (model.includes('gemma')) {
326 return tokenizers.GEMMA;326 return tokenizers.GEMMA;
327 }327 }
328 if (model.includes('nemo') || model.includes('pixtral')) {
329 return tokenizers.NEMO;
330 }
331 if (model.includes('deepseek')) {
332 return tokenizers.DEEPSEEK;
333 }
328 if (model.includes('yi')) {334 if (model.includes('yi')) {
329 return tokenizers.YI;335 return tokenizers.YI;
330 }336 }