Fix Qwen and Command tokenizers not used for logit bias

0207794a2b1ae02f71a84dad0e496d712b559c8b

Cohee <18619528+Cohee1207@users.noreply.github.com>

public/scripts/textgen-settings.js+2 -2 Viewed

 import { power_user, registerDebugFunction } from './power-user.js';
 import { getEventSourceStream } from './sse-stream.js';
 import { getCurrentDreamGenModelTokenizer, getCurrentOpenRouterModelTokenizer } from './textgen-models.js';
-import { SENTENCEPIECE_TOKENIZERS, TEXTGEN_TOKENIZERS, getTextTokens, tokenizers } from './tokenizers.js';
+import { ENCODE_TOKENIZERS, TEXTGEN_TOKENIZERS, getTextTokens, tokenizers } from './tokenizers.js';
 import { getSortableDelay, onlyUnique } from './utils.js';
 export {

@@ -353,7 +353,7 @@ function getTokenizerForTokenIds() {
353	return tokenizers.API_CURRENT;	353	return tokenizers.API_CURRENT;
354	}	354	}
355		355
356	if (SENTENCEPIECE_TOKENIZERS.includes(power_user.tokenizer)) {	356	if (ENCODE_TOKENIZERS.includes(power_user.tokenizer)) {
357	return power_user.tokenizer;	357	return power_user.tokenizer;
358	}	358	}
359		359

public/scripts/tokenizers.js+5 -1 Viewed

     BEST_MATCH: 99,
 };
-export const SENTENCEPIECE_TOKENIZERS = [
+// A list of local tokenizers that support encoding and decoding token ids.
+export const ENCODE_TOKENIZERS = [
     tokenizers.LLAMA,
     tokenizers.MISTRAL,
     tokenizers.YI,
     tokenizers.LLAMA3,
     tokenizers.GEMMA,
     tokenizers.JAMBA,
+    tokenizers.QWEN2,
+    tokenizers.COMMAND_R,
     // uncomment when NovelAI releases Kayra and Clio weights, lol
     //tokenizers.NERD,
     //tokenizers.NERD2,
 ];
+// A list of Text Completion sources that support remote tokenization.
 export const TEXTGEN_TOKENIZERS = [OOBA, TABBY, KOBOLDCPP, LLAMACPP, VLLM, APHRODITE];
 const TOKENIZER_URLS = {