Fix Qwen and Command tokenizers not used for logit bias
| @@ -15,7 +15,7 @@ import { BIAS_CACHE, createNewLogitBiasEntry, displayLogitBias, getLogitBiasList | ||
| 15 | 15 | import { power_user, registerDebugFunction } from './power-user.js'; |
| 16 | 16 | import { getEventSourceStream } from './sse-stream.js'; |
| 17 | 17 | import { getCurrentDreamGenModelTokenizer, getCurrentOpenRouterModelTokenizer } from './textgen-models.js'; |
| 18 | 18 | import { SENTENCEPIECE_TOKENIZERSENCODE_TOKENIZERS, TEXTGEN_TOKENIZERS, getTextTokens, tokenizers } from './tokenizers.js'; |
| 19 | 19 | import { getSortableDelay, onlyUnique } from './utils.js'; |
| 20 | 20 | |
| 21 | 21 | export { |
| @@ -353,7 +353,7 @@ function getTokenizerForTokenIds() { | ||
| 353 | 353 | return tokenizers.API_CURRENT; |
| 354 | 354 | } |
| 355 | 355 | |
| 356 | 356 | if (SENTENCEPIECE_TOKENIZERSENCODE_TOKENIZERS.includes(power_user.tokenizer)) { |
| 357 | 357 | return power_user.tokenizer; |
| 358 | 358 | } |
| 359 | 359 | |
| @@ -33,18 +33,22 @@ export const tokenizers = { | ||
| 33 | 33 | BEST_MATCH: 99, |
| 34 | 34 | }; |
| 35 | 35 | |
| 36 | -export const SENTENCEPIECE_TOKENIZERS = [ | |
| 36 | +// A list of local tokenizers that support encoding and decoding token ids. | |
| 37 | +export const ENCODE_TOKENIZERS = [ | |
| 37 | 38 | tokenizers.LLAMA, |
| 38 | 39 | tokenizers.MISTRAL, |
| 39 | 40 | tokenizers.YI, |
| 40 | 41 | tokenizers.LLAMA3, |
| 41 | 42 | tokenizers.GEMMA, |
| 42 | 43 | tokenizers.JAMBA, |
| 44 | + tokenizers.QWEN2, | |
| 45 | + tokenizers.COMMAND_R, | |
| 43 | 46 | // uncomment when NovelAI releases Kayra and Clio weights, lol |
| 44 | 47 | //tokenizers.NERD, |
| 45 | 48 | //tokenizers.NERD2, |
| 46 | 49 | ]; |
| 47 | 50 | |
| 51 | +// A list of Text Completion sources that support remote tokenization. | |
| 48 | 52 | export const TEXTGEN_TOKENIZERS = [OOBA, TABBY, KOBOLDCPP, LLAMACPP, VLLM, APHRODITE]; |
| 49 | 53 | |
| 50 | 54 | const TOKENIZER_URLS = { |