Merge pull request #3245 from Succubyss/revert-manual_selection Reverts MANUAL_SELECTION from tokenizers.js

170378e6152fff7d9d578735668add1f04ef5667

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: 56da6eb39d4b2567ea93b97febbd225220db6cf1 97dc0417773b6ff6a4f7fad6e5bec04c0f44d723
1 files changed, +1 -10Showing whitespace changes
public/scripts/tokenizers.js+1 -10
@@ -33,7 +33,6 @@ export const tokenizers = {
33 NEMO: 17,33 NEMO: 17,
34 DEEPSEEK: 18,34 DEEPSEEK: 18,
35 BEST_MATCH: 99,35 BEST_MATCH: 99,
36 MANUAL_SELECTION: 411,
37};36};
3837
39// A list of local tokenizers that support encoding and decoding token ids.38// A list of local tokenizers that support encoding and decoding token ids.
@@ -1073,14 +1072,9 @@ function decodeTextTokensFromServer(endpoint, ids, resolve) {
1073 * Encodes a string to tokens using the server API.1072 * Encodes a string to tokens using the server API.
1074 * @param {number} tokenizerType Tokenizer type.1073 * @param {number} tokenizerType Tokenizer type.
1075 * @param {string} str String to tokenize.1074 * @param {string} str String to tokenize.
1076 * @param {string} overrideModel Tokenizer for {tokenizers.MANUAL_SELECTION}.
1077 * @returns {number[]} Array of token ids.1075 * @returns {number[]} Array of token ids.
1078 */1076 */
1079export function getTextTokens(tokenizerType, str, overrideModel = undefined) {1077export function getTextTokens(tokenizerType, str) {
1080 if (overrideModel && tokenizerType !== tokenizers.MANUAL_SELECTION) {
1081 console.warn('overrideModel must be undefined unless using tokenizers.MANUAL_SELECTION', tokenizerType);
1082 return [];
1083 }
1084 switch (tokenizerType) {1078 switch (tokenizerType) {
1085 case tokenizers.API_CURRENT:1079 case tokenizers.API_CURRENT:
1086 return getTextTokens(currentRemoteTokenizerAPI(), str);1080 return getTextTokens(currentRemoteTokenizerAPI(), str);
@@ -1101,9 +1095,6 @@ export function getTextTokens(tokenizerType, str, overrideModel = undefined) {
1101 console.warn('This tokenizer type does not support encoding', tokenizerType);1095 console.warn('This tokenizer type does not support encoding', tokenizerType);
1102 return [];1096 return [];
1103 }1097 }
1104 if (tokenizerType === tokenizers.MANUAL_SELECTION) {
1105 endpointUrl += `?model=${overrideModel}`;
1106 }
1107 if (tokenizerType === tokenizers.OPENAI) {1098 if (tokenizerType === tokenizers.OPENAI) {
1108 endpointUrl += `?model=${getTokenizerModel()}`;1099 endpointUrl += `?model=${getTokenizerModel()}`;
1109 }1100 }