Merge pull request #3245 from Succubyss/revert-manual_selection Reverts MANUAL_SELECTION from tokenizers.js

170378e6152fff7d9d578735668add1f04ef5667

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

public/scripts/tokenizers.js+1 -10 Viewed

     NEMO: 17,
     DEEPSEEK: 18,
     BEST_MATCH: 99,
-    MANUAL_SELECTION: 411,
 };
 // A list of local tokenizers that support encoding and decoding token ids.

  * Encodes a string to tokens using the server API.
  * @param {number} tokenizerType Tokenizer type.
  * @param {string} str String to tokenize.
- * @param {string} overrideModel Tokenizer for {tokenizers.MANUAL_SELECTION}.
  * @returns {number[]} Array of token ids.
  */
-export function getTextTokens(tokenizerType, str, overrideModel = undefined) {
+export function getTextTokens(tokenizerType, str) {
-    if (overrideModel && tokenizerType !== tokenizers.MANUAL_SELECTION) {
-        console.warn('overrideModel must be undefined unless using tokenizers.MANUAL_SELECTION', tokenizerType);
-        return [];
-    }
     switch (tokenizerType) {
         case tokenizers.API_CURRENT:
             return getTextTokens(currentRemoteTokenizerAPI(), str);

@@ -1101,9 +1095,6 @@ export function getTextTokens(tokenizerType, str, overrideModel = undefined) {
1101	console.warn('This tokenizer type does not support encoding', tokenizerType);	1095	console.warn('This tokenizer type does not support encoding', tokenizerType);
1102	return [];	1096	return [];
1103	}	1097	}
1104	if (tokenizerType === tokenizers.MANUAL_SELECTION) {
1105	endpointUrl += `?model=${overrideModel}`;
1106	}
1107	if (tokenizerType === tokenizers.OPENAI) {	1098	if (tokenizerType === tokenizers.OPENAI) {
1108	endpointUrl += `?model=${getTokenizerModel()}`;	1099	endpointUrl += `?model=${getTokenizerModel()}`;
1109	}	1100	}