Revert MANUAL_SELECTION This partially reverts commit c89804677a461f9854403e8ec8826e273f2bf18c (keeps the turbo-0301 removal).

97dc0417773b6ff6a4f7fad6e5bec04c0f44d723

Succubyss <87207237+Succubyss@users.noreply.github.com>

public/scripts/tokenizers.js+1 -10 Viewed

     NEMO: 17,
     DEEPSEEK: 18,
     BEST_MATCH: 99,
 -    MANUAL_SELECTION: 411,
 };
 // A list of local tokenizers that support encoding and decoding token ids.

  * Encodes a string to tokens using the server API.
  * @param {number} tokenizerType Tokenizer type.
  * @param {string} str String to tokenize.
 - * @param {string} overrideModel Tokenizer for {tokenizers.MANUAL_SELECTION}.
  * @returns {number[]} Array of token ids.
  */
 export function getTextTokens(tokenizerType, str, overrideModel = undefined) {
 -    if (overrideModel && tokenizerType !== tokenizers.MANUAL_SELECTION) {
 -        console.warn('overrideModel must be undefined unless using tokenizers.MANUAL_SELECTION', tokenizerType);
 -        return [];
 -    }
     switch (tokenizerType) {
         case tokenizers.API_CURRENT:
             return getTextTokens(currentRemoteTokenizerAPI(), str);

@@ -1101,9 +1095,6 @@ export function getTextTokens(tokenizerType, str, overrideModel = undefined) {
1101	1095	console.warn('This tokenizer type does not support encoding', tokenizerType);
1102	1096	return [];
1103	1097	}
1104		- if (tokenizerType === tokenizers.MANUAL_SELECTION) {
1105		- endpointUrl += `?model=${overrideModel}`;
1106		- }
1107	1098	if (tokenizerType === tokenizers.OPENAI) {
1108	1099	endpointUrl += `?model=${getTokenizerModel()}`;
1109	1100	}