Merge pull request #3221 from Succubyss/manual_selection tokenizers.js: Add MANUAL_SELECTION (& removes turbo-0301 lines)

0cb82b32881e7edfcf2d933594913d0bcb419424

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

public/scripts/tokenizers.js+10 -8 Viewed

     COMMAND_R: 16,
     NEMO: 17,
     BEST_MATCH: 99,
+    MANUAL_SELECTION: 411,
 };
 // A list of local tokenizers that support encoding and decoding token ids.

         return oai_settings.openai_model;
+    }
-    const turbo0301Tokenizer = 'gpt-3.5-turbo-0301';
     const turboTokenizer = 'gpt-3.5-turbo';
     const gpt4Tokenizer = 'gpt-4';
     const gpt4oTokenizer = 'gpt-4o';

@@ -562,9 +562,6 @@ export function getTokenizerModel() {
562	if (oai_settings.windowai_model.includes('gpt-4')) {	562	if (oai_settings.windowai_model.includes('gpt-4')) {
563	return gpt4Tokenizer;	563	return gpt4Tokenizer;
564	}	564	}
565	else if (oai_settings.windowai_model.includes('gpt-3.5-turbo-0301')) {
566	return turbo0301Tokenizer;
567	}
568	else if (oai_settings.windowai_model.includes('gpt-3.5-turbo')) {	565	else if (oai_settings.windowai_model.includes('gpt-3.5-turbo')) {
569	return turboTokenizer;	566	return turboTokenizer;
570	}	567	}

@@ -610,9 +607,6 @@ export function getTokenizerModel() {
610	else if (oai_settings.openrouter_model.includes('gpt-4')) {	607	else if (oai_settings.openrouter_model.includes('gpt-4')) {
611	return gpt4Tokenizer;	608	return gpt4Tokenizer;
612	}	609	}
613	else if (oai_settings.openrouter_model.includes('gpt-3.5-turbo-0301')) {
614	return turbo0301Tokenizer;
615	}
616	else if (oai_settings.openrouter_model.includes('gpt-3.5-turbo')) {	610	else if (oai_settings.openrouter_model.includes('gpt-3.5-turbo')) {
617	return turboTokenizer;	611	return turboTokenizer;
618	}	612	}

  * Encodes a string to tokens using the server API.
  * @param {number} tokenizerType Tokenizer type.
  * @param {string} str String to tokenize.
+ * @param {string} overrideModel Tokenizer for {tokenizers.MANUAL_SELECTION}.
  * @returns {number[]} Array of token ids.
  */
-export function getTextTokens(tokenizerType, str) {
+export function getTextTokens(tokenizerType, str, overrideModel = undefined) {
+    if (overrideModel && tokenizerType !== tokenizers.MANUAL_SELECTION) {
+        console.warn('overrideModel must be undefined unless using tokenizers.MANUAL_SELECTION', tokenizerType);
+        return [];
+    }
     switch (tokenizerType) {
         case tokenizers.API_CURRENT:
             return getTextTokens(currentRemoteTokenizerAPI(), str);

@@ -1087,6 +1086,9 @@ export function getTextTokens(tokenizerType, str) {
1087	console.warn('This tokenizer type does not support encoding', tokenizerType);	1086	console.warn('This tokenizer type does not support encoding', tokenizerType);
1088	return [];	1087	return [];
1089	}	1088	}
		1089	if (tokenizerType === tokenizers.MANUAL_SELECTION) {
		1090	endpointUrl += `?model=${overrideModel}`;
		1091	}
1090	if (tokenizerType === tokenizers.OPENAI) {	1092	if (tokenizerType === tokenizers.OPENAI) {
1091	endpointUrl += `?model=${getTokenizerModel()}`;	1093	endpointUrl += `?model=${getTokenizerModel()}`;
1092	}	1094	}