Merge pull request #3221 from Succubyss/manual_selection tokenizers.js: Add MANUAL_SELECTION (& removes turbo-0301 lines)

0cb82b32881e7edfcf2d933594913d0bcb419424

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

public/scripts/tokenizers.js+10 -8 Viewed

     COMMAND_R: 16,
     NEMO: 17,
     BEST_MATCH: 99,
 +    MANUAL_SELECTION: 411,
 };
 // A list of local tokenizers that support encoding and decoding token ids.

         return oai_settings.openai_model;
+    }
 -    const turbo0301Tokenizer = 'gpt-3.5-turbo-0301';
     const turboTokenizer = 'gpt-3.5-turbo';
     const gpt4Tokenizer = 'gpt-4';
     const gpt4oTokenizer = 'gpt-4o';

@@ -562,9 +562,6 @@ export function getTokenizerModel() {
562	562	if (oai_settings.windowai_model.includes('gpt-4')) {
563	563	return gpt4Tokenizer;
564	564	}
565		- else if (oai_settings.windowai_model.includes('gpt-3.5-turbo-0301')) {
566		- return turbo0301Tokenizer;
567		- }
568	565	else if (oai_settings.windowai_model.includes('gpt-3.5-turbo')) {
569	566	return turboTokenizer;
570	567	}

@@ -610,9 +607,6 @@ export function getTokenizerModel() {
610	607	else if (oai_settings.openrouter_model.includes('gpt-4')) {
611	608	return gpt4Tokenizer;
612	609	}
613		- else if (oai_settings.openrouter_model.includes('gpt-3.5-turbo-0301')) {
614		- return turbo0301Tokenizer;
615		- }
616	610	else if (oai_settings.openrouter_model.includes('gpt-3.5-turbo')) {
617	611	return turboTokenizer;
618	612	}

  * Encodes a string to tokens using the server API.
  * @param {number} tokenizerType Tokenizer type.
  * @param {string} str String to tokenize.
 + * @param {string} overrideModel Tokenizer for {tokenizers.MANUAL_SELECTION}.
  * @returns {number[]} Array of token ids.
  */
 export function getTextTokens(tokenizerType, str, overrideModel = undefined) {
 +    if (overrideModel && tokenizerType !== tokenizers.MANUAL_SELECTION) {
 +        console.warn('overrideModel must be undefined unless using tokenizers.MANUAL_SELECTION', tokenizerType);
 +        return [];
 +    }
     switch (tokenizerType) {
         case tokenizers.API_CURRENT:
             return getTextTokens(currentRemoteTokenizerAPI(), str);

@@ -1087,6 +1086,9 @@ export function getTextTokens(tokenizerType, str) {
1087	1086	console.warn('This tokenizer type does not support encoding', tokenizerType);
1088	1087	return [];
1089	1088	}
	1089	+ if (tokenizerType === tokenizers.MANUAL_SELECTION) {
	1090	+ endpointUrl += `?model=${overrideModel}`;
	1091	+ }
1090	1092	if (tokenizerType === tokenizers.OPENAI) {
1091	1093	endpointUrl += `?model=${getTokenizerModel()}`;
1092	1094	}