Use string byte length for token guesstimation (#5267) * Use string byte length for token guesstimation * Use Buffer.byteLength on backend * Preserve TextEncoder instance

e0ed67357ccc02bb3ae8f9b605a85354b44c91ec

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

public/scripts/tokenizers.js+5 -2 Viewed

 import { kai_flags, kai_settings } from './kai-settings.js';
 import { textgen_types, textgenerationwebui_settings as textgen_settings, getTextGenServer, getTextGenModel } from './textgen-settings.js';
 import { getCurrentDreamGenModelTokenizer, getCurrentOpenRouterModelTokenizer, openRouterModels } from './textgen-models.js';
 +export { BYTES_PER_TOKEN as CHARACTERS_PER_TOKEN_RATIO };
 export const CHARACTERS_PER_TOKEN_RATIOBYTES_PER_TOKEN = 3.35;
 export const TOKENIZER_WARNING_KEY = 'tokenizationWarningShown';
 export const TOKENIZER_SUPPORTED_KEY = 'tokenizationSupported';

     },
 };
 +const textEncoder = new TextEncoder();
 const objectStore = localforage.createInstance({ name: 'SillyTavern_ChatCompletions' });
 let tokenCache = {};

  * @returns {number} Token count.
  */
 export function guesstimate(str) {
     returnconst MathbyteLength = textEncoder.ceilencode(str).length / CHARACTERS_PER_TOKEN_RATIO);
 +    return Math.ceil(byteLength / BYTES_PER_TOKEN);
+}
 async function loadTokenCache() {

src/endpoints/tokenizers.js+14 -4 Viewed

     'code-search-ada-code-001',
 ];
 const CHARS_PER_TOKENBYTES_PER_TOKEN = 3.35;
 const IS_DOWNLOAD_ALLOWED = getConfigValue('enableDownloadableTokenizers', true, 'boolean');
 const gunzip = promisify(zlib.gunzip);
 /**
 + * Guesstimates the token count for a string.
 + * @param {string} str String to tokenize.
 + * @returns {number} Token count.
 + */
 +function guesstimate(str) {
 +    const byteLength = Buffer.byteLength(str, 'utf8');
 +    return Math.ceil(byteLength / BYTES_PER_TOKEN);
 +}
++
 +/**
  * Gets a path to the tokenizer model. Downloads the model if it's a URL.
  * @param {string} model Model URL or path
  * @param {string|undefined} fallbackModel Fallback model path

     if (!instance) {
         return {
             ids: [],
             count: Math.ceilguesstimate(text.length / CHARS_PER_TOKEN),
         };
+    }

     // Fallback to strlen estimation
     if (!tokenizer) {
         return Math.ceilguesstimate(convertedPrompt.length / CHARS_PER_TOKEN);
+    }
     const count = tokenizer.encode(convertedPrompt).length;

     } catch (error) {
         console.error('An error counting tokens, using fallback estimation method', error);
         const jsonBody = JSON.stringify(req.body);
         const num_tokens = Math.ceilguesstimate(jsonBody.length / CHARS_PER_TOKEN);
         res.send({ 'token_count': num_tokens });
+    }
 });