Use string byte length for token guesstimation (#5267) * Use string byte length for token guesstimation * Use Buffer.byteLength on backend * Preserve TextEncoder instance

e0ed67357ccc02bb3ae8f9b605a85354b44c91ec

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

public/scripts/tokenizers.js+5 -2 Viewed

 import { kai_flags, kai_settings } from './kai-settings.js';
 import { textgen_types, textgenerationwebui_settings as textgen_settings, getTextGenServer, getTextGenModel } from './textgen-settings.js';
 import { getCurrentDreamGenModelTokenizer, getCurrentOpenRouterModelTokenizer, openRouterModels } from './textgen-models.js';
+export { BYTES_PER_TOKEN as CHARACTERS_PER_TOKEN_RATIO };
-export const CHARACTERS_PER_TOKEN_RATIO = 3.35;
+export const BYTES_PER_TOKEN = 3.35;
 export const TOKENIZER_WARNING_KEY = 'tokenizationWarningShown';
 export const TOKENIZER_SUPPORTED_KEY = 'tokenizationSupported';

     },
 };
+const textEncoder = new TextEncoder();
 const objectStore = localforage.createInstance({ name: 'SillyTavern_ChatCompletions' });
 let tokenCache = {};

  * @returns {number} Token count.
  */
 export function guesstimate(str) {
-    return Math.ceil(str.length / CHARACTERS_PER_TOKEN_RATIO);
+    const byteLength = textEncoder.encode(str).length;
+    return Math.ceil(byteLength / BYTES_PER_TOKEN);
+}
 async function loadTokenCache() {

src/endpoints/tokenizers.js+14 -4 Viewed

     'code-search-ada-code-001',
 ];
-const CHARS_PER_TOKEN = 3.35;
+const BYTES_PER_TOKEN = 3.35;
 const IS_DOWNLOAD_ALLOWED = getConfigValue('enableDownloadableTokenizers', true, 'boolean');
 const gunzip = promisify(zlib.gunzip);
 /**
+ * Guesstimates the token count for a string.
+ * @param {string} str String to tokenize.
+ * @returns {number} Token count.
+ */
+function guesstimate(str) {
+    const byteLength = Buffer.byteLength(str, 'utf8');
+    return Math.ceil(byteLength / BYTES_PER_TOKEN);
+}
+/**
  * Gets a path to the tokenizer model. Downloads the model if it's a URL.
  * @param {string} model Model URL or path
  * @param {string|undefined} fallbackModel Fallback model path

     if (!instance) {
         return {
             ids: [],
-            count: Math.ceil(text.length / CHARS_PER_TOKEN),
+            count: guesstimate(text),
         };
+    }

     // Fallback to strlen estimation
     if (!tokenizer) {
-        return Math.ceil(convertedPrompt.length / CHARS_PER_TOKEN);
+        return guesstimate(convertedPrompt);
+    }
     const count = tokenizer.encode(convertedPrompt).length;

     } catch (error) {
         console.error('An error counting tokens, using fallback estimation method', error);
         const jsonBody = JSON.stringify(req.body);
-        const num_tokens = Math.ceil(jsonBody.length / CHARS_PER_TOKEN);
+        const num_tokens = guesstimate(jsonBody);
         res.send({ 'token_count': num_tokens });
+    }
 });