Silently redirect tokenization to the next matching tokenizer

8c5b02d2eafa6a1172a8c0a2e4ca53648d4b3e63

Cohee <18619528+Cohee1207@users.noreply.github.com>

public/scripts/tokenizers.js+11 -6 Viewed

 function apiFailureTokenCount(str) {
     console.error('Error counting tokens');
+    let shouldTryAgain = false;
     if (!sessionStorage.getItem(TOKENIZER_WARNING_KEY)) {
-        toastr.warning(
+        const bestMatchBefore = getTokenizerBestMatch(main_api);
-            'Your selected API doesn\'t support the tokenization endpoint. Using estimated counts.',
-            'Error counting tokens',
-            { timeOut: 10000, preventDuplicates: true },
-        );
         sessionStorage.setItem(TOKENIZER_WARNING_KEY, String(true));
+        const bestMatchAfter = getTokenizerBestMatch(main_api);
+        if ([tokenizers.API_TEXTGENERATIONWEBUI, tokenizers.API_KOBOLD].includes(bestMatchBefore) && bestMatchBefore !== bestMatchAfter) {
+            shouldTryAgain = true;
+        }
+    }
+    // Only try again if we guarantee not to be looped by the same error
+    if (shouldTryAgain && power_user.tokenizer === tokenizers.BEST_MATCH) {
+        return getTokenCount(str);
+    }
     return guesstimate(str);