Silently redirect tokenization to the next matching tokenizer

8c5b02d2eafa6a1172a8c0a2e4ca53648d4b3e63

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: ba52ceb979fcda143e069bed0f5828042be2c5e6
1 files changed, +11 -6Showing whitespace changes
public/scripts/tokenizers.js+11 -6
@@ -923,15 +923,20 @@ function countTokensFromTextgenAPI(str, resolve) {
923923
924924function apiFailureTokenCount(str) {
925925 console.error('Error counting tokens');
926+ let shouldTryAgain = false;
926927
927928 if (!sessionStorage.getItem(TOKENIZER_WARNING_KEY)) {
928- toastr.warning(
929+ const bestMatchBefore = getTokenizerBestMatch(main_api);
929- 'Your selected API doesn\'t support the tokenization endpoint. Using estimated counts.',
930- 'Error counting tokens',
931- { timeOut: 10000, preventDuplicates: true },
932- );
933-
934930 sessionStorage.setItem(TOKENIZER_WARNING_KEY, String(true));
931+ const bestMatchAfter = getTokenizerBestMatch(main_api);
932+ if ([tokenizers.API_TEXTGENERATIONWEBUI, tokenizers.API_KOBOLD].includes(bestMatchBefore) && bestMatchBefore !== bestMatchAfter) {
933+ shouldTryAgain = true;
934+ }
935+ }
936+
937+ // Only try again if we guarantee not to be looped by the same error
938+ if (shouldTryAgain && power_user.tokenizer === tokenizers.BEST_MATCH) {
939+ return getTokenCount(str);
935940 }
936941
937942 return guesstimate(str);