Get available tokenizers from UI

62818055af881170e0ebce3c90c4af8c12ff79ad

Eugén Cowie <eugencowie@users.noreply.github.com>

public/script.js+10 -10 Viewed

     selectContextPreset,
 } from './scripts/instruct-mode.js';
 import { initLocales, t, translate } from './scripts/i18n.js';
-import { getFriendlyTokenizerName, getTokenCount, getTokenCountAsync, getTokenizerModel, initTokenizers, saveTokenCache, selectTokenizer, TOKENIZER_NAME_MAP, tokenizers } from './scripts/tokenizers.js';
+import { getAvailableTokenizers, getFriendlyTokenizerName, getTokenCount, getTokenCountAsync, getTokenizerModel, initTokenizers, saveTokenCache, selectTokenizer } from './scripts/tokenizers.js';
 import {
     user_avatar,
     getUserAvatars,

 async function selectTokenizerCallback(_, name) {
     if (!name) {
-        return TOKENIZER_NAME_MAP[power_user.tokenizer];
+        return getFriendlyTokenizerName(main_api).tokenizerName;
+    }
-    const tokenizerNames = Object.values(TOKENIZER_NAME_MAP);
+    const tokenizers = getAvailableTokenizers();
-    const fuse = new Fuse(tokenizerNames);
+    const fuse = new Fuse(tokenizers, { keys: ['tokenizerName'] });
     const result = fuse.search(name);
     if (result.length === 0) {

@@ -8465,9 +8465,9 @@ async function selectTokenizerCallback(_, name) {
8465	return '';	8465	return '';
8466	}	8466	}
8467		8467
8468	const foundName = result[0].item;	8468	const foundTokenizer = result[0].item;
8469	selectTokenizer(tokenizers[foundName.toUpperCase()]);	8469	selectTokenizer(foundTokenizer.tokenizerName, foundTokenizer.tokenizerId);
8470	return foundName;	8470	return foundTokenizer;
8471	}	8471	}
8472		8472
8473	async function enableInstructCallback() {	8473	async function enableInstructCallback() {

             SlashCommandArgument.fromProps({
                 description: 'tokenizer name',
                 typeList: [ARGUMENT_TYPE.STRING],
-                enumList: Object.values(TOKENIZER_NAME_MAP).map(tokenizer =>
+                enumList: getAvailableTokenizers().map(tokenizer =>
-                    new SlashCommandEnumValue(tokenizer, null, enumTypes.enum, enumIcons.default)),
+                    new SlashCommandEnumValue(tokenizer.tokenizerName, null, enumTypes.enum, enumIcons.default)),
             }),
         ],
         helpString: `

             </div>
             <div>
                 <strong>Available tokenizers:</strong>
-                <pre><code>${Object.values(TOKENIZER_NAME_MAP).join(', ')}</code></pre>
+                <pre><code>${getAvailableTokenizers().map(t => t.tokenizerName).join(', ')}</code></pre>
             </div>
+        `
     }));

public/scripts/tokenizers.js+12 -6 Viewed

+}
 /**
- * Maps tokenizer IDs to their names.
+ * Gets all tokenizers available to the user.
- * @example { 0: 'none', 1: 'gpt2', ... }
+ * @returns { { tokenizerName: string, tokenizerId: number }[] } Tokenizer info.
  */
-export const TOKENIZER_NAME_MAP = Object.fromEntries(
+export function getAvailableTokenizers() {
-    Object.entries(tokenizers).map(([name, id]) => [id, name.toLowerCase()]));
+    const tokenizerOptions = $('#tokenizer').find('option').toArray();
+    return tokenizerOptions.map(tokenizerOption => ({
+        tokenizerName: tokenizerOption.text,
+        tokenizerId: Number(tokenizerOption.value),
+    }))
+}
 /**
  * Selects tokenizer if not already selected.
+ * @param {string} tokenizerName Tokenizer name.
  * @param {number} tokenizerId Tokenizer ID.
  */
-export function selectTokenizer(tokenizerId) {
+export function selectTokenizer(tokenizerName, tokenizerId) {
     if (tokenizerId !== power_user.tokenizer) {
         $('#tokenizer').val(tokenizerId).trigger('change');
-        toastr.info(`Tokenizer: "${TOKENIZER_NAME_MAP[tokenizerId]}" selected`);
+        toastr.info(`Tokenizer: "${tokenizerName}" selected`);
+    }
+}