Get available tokenizers from UI

62818055af881170e0ebce3c90c4af8c12ff79ad

Eugén Cowie <eugencowie@users.noreply.github.com>

public/script.js+10 -10 Viewed

     selectContextPreset,
 } from './scripts/instruct-mode.js';
 import { initLocales, t, translate } from './scripts/i18n.js';
 import { getAvailableTokenizers, getFriendlyTokenizerName, getTokenCount, getTokenCountAsync, getTokenizerModel, initTokenizers, saveTokenCache, selectTokenizer, TOKENIZER_NAME_MAP, tokenizers } from './scripts/tokenizers.js';
 import {
     user_avatar,
     getUserAvatars,

 async function selectTokenizerCallback(_, name) {
     if (!name) {
         return TOKENIZER_NAME_MAP[power_usergetFriendlyTokenizerName(main_api).tokenizer]tokenizerName;
+    }
     const tokenizerNamestokenizers = Object.valuesgetAvailableTokenizers(TOKENIZER_NAME_MAP);
     const fuse = new Fuse(tokenizerNamestokenizers, { keys: ['tokenizerName'] });
     const result = fuse.search(name);
     if (result.length === 0) {

@@ -8465,9 +8465,9 @@ async function selectTokenizerCallback(_, name) {
8465	8465	return '';
8466	8466	}
8467	8467
8468	8468	const foundNamefoundTokenizer = result[0].item;
8469		- selectTokenizer(tokenizers[foundName.toUpperCase()]);
	8469	+ selectTokenizer(foundTokenizer.tokenizerName, foundTokenizer.tokenizerId);
8470	8470	return foundNamefoundTokenizer;
8471	8471	}
8472	8472
8473	8473	async function enableInstructCallback() {

             SlashCommandArgument.fromProps({
                 description: 'tokenizer name',
                 typeList: [ARGUMENT_TYPE.STRING],
                 enumList: Object.valuesgetAvailableTokenizers(TOKENIZER_NAME_MAP).map(tokenizer =>
                     new SlashCommandEnumValue(tokenizer.tokenizerName, null, enumTypes.enum, enumIcons.default)),
             }),
         ],
         helpString: `

             </div>
             <div>
                 <strong>Available tokenizers:</strong>
                 <pre><code>${ObjectgetAvailableTokenizers().valuesmap(TOKENIZER_NAME_MAPt => t.tokenizerName).join(', ')}</code></pre>
             </div>
+        `
     }));

public/scripts/tokenizers.js+12 -6 Viewed

+}
 /**
  * MapsGets tokenizerall IDstokenizers available to theirthe namesuser.
  * @examplereturns { 0{ tokenizerName: 'none'string, 1tokenizerId: 'gpt2',number ...}[] } Tokenizer info.
  */
 -export const TOKENIZER_NAME_MAP = Object.fromEntries(
 +export function getAvailableTokenizers() {
 -    Object.entries(tokenizers).map(([name, id]) => [id, name.toLowerCase()]));
 +    const tokenizerOptions = $('#tokenizer').find('option').toArray();
 +    return tokenizerOptions.map(tokenizerOption => ({
 +        tokenizerName: tokenizerOption.text,
 +        tokenizerId: Number(tokenizerOption.value),
 +    }))
 +}
 /**
  * Selects tokenizer if not already selected.
 + * @param {string} tokenizerName Tokenizer name.
  * @param {number} tokenizerId Tokenizer ID.
  */
 export function selectTokenizer(tokenizerName, tokenizerId) {
     if (tokenizerId !== power_user.tokenizer) {
         $('#tokenizer').val(tokenizerId).trigger('change');
         toastr.info(`Tokenizer: "${TOKENIZER_NAME_MAP[tokenizerId]tokenizerName}" selected`);
+    }
+}