Add slash command to set tokenizer

5f92b8a09e3638475756b575e4859f7f4e9f907a

Eugén Cowie <eugencowie@users.noreply.github.com>

public/script.js+42 -1 Viewed

     selectContextPreset,
 } from './scripts/instruct-mode.js';
 import { initLocales, t, translate } from './scripts/i18n.js';
-import { getFriendlyTokenizerName, getTokenCount, getTokenCountAsync, getTokenizerModel, initTokenizers, saveTokenCache } from './scripts/tokenizers.js';
+import { getFriendlyTokenizerName, getTokenCount, getTokenCountAsync, getTokenizerModel, initTokenizers, saveTokenCache, selectTokenizer, TOKENIZER_NAME_MAP, tokenizers } from './scripts/tokenizers.js';
 import {
     user_avatar,
     getUserAvatars,

     return foundName;
+}
+async function selectTokenizerCallback(_, name) {
+    if (!name) {
+        return TOKENIZER_NAME_MAP[power_user.tokenizer];
+    }
+    const tokenizerNames = Object.values(TOKENIZER_NAME_MAP);
+    const fuse = new Fuse(tokenizerNames);
+    const result = fuse.search(name);
+    if (result.length === 0) {
+        toastr.warning(`Tokenizer "${name}" not found`);
+        return '';
+    }
+    const foundName = result[0].item;
+    selectTokenizer(tokenizers[foundName.toUpperCase()]);
+    return foundName;
+}
 async function enableInstructCallback() {
     $('#instruct_enabled').prop('checked', true).trigger('change');
     return '';

         `,
     }));
     SlashCommandParser.addCommandObject(SlashCommand.fromProps({
+        name: 'tokenizer',
+        callback: selectTokenizerCallback,
+        returns: 'current tokenizer',
+        unnamedArgumentList: [
+            SlashCommandArgument.fromProps({
+                description: 'tokenizer name',
+                typeList: [ARGUMENT_TYPE.STRING],
+                enumList: Object.values(TOKENIZER_NAME_MAP).map(tokenizer =>
+                    new SlashCommandEnumValue(tokenizer, null, enumTypes.enum, enumIcons.default)),
+            }),
+        ],
+        helpString: `
+            <div>
+                Selects tokenizer by name. Gets the current tokenizer if no name is provided.
+            </div>
+            <div>
+                <strong>Available tokenizers:</strong>
+                <pre><code>${Object.values(TOKENIZER_NAME_MAP).join(', ')}</code></pre>
+            </div>
+        `
+    }));
+    SlashCommandParser.addCommandObject(SlashCommand.fromProps({
         name: 'instruct-on',
         callback: enableInstructCallback,
         helpString: 'Enables instruct mode.',

public/scripts/tokenizers.js+18 -0 Viewed

+}
 /**
+ * Maps tokenizer IDs to their names.
+ * @example { 0: 'none', 1: 'gpt2', ... }
+ */
+export const TOKENIZER_NAME_MAP = Object.fromEntries(
+    Object.entries(tokenizers).map(([name, id]) => [id, name.toLowerCase()]));
+/**
+ * Selects tokenizer if not already selected.
+ * @param {number} tokenizerId Tokenizer ID.
+ */
+export function selectTokenizer(tokenizerId) {
+    if (tokenizerId !== power_user.tokenizer) {
+        $('#tokenizer').val(tokenizerId).trigger('change');
+        toastr.info(`Tokenizer: "${TOKENIZER_NAME_MAP[tokenizerId]}" selected`);
+    }
+}
+/**
  * Gets the friendly name of the current tokenizer.
  * @param {string} forApi API to get the tokenizer for. Defaults to the main API.
  * @returns { { tokenizerName: string, tokenizerId: number } } Tokenizer info