Merge pull request #2683 from eugencowie/tokenizer-slash-command Add slash command to set tokenizer

02d6df3acec2963d3c321a7f6efa335430f1769e

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

Parents: 44e62156b7cfd5ed2d23673f341ae90cc142fa45 976be98dd755b1e2ff5bb8f3f9cceef5520f6c4e

public/scripts/slash-commands.js+44 -1 Viewed

 import { autoSelectPersona, retriggerFirstMessageOnEmptyChat, setPersonaLockState, togglePersonaLock, user_avatar } from './personas.js';
 import { addEphemeralStoppingString, chat_styles, flushEphemeralStoppingStrings, power_user } from './power-user.js';
 import { SERVER_INPUTS, textgen_types, textgenerationwebui_settings } from './textgen-settings.js';
 import { decodeTextTokens, getAvailableTokenizers, getFriendlyTokenizerName, getTextTokens, getTokenCountAsync, selectTokenizer } from './tokenizers.js';
 import { debounce, delay, isFalseBoolean, isTrueBoolean, showFontAwesomePicker, stringToRange, trimToEndSentence, trimToStartSentence, waitUntilCondition } from './utils.js';
 import { registerVariableCommands, resolveVariable } from './variables.js';
 import { background_settings } from './backgrounds.js';

             </div>
         `,
     }));
 +    SlashCommandParser.addCommandObject(SlashCommand.fromProps({
 +        name: 'tokenizer',
 +        callback: selectTokenizerCallback,
 +        returns: 'current tokenizer',
 +        unnamedArgumentList: [
 +            SlashCommandArgument.fromProps({
 +                description: 'tokenizer name',
 +                typeList: [ARGUMENT_TYPE.STRING],
 +                enumList: getAvailableTokenizers().map(tokenizer =>
 +                    new SlashCommandEnumValue(tokenizer.tokenizerKey, tokenizer.tokenizerName, enumTypes.enum, enumIcons.default)),
 +            }),
 +        ],
 +        helpString: `
 +            <div>
 +                Selects tokenizer by name. Gets the current tokenizer if no name is provided.
 +            </div>
 +            <div>
 +                <strong>Available tokenizers:</strong>
 +                <pre><code>${getAvailableTokenizers().map(t => t.tokenizerKey).join(', ')}</code></pre>
 +            </div>
 +        `,
 +    }));
     registerVariableCommands();
+}

     return textgenerationwebui_settings.server_urls[type] ?? '';
+}
 +async function selectTokenizerCallback(_, name) {
 +    if (!name) {
 +        return getAvailableTokenizers().find(tokenizer => tokenizer.tokenizerId === power_user.tokenizer)?.tokenizerKey ?? '';
 +    }
++
 +    const tokenizers = getAvailableTokenizers();
 +    const fuse = new Fuse(tokenizers, { keys: ['tokenizerKey', 'tokenizerName'] });
 +    const result = fuse.search(name);
++
 +    if (result.length === 0) {
 +        toastr.warning(`Tokenizer "${name}" not found`);
 +        return '';
 +    }
++
 +    /** @type {import('./tokenizers.js').Tokenizer} */
 +    const foundTokenizer = result[0].item;
 +    selectTokenizer(foundTokenizer.tokenizerId);
++
 +    return foundTokenizer.tokenizerKey;
 +}
++
 export let isExecutingCommandsFromChatInput = false;
 export let commandsFromChatInputAbortController;

public/scripts/tokenizers.js+40 -2 Viewed

+}
 /**
 + * @typedef {object} Tokenizer
 + * @property {number} tokenizerId - The id of the tokenizer option
 + * @property {string} tokenizerKey - Internal name/key of the tokenizer
 + * @property {string} tokenizerName - Human-readable detailed name of the tokenizer (as displayed in the UI)
 + */
++
 +/**
 + * Gets all tokenizers available to the user.
 + * @returns {Tokenizer[]} Tokenizer info.
 + */
 +export function getAvailableTokenizers() {
 +    const tokenizerOptions = $('#tokenizer').find('option').toArray();
 +    return tokenizerOptions.map(tokenizerOption => ({
 +        tokenizerId: Number(tokenizerOption.value),
 +        tokenizerKey: Object.entries(tokenizers).find(([_, value]) => value === Number(tokenizerOption.value))[0].toLocaleLowerCase(),
 +        tokenizerName: tokenizerOption.text,
 +    }))
 +}
++
 +/**
 + * Selects tokenizer if not already selected.
 + * @param {number} tokenizerId Tokenizer ID.
 + */
 +export function selectTokenizer(tokenizerId) {
 +    if (tokenizerId !== power_user.tokenizer) {
 +        const tokenizer = getAvailableTokenizers().find(tokenizer => tokenizer.tokenizerId === tokenizerId);
 +        if (!tokenizer) {
 +            console.warn('Failed to find tokenizer with id', tokenizerId);
 +            return;
 +        }
 +        $('#tokenizer').val(tokenizer.tokenizerId).trigger('change');
 +        toastr.info(`Tokenizer: "${tokenizer.tokenizerName}" selected`);
 +    }
 +}
++
 +/**
  * Gets the friendly name of the current tokenizer.
  * @param {string} forApi API to get the tokenizer for. Defaults to the main API.
  * @returns { { tokenizerName: string, tokenizerId: number } Tokenizer} Tokenizer info
  */
 export function getFriendlyTokenizerName(forApi) {
     if (!forApi) {

         ? tokenizers.OPENAI
         : tokenizerId;
 -    return { tokenizerName, tokenizerId };
 +    const tokenizerKey = Object.entries(tokenizers).find(([_, value]) => value === tokenizerId)[0].toLocaleLowerCase();
++
 +    return { tokenizerName, tokenizerKey, tokenizerId };
+}
 /**