Move command registration
| @@ -212,7 +212,7 @@ import { | |||
| 212 | selectContextPreset, | 212 | selectContextPreset, |
| 213 | } from './scripts/instruct-mode.js'; | 213 | } from './scripts/instruct-mode.js'; |
| 214 | import { initLocales, t, translate } from './scripts/i18n.js'; | 214 | import { initLocales, t, translate } from './scripts/i18n.js'; |
| 215 | import { getAvailableTokenizers, getFriendlyTokenizerName, getTokenCount, getTokenCountAsync, getTokenizerModel, initTokenizers, saveTokenCache, selectTokenizer } from './scripts/tokenizers.js'; | 215 | import { getFriendlyTokenizerName, getTokenCount, getTokenCountAsync, getTokenizerModel, initTokenizers, saveTokenCache } from './scripts/tokenizers.js'; |
| 216 | import { | 216 | import { |
| 217 | user_avatar, | 217 | user_avatar, |
| 218 | getUserAvatars, | 218 | getUserAvatars, |
| @@ -8451,27 +8451,6 @@ async function selectInstructCallback(_, name) { | |||
| 8451 | return foundName; | 8451 | return foundName; |
| 8452 | } | 8452 | } |
| 8453 | 8453 | ||
| 8454 | async function selectTokenizerCallback(_, name) { | ||
| 8455 | if (!name) { | ||
| 8456 | return getAvailableTokenizers().find(tokenizer => tokenizer.tokenizerId === power_user.tokenizer)?.tokenizerKey ?? ''; | ||
| 8457 | } | ||
| 8458 | |||
| 8459 | const tokenizers = getAvailableTokenizers(); | ||
| 8460 | const fuse = new Fuse(tokenizers, { keys: ['tokenizerKey', 'tokenizerName'] }); | ||
| 8461 | const result = fuse.search(name); | ||
| 8462 | |||
| 8463 | if (result.length === 0) { | ||
| 8464 | toastr.warning(`Tokenizer "${name}" not found`); | ||
| 8465 | return ''; | ||
| 8466 | } | ||
| 8467 | |||
| 8468 | /** @type {import('./scripts/tokenizers.js').Tokenizer} */ | ||
| 8469 | const foundTokenizer = result[0].item; | ||
| 8470 | selectTokenizer(foundTokenizer.tokenizerId); | ||
| 8471 | |||
| 8472 | return foundTokenizer.tokenizerKey; | ||
| 8473 | } | ||
| 8474 | |||
| 8475 | async function enableInstructCallback() { | 8454 | async function enableInstructCallback() { |
| 8476 | $('#instruct_enabled').prop('checked', true).trigger('change'); | 8455 | $('#instruct_enabled').prop('checked', true).trigger('change'); |
| 8477 | return ''; | 8456 | return ''; |
| @@ -9117,28 +9096,6 @@ jQuery(async function () { | |||
| 9117 | `, | 9096 | `, |
| 9118 | })); | 9097 | })); |
| 9119 | SlashCommandParser.addCommandObject(SlashCommand.fromProps({ | 9098 | SlashCommandParser.addCommandObject(SlashCommand.fromProps({ |
| 9120 | name: 'tokenizer', | ||
| 9121 | callback: selectTokenizerCallback, | ||
| 9122 | returns: 'current tokenizer', | ||
| 9123 | unnamedArgumentList: [ | ||
| 9124 | SlashCommandArgument.fromProps({ | ||
| 9125 | description: 'tokenizer name', | ||
| 9126 | typeList: [ARGUMENT_TYPE.STRING], | ||
| 9127 | enumList: getAvailableTokenizers().map(tokenizer => | ||
| 9128 | new SlashCommandEnumValue(tokenizer.tokenizerKey, tokenizer.tokenizerName, enumTypes.enum, enumIcons.default)), | ||
| 9129 | }), | ||
| 9130 | ], | ||
| 9131 | helpString: ` | ||
| 9132 | <div> | ||
| 9133 | Selects tokenizer by name. Gets the current tokenizer if no name is provided. | ||
| 9134 | </div> | ||
| 9135 | <div> | ||
| 9136 | <strong>Available tokenizers:</strong> | ||
| 9137 | <pre><code>${getAvailableTokenizers().map(t => t.tokenizerKey).join(', ')}</code></pre> | ||
| 9138 | </div> | ||
| 9139 | ` | ||
| 9140 | })); | ||
| 9141 | SlashCommandParser.addCommandObject(SlashCommand.fromProps({ | ||
| 9142 | name: 'instruct-on', | 9099 | name: 'instruct-on', |
| 9143 | callback: enableInstructCallback, | 9100 | callback: enableInstructCallback, |
| 9144 | helpString: 'Enables instruct mode.', | 9101 | helpString: 'Enables instruct mode.', |
| @@ -52,7 +52,7 @@ import { chat_completion_sources, oai_settings, setupChatCompletionPromptManager | |||
| 52 | import { autoSelectPersona, retriggerFirstMessageOnEmptyChat, setPersonaLockState, togglePersonaLock, user_avatar } from './personas.js'; | 52 | import { autoSelectPersona, retriggerFirstMessageOnEmptyChat, setPersonaLockState, togglePersonaLock, user_avatar } from './personas.js'; |
| 53 | import { addEphemeralStoppingString, chat_styles, flushEphemeralStoppingStrings, power_user } from './power-user.js'; | 53 | import { addEphemeralStoppingString, chat_styles, flushEphemeralStoppingStrings, power_user } from './power-user.js'; |
| 54 | import { SERVER_INPUTS, textgen_types, textgenerationwebui_settings } from './textgen-settings.js'; | 54 | import { SERVER_INPUTS, textgen_types, textgenerationwebui_settings } from './textgen-settings.js'; |
| 55 | import { decodeTextTokens, getFriendlyTokenizerName, getTextTokens, getTokenCountAsync } from './tokenizers.js'; | 55 | import { decodeTextTokens, getAvailableTokenizers, getFriendlyTokenizerName, getTextTokens, getTokenCountAsync, selectTokenizer } from './tokenizers.js'; |
| 56 | import { debounce, delay, isFalseBoolean, isTrueBoolean, showFontAwesomePicker, stringToRange, trimToEndSentence, trimToStartSentence, waitUntilCondition } from './utils.js'; | 56 | import { debounce, delay, isFalseBoolean, isTrueBoolean, showFontAwesomePicker, stringToRange, trimToEndSentence, trimToStartSentence, waitUntilCondition } from './utils.js'; |
| 57 | import { registerVariableCommands, resolveVariable } from './variables.js'; | 57 | import { registerVariableCommands, resolveVariable } from './variables.js'; |
| 58 | import { background_settings } from './backgrounds.js'; | 58 | import { background_settings } from './backgrounds.js'; |
| @@ -1558,6 +1558,28 @@ export function initDefaultSlashCommands() { | |||
| 1558 | </div> | 1558 | </div> |
| 1559 | `, | 1559 | `, |
| 1560 | })); | 1560 | })); |
| 1561 | SlashCommandParser.addCommandObject(SlashCommand.fromProps({ | ||
| 1562 | name: 'tokenizer', | ||
| 1563 | callback: selectTokenizerCallback, | ||
| 1564 | returns: 'current tokenizer', | ||
| 1565 | unnamedArgumentList: [ | ||
| 1566 | SlashCommandArgument.fromProps({ | ||
| 1567 | description: 'tokenizer name', | ||
| 1568 | typeList: [ARGUMENT_TYPE.STRING], | ||
| 1569 | enumList: getAvailableTokenizers().map(tokenizer => | ||
| 1570 | new SlashCommandEnumValue(tokenizer.tokenizerKey, tokenizer.tokenizerName, enumTypes.enum, enumIcons.default)), | ||
| 1571 | }), | ||
| 1572 | ], | ||
| 1573 | helpString: ` | ||
| 1574 | <div> | ||
| 1575 | Selects tokenizer by name. Gets the current tokenizer if no name is provided. | ||
| 1576 | </div> | ||
| 1577 | <div> | ||
| 1578 | <strong>Available tokenizers:</strong> | ||
| 1579 | <pre><code>${getAvailableTokenizers().map(t => t.tokenizerKey).join(', ')}</code></pre> | ||
| 1580 | </div> | ||
| 1581 | `, | ||
| 1582 | })); | ||
| 1561 | 1583 | ||
| 1562 | registerVariableCommands(); | 1584 | registerVariableCommands(); |
| 1563 | } | 1585 | } |
| @@ -3562,6 +3584,27 @@ async function setApiUrlCallback({ api = null, connect = 'true' }, url) { | |||
| 3562 | return textgenerationwebui_settings.server_urls[type] ?? ''; | 3584 | return textgenerationwebui_settings.server_urls[type] ?? ''; |
| 3563 | } | 3585 | } |
| 3564 | 3586 | ||
| 3587 | async function selectTokenizerCallback(_, name) { | ||
| 3588 | if (!name) { | ||
| 3589 | return getAvailableTokenizers().find(tokenizer => tokenizer.tokenizerId === power_user.tokenizer)?.tokenizerKey ?? ''; | ||
| 3590 | } | ||
| 3591 | |||
| 3592 | const tokenizers = getAvailableTokenizers(); | ||
| 3593 | const fuse = new Fuse(tokenizers, { keys: ['tokenizerKey', 'tokenizerName'] }); | ||
| 3594 | const result = fuse.search(name); | ||
| 3595 | |||
| 3596 | if (result.length === 0) { | ||
| 3597 | toastr.warning(`Tokenizer "${name}" not found`); | ||
| 3598 | return ''; | ||
| 3599 | } | ||
| 3600 | |||
| 3601 | /** @type {import('./tokenizers.js').Tokenizer} */ | ||
| 3602 | const foundTokenizer = result[0].item; | ||
| 3603 | selectTokenizer(foundTokenizer.tokenizerId); | ||
| 3604 | |||
| 3605 | return foundTokenizer.tokenizerKey; | ||
| 3606 | } | ||
| 3607 | |||
| 3565 | export let isExecutingCommandsFromChatInput = false; | 3608 | export let isExecutingCommandsFromChatInput = false; |
| 3566 | export let commandsFromChatInputAbortController; | 3609 | export let commandsFromChatInputAbortController; |
| 3567 | 3610 | ||