Small update to enum display + consistency
| @@ -8453,11 +8453,11 @@ async function selectInstructCallback(_, name) { | ||
| 8453 | 8453 | |
| 8454 | 8454 | async function selectTokenizerCallback(_, name) { |
| 8455 | 8455 | if (!name) { |
| 8456 | - return getFriendlyTokenizerName(main_api).tokenizerName; | |
| 8456 | + return getAvailableTokenizers().find(tokenizer => tokenizer.tokenizerId === power_user.tokenizer)?.tokenizerKey ?? ''; | |
| 8457 | 8457 | } |
| 8458 | 8458 | |
| 8459 | 8459 | const tokenizers = getAvailableTokenizers(); |
| 8460 | 8460 | const fuse = new Fuse(tokenizers, { keys: ['tokenizerKey', 'tokenizerName'] }); |
| 8461 | 8461 | const result = fuse.search(name); |
| 8462 | 8462 | |
| 8463 | 8463 | if (result.length === 0) { |
| @@ -8465,9 +8465,11 @@ async function selectTokenizerCallback(_, name) { | ||
| 8465 | 8465 | return ''; |
| 8466 | 8466 | } |
| 8467 | 8467 | |
| 8468 | + /** @type {import('./scripts/tokenizers.js').Tokenizer} */ | |
| 8468 | 8469 | const foundTokenizer = result[0].item; |
| 8469 | 8470 | selectTokenizer(foundTokenizer.tokenizerName, foundTokenizer.tokenizerId); |
| 8470 | - return foundTokenizer.tokenizerName; | |
| 8471 | + | |
| 8472 | + return foundTokenizer.tokenizerKey; | |
| 8471 | 8473 | } |
| 8472 | 8474 | |
| 8473 | 8475 | async function enableInstructCallback() { |
| @@ -9123,7 +9125,7 @@ jQuery(async function () { | ||
| 9123 | 9125 | description: 'tokenizer name', |
| 9124 | 9126 | typeList: [ARGUMENT_TYPE.STRING], |
| 9125 | 9127 | enumList: getAvailableTokenizers().map(tokenizer => |
| 9126 | 9128 | new SlashCommandEnumValue(tokenizer.tokenizerNametokenizerKey, nulltokenizer.tokenizerName, enumTypes.enum, enumIcons.default)), |
| 9127 | 9129 | }), |
| 9128 | 9130 | ], |
| 9129 | 9131 | helpString: ` |
| @@ -9132,7 +9134,7 @@ jQuery(async function () { | ||
| 9132 | 9134 | </div> |
| 9133 | 9135 | <div> |
| 9134 | 9136 | <strong>Available tokenizers:</strong> |
| 9135 | 9137 | <pre><code>${getAvailableTokenizers().map(t => t.tokenizerNametokenizerKey).join(', ')}</code></pre> |
| 9136 | 9138 | </div> |
| 9137 | 9139 | ` |
| 9138 | 9140 | })); |
| @@ -148,33 +148,45 @@ async function resetTokenCache() { | ||
| 148 | 148 | } |
| 149 | 149 | |
| 150 | 150 | /** |
| 151 | + * @typedef {object} Tokenizer | |
| 152 | + * @property {number} tokenizerId - The id of the tokenizer option | |
| 153 | + * @property {string} tokenizerKey - Internal name/key of the tokenizer | |
| 154 | + * @property {string} tokenizerName - Human-readable detailed name of the tokenizer (as displayed in the UI) | |
| 155 | + */ | |
| 156 | + | |
| 157 | +/** | |
| 151 | 158 | * Gets all tokenizers available to the user. |
| 152 | 159 | * @returns { { tokenizerName: string, tokenizerId: number }Tokenizer[] } Tokenizer info. |
| 153 | 160 | */ |
| 154 | 161 | export function getAvailableTokenizers() { |
| 155 | 162 | const tokenizerOptions = $('#tokenizer').find('option').toArray(); |
| 156 | 163 | return tokenizerOptions.map(tokenizerOption => ({ |
| 157 | - tokenizerName: tokenizerOption.text, | |
| 158 | 164 | tokenizerId: Number(tokenizerOption.value), |
| 165 | + tokenizerKey: Object.entries(tokenizers).find(([_, value]) => value === Number(tokenizerOption.value))[0].toLocaleLowerCase(), | |
| 166 | + tokenizerName: tokenizerOption.text, | |
| 159 | 167 | })) |
| 160 | 168 | } |
| 161 | 169 | |
| 162 | 170 | /** |
| 163 | 171 | * Selects tokenizer if not already selected. |
| 164 | - * @param {string} tokenizerName Tokenizer name. | |
| 165 | 172 | * @param {number} tokenizerId Tokenizer ID. |
| 166 | 173 | */ |
| 167 | 174 | export function selectTokenizer(tokenizerName, tokenizerId) { |
| 168 | 175 | if (tokenizerId !== power_user.tokenizer) { |
| 169 | - $('#tokenizer').val(tokenizerId).trigger('change'); | |
| 176 | + const tokenizer = getAvailableTokenizers().find(tokenizer => tokenizer.tokenizerId === tokenizerId); | |
| 170 | - toastr.info(`Tokenizer: "${tokenizerName}" selected`); | |
| 177 | + if (!tokenizer) { | |
| 178 | + console.warn('Failed to find tokenizer with id', tokenizerId); | |
| 179 | + return; | |
| 180 | + } | |
| 181 | + $('#tokenizer').val(tokenizer.tokenizerId).trigger('change'); | |
| 182 | + toastr.info(`Tokenizer: "${tokenizer.tokenizerName}" selected`); | |
| 171 | 183 | } |
| 172 | 184 | } |
| 173 | 185 | |
| 174 | 186 | /** |
| 175 | 187 | * Gets the friendly name of the current tokenizer. |
| 176 | 188 | * @param {string} forApi API to get the tokenizer for. Defaults to the main API. |
| 177 | 189 | * @returns { { tokenizerName: string, tokenizerId: number } Tokenizer} Tokenizer info |
| 178 | 190 | */ |
| 179 | 191 | export function getFriendlyTokenizerName(forApi) { |
| 180 | 192 | if (!forApi) { |
| @@ -209,7 +221,9 @@ export function getFriendlyTokenizerName(forApi) { | ||
| 209 | 221 | ? tokenizers.OPENAI |
| 210 | 222 | : tokenizerId; |
| 211 | 223 | |
| 212 | - return { tokenizerName, tokenizerId }; | |
| 224 | + const tokenizerKey = Object.entries(tokenizers).find(([_, value]) => value === tokenizerId)[0].toLocaleLowerCase(); | |
| 225 | + | |
| 226 | + return { tokenizerName, tokenizerKey, tokenizerId }; | |
| 213 | 227 | } |
| 214 | 228 | |
| 215 | 229 | /** |