expose tokenizers & getTextTokens in getContext
| @@ -63,7 +63,7 @@ import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from ' | ||
| 63 | 63 | import { SlashCommandParser } from './slash-commands/SlashCommandParser.js'; |
| 64 | 64 | import { tag_map, tags } from './tags.js'; |
| 65 | 65 | import { textgenerationwebui_settings } from './textgen-settings.js'; |
| 66 | 66 | import { tokenizers, getTextTokens, getTokenCount, getTokenCountAsync, getTokenizerModel } from './tokenizers.js'; |
| 67 | 67 | import { ToolManager } from './tool-calling.js'; |
| 68 | 68 | import { timestampToMoment } from './utils.js'; |
| 69 | 69 | |
| @@ -95,6 +95,8 @@ export function getContext() { | ||
| 95 | 95 | sendStreamingRequest, |
| 96 | 96 | sendGenerationRequest, |
| 97 | 97 | stopGeneration, |
| 98 | + tokenizers, | |
| 99 | + getTextTokens, | |
| 98 | 100 | /** @deprecated Use getTokenCountAsync instead */ |
| 99 | 101 | getTokenCount, |
| 100 | 102 | getTokenCountAsync, |