expose tokenizers & getTextTokens in getContext

0e5100180b38c47049cb51ec690e9de87f2ece0d

Succubyss <87207237+Succubyss@users.noreply.github.com>

public/scripts/st-context.js+3 -1 Viewed

 import { SlashCommandParser } from './slash-commands/SlashCommandParser.js';
 import { tag_map, tags } from './tags.js';
 import { textgenerationwebui_settings } from './textgen-settings.js';
-import { getTokenCount, getTokenCountAsync, getTokenizerModel } from './tokenizers.js';
+import { tokenizers, getTextTokens, getTokenCount, getTokenCountAsync, getTokenizerModel } from './tokenizers.js';
 import { ToolManager } from './tool-calling.js';
 import { timestampToMoment } from './utils.js';

         sendStreamingRequest,
         sendGenerationRequest,
         stopGeneration,
+        tokenizers,
+        getTextTokens,
         /** @deprecated Use getTokenCountAsync instead */
         getTokenCount,
         getTokenCountAsync,