Move text completion settings to respective module (#4239)

c51ce1d018a946018aeff1673414ba4b04ac20a2

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: 3d488a5e7bc1b0951060a28a93262c503eae5bea
2 files changed, +187 -200Ignore whitespace
public/script.js+5 -196
@@ -30,10 +30,9 @@ import {
3030 generateTextGenWithStreaming,
3131 getTextGenGenerationData,
3232 textgen_types,
33- getTextGenServer,
34- validateTextGenUrl,
3533 parseTextgenLogprobs,
3634 parseTabbyLogprobs,
35+ initTextGenSettings,
3736} from './scripts/textgen-settings.js';
3837
3938import {
@@ -206,12 +205,7 @@ import {
206205 tag_import_setting,
207206 applyCharacterTagsToMessageDivs,
208207} from './scripts/tags.js';
209-import {
208+import { initSecrets, readSecretState } from './scripts/secrets.js';
210- SECRET_KEYS,
211- initSecrets,
212- readSecretState,
213- writeSecret,
214-} from './scripts/secrets.js';
215209import { EventEmitter } from './lib/eventemitter.js';
216210import { markdownExclusionExt } from './scripts/showdown-exclusion.js';
217211import { markdownUnderscoreExt } from './scripts/showdown-underscore.js';
@@ -227,14 +221,13 @@ import {
227221 formatInstructModePrompt,
228222 formatInstructModeExamples,
229223 getInstructStoppingSequences,
230- autoSelectInstructPreset,
231224 formatInstructModeSystemPrompt,
232225 selectInstructPreset,
233226 instruct_presets,
234227 selectContextPreset,
235228} from './scripts/instruct-mode.js';
236229import { initLocales, t } from './scripts/i18n.js';
237230import { getFriendlyTokenizerName, getTokenCount, getTokenCountAsync, initTokenizers, saveTokenCache, TOKENIZER_SUPPORTED_KEY } from './scripts/tokenizers.js';
238231import {
239232 user_avatar,
240233 getUserAvatars,
@@ -249,20 +242,7 @@ import {
249242import { getBackgrounds, initBackgrounds, loadBackgroundSettings, background_settings } from './scripts/backgrounds.js';
250243import { hideLoader, showLoader } from './scripts/loader.js';
251244import { BulkEditOverlay, CharacterContextMenu } from './scripts/BulkEditOverlay.js';
252-import {
245+import { initTextGenModels } from './scripts/textgen-models.js';
253- loadFeatherlessModels,
254- loadMancerModels,
255- loadOllamaModels,
256- loadTogetherAIModels,
257- loadInfermaticAIModels,
258- loadOpenRouterModels,
259- loadVllmModels,
260- loadAphroditeModels,
261- loadDreamGenModels,
262- initTextGenModels,
263- loadTabbyModels,
264- loadGenericModels,
265-} from './scripts/textgen-models.js';
266246import { appendFileContent, hasPendingFileAttachment, populateFileAttachment, decodeStyleTags, encodeStyleTags, isExternalMediaAllowed, preserveNeutralChat, restoreNeutralChat, formatCreatorNotes, initChatUtilities, addDOMPurifyHooks } from './scripts/chats.js';
267247import { getPresetManager, initPresetManager } from './scripts/preset-manager.js';
268248import { evaluateMacros, getLastMessageId, initMacros } from './scripts/macros.js';
@@ -290,7 +270,6 @@ import { applyBrowserFixes } from './scripts/browser-fixes.js';
290270import { initServerHistory } from './scripts/server-history.js';
291271import { initSettingsSearch } from './scripts/setting-search.js';
292272import { initBulkEdit } from './scripts/bulk-edit.js';
293-import { deriveTemplatesFromChatTemplate } from './scripts/chat-templates.js';
294273import { getContext } from './scripts/st-context.js';
295274import { extractReasoningFromData, initReasoning, parseReasoningInSwipes, PromptReasoning, ReasoningHandler, removeReasoningFromString, updateReasoningUI } from './scripts/reasoning.js';
296275import { accountStorage } from './scripts/util/AccountStorage.js';
@@ -880,6 +859,7 @@ async function firstLoadInit() {
880859 initDefaultSlashCommands();
881860 initTextGenModels();
882861 initOpenAI();
862+ initTextGenSettings();
883863 initKoboldSettings();
884864 initNovelAISettings();
885865 initSystemPrompts();
@@ -993,145 +973,6 @@ async function getStatusHorde() {
993973 return resultCheckStatus();
994974}
995975
996-async function getStatusTextgen() {
997- const url = '/api/backends/text-completions/status';
998-
999- const endpoint = getTextGenServer();
1000-
1001- if (!endpoint) {
1002- console.warn('No endpoint for status check');
1003- setOnlineStatus('no_connection');
1004- return resultCheckStatus();
1005- }
1006-
1007- if ([textgen_types.GENERIC, textgen_types.OOBA].includes(textgen_settings.type) && textgen_settings.bypass_status_check) {
1008- setOnlineStatus(t`Status check bypassed`);
1009- return resultCheckStatus();
1010- }
1011-
1012- try {
1013- const response = await fetch(url, {
1014- method: 'POST',
1015- headers: getRequestHeaders(),
1016- body: JSON.stringify({
1017- api_server: endpoint,
1018- api_type: textgen_settings.type,
1019- }),
1020- signal: abortStatusCheck.signal,
1021- });
1022-
1023- const data = await response.json();
1024-
1025- if (textgen_settings.type === textgen_types.MANCER) {
1026- loadMancerModels(data?.data);
1027- setOnlineStatus(textgen_settings.mancer_model);
1028- } else if (textgen_settings.type === textgen_types.TOGETHERAI) {
1029- loadTogetherAIModels(data?.data);
1030- setOnlineStatus(textgen_settings.togetherai_model);
1031- } else if (textgen_settings.type === textgen_types.OLLAMA) {
1032- loadOllamaModels(data?.data);
1033- setOnlineStatus(textgen_settings.ollama_model || t`Connected`);
1034- } else if (textgen_settings.type === textgen_types.INFERMATICAI) {
1035- loadInfermaticAIModels(data?.data);
1036- setOnlineStatus(textgen_settings.infermaticai_model);
1037- } else if (textgen_settings.type === textgen_types.DREAMGEN) {
1038- loadDreamGenModels(data?.data);
1039- setOnlineStatus(textgen_settings.dreamgen_model);
1040- } else if (textgen_settings.type === textgen_types.OPENROUTER) {
1041- loadOpenRouterModels(data?.data);
1042- setOnlineStatus(textgen_settings.openrouter_model);
1043- } else if (textgen_settings.type === textgen_types.VLLM) {
1044- loadVllmModels(data?.data);
1045- setOnlineStatus(textgen_settings.vllm_model);
1046- } else if (textgen_settings.type === textgen_types.APHRODITE) {
1047- loadAphroditeModels(data?.data);
1048- setOnlineStatus(textgen_settings.aphrodite_model);
1049- } else if (textgen_settings.type === textgen_types.FEATHERLESS) {
1050- loadFeatherlessModels(data?.data);
1051- setOnlineStatus(textgen_settings.featherless_model);
1052- } else if (textgen_settings.type === textgen_types.TABBY) {
1053- loadTabbyModels(data?.data);
1054- setOnlineStatus(textgen_settings.tabby_model || data?.result);
1055- } else if (textgen_settings.type === textgen_types.GENERIC) {
1056- loadGenericModels(data?.data);
1057- setOnlineStatus(textgen_settings.generic_model || data?.result || t`Connected`);
1058- } else {
1059- setOnlineStatus(data?.result);
1060- }
1061-
1062- if (!online_status) {
1063- setOnlineStatus('no_connection');
1064- }
1065-
1066- power_user.chat_template_hash = '';
1067-
1068- // Determine instruct mode preset
1069- const autoSelected = autoSelectInstructPreset(online_status);
1070-
1071- const supportsTokenization = response.headers.get('x-supports-tokenization') === 'true';
1072- supportsTokenization ? sessionStorage.setItem(TOKENIZER_SUPPORTED_KEY, 'true') : sessionStorage.removeItem(TOKENIZER_SUPPORTED_KEY);
1073-
1074- const wantsInstructDerivation = !autoSelected && (power_user.instruct.enabled && power_user.instruct_derived);
1075- const wantsContextDerivation = !autoSelected && power_user.context_derived;
1076- const wantsContextSize = power_user.context_size_derived;
1077- const supportsChatTemplate = [textgen_types.KOBOLDCPP, textgen_types.LLAMACPP].includes(textgen_settings.type);
1078-
1079- if (supportsChatTemplate && (wantsInstructDerivation || wantsContextDerivation || wantsContextSize)) {
1080- const response = await fetch('/api/backends/text-completions/props', {
1081- method: 'POST',
1082- headers: getRequestHeaders(),
1083- body: JSON.stringify({
1084- api_server: endpoint,
1085- api_type: textgen_settings.type,
1086- }),
1087- });
1088-
1089- if (response.ok) {
1090- const data = await response.json();
1091- if (data) {
1092- const { chat_template, chat_template_hash } = data;
1093- power_user.chat_template_hash = chat_template_hash;
1094-
1095- if (wantsContextSize && 'default_generation_settings' in data) {
1096- const backend_max_context = data['default_generation_settings']['n_ctx'];
1097- const old_value = max_context;
1098- if (max_context !== backend_max_context) {
1099- setGenerationParamsFromPreset({ max_length: backend_max_context });
1100- }
1101- if (old_value !== max_context) {
1102- console.log(`Auto-switched max context from ${old_value} to ${max_context}`);
1103- toastr.info(`${old_value} ⇒ ${max_context}`, 'Context Size Changed');
1104- }
1105- }
1106- console.log(`We have chat template ${chat_template.split('\n')[0]}...`);
1107- const { context, instruct } = await deriveTemplatesFromChatTemplate(chat_template, chat_template_hash);
1108- if (wantsContextDerivation && context) {
1109- selectContextPreset(context, { isAuto: true });
1110- }
1111- if (wantsInstructDerivation && power_user.instruct.enabled && instruct) {
1112- selectInstructPreset(instruct, { isAuto: true });
1113- }
1114- }
1115- }
1116- }
1117-
1118- // We didn't get a 200 status code, but the endpoint has an explanation. Which means it DID connect, but I digress.
1119- if (online_status === 'no_connection' && data.response) {
1120- toastr.error(data.response, t`API Error`, { timeOut: 5000, preventDuplicates: true });
1121- }
1122- } catch (err) {
1123- if (err instanceof AbortReason) {
1124- console.info('Status check aborted.', err.reason);
1125- } else {
1126- console.error('Error getting status', err);
1127-
1128- }
1129- setOnlineStatus('no_connection');
1130- }
1131-
1132- return resultCheckStatus();
1133-}
1134-
1135976export function startStatusLoading() {
1136977 $('.api_loading').show();
1137978 $('.api_button').addClass('disabled');
@@ -10636,38 +10477,6 @@ jQuery(async function () {
1063610477 }
1063710478 });
1063810479
10639- $('#api_button_textgenerationwebui').on('click', async function (e) {
10640- const keys = [
10641- { id: 'api_key_mancer', secret: SECRET_KEYS.MANCER },
10642- { id: 'api_key_vllm', secret: SECRET_KEYS.VLLM },
10643- { id: 'api_key_aphrodite', secret: SECRET_KEYS.APHRODITE },
10644- { id: 'api_key_tabby', secret: SECRET_KEYS.TABBY },
10645- { id: 'api_key_togetherai', secret: SECRET_KEYS.TOGETHERAI },
10646- { id: 'api_key_ooba', secret: SECRET_KEYS.OOBA },
10647- { id: 'api_key_infermaticai', secret: SECRET_KEYS.INFERMATICAI },
10648- { id: 'api_key_dreamgen', secret: SECRET_KEYS.DREAMGEN },
10649- { id: 'api_key_openrouter-tg', secret: SECRET_KEYS.OPENROUTER },
10650- { id: 'api_key_koboldcpp', secret: SECRET_KEYS.KOBOLDCPP },
10651- { id: 'api_key_llamacpp', secret: SECRET_KEYS.LLAMACPP },
10652- { id: 'api_key_featherless', secret: SECRET_KEYS.FEATHERLESS },
10653- { id: 'api_key_huggingface', secret: SECRET_KEYS.HUGGINGFACE },
10654- { id: 'api_key_generic', secret: SECRET_KEYS.GENERIC },
10655- ];
10656-
10657- for (const key of keys) {
10658- const keyValue = String($(`#${key.id}`).val()).trim();
10659- if (keyValue.length) {
10660- await writeSecret(key.secret, keyValue);
10661- }
10662- }
10663-
10664- validateTextGenUrl();
10665- startStatusLoading();
10666- main_api = 'textgenerationwebui';
10667- saveSettingsDebounced();
10668- getStatusTextgen();
10669- });
10670-
1067110480
1067210481 const button = $('#options_button');
1067310482 const menu = $('#options');
public/scripts/textgen-settings.js+182 -4
@@ -1,22 +1,30 @@
11import {
2+ abortStatusCheck,
23 eventSource,
34 event_types,
45 getRequestHeaders,
56 getStoppingStrings,
67 main_api,
78 max_context,
9+ online_status,
10+ resultCheckStatus,
811 saveSettingsDebounced,
912 setGenerationParamsFromPreset,
1013 setOnlineStatus,
14+ startStatusLoading,
1115 substituteParams,
1216} from '../script.js';
17+import { deriveTemplatesFromChatTemplate } from './chat-templates.js';
1318import { t } from './i18n.js';
19+import { autoSelectInstructPreset, selectContextPreset, selectInstructPreset } from './instruct-mode.js';
1420import { BIAS_CACHE, createNewLogitBiasEntry, displayLogitBias, getLogitBiasListResult } from './logit-bias.js';
1521
1622import { power_user, registerDebugFunction } from './power-user.js';
23+import { SECRET_KEYS, writeSecret } from './secrets.js';
1724import { getEventSourceStream } from './sse-stream.js';
1825import { getCurrentDreamGenModelTokenizer, getCurrentOpenRouterModelTokenizer, loadAphroditeModels, loadDreamGenModels, loadFeatherlessModels, loadGenericModels, loadInfermaticAIModels, loadMancerModels, loadOllamaModels, loadOpenRouterModels, loadTabbyModels, loadTogetherAIModels, loadVllmModels } from './textgen-models.js';
1926import { ENCODE_TOKENIZERS, TEXTGEN_TOKENIZERS, TOKENIZER_SUPPORTED_KEY, getTextTokens, tokenizers } from './tokenizers.js';
27+import { AbortReason } from './util/AbortReason.js';
2028import { getSortableDelay, onlyUnique, arraysEqual } from './utils.js';
2129
2230export const textgen_types = {
@@ -621,7 +629,146 @@ function sortAphroditeItemsByOrder(orderArray) {
621629 });
622630}
623631
624632jQuery(async function getStatusTextgen() {
633+ const url = '/api/backends/text-completions/status';
634+
635+ const endpoint = getTextGenServer();
636+
637+ if (!endpoint) {
638+ console.warn('No endpoint for status check');
639+ setOnlineStatus('no_connection');
640+ return resultCheckStatus();
641+ }
642+
643+ if ([textgen_types.GENERIC, textgen_types.OOBA].includes(settings.type) && settings.bypass_status_check) {
644+ setOnlineStatus(t`Status check bypassed`);
645+ return resultCheckStatus();
646+ }
647+
648+ try {
649+ const response = await fetch(url, {
650+ method: 'POST',
651+ headers: getRequestHeaders(),
652+ body: JSON.stringify({
653+ api_server: endpoint,
654+ api_type: settings.type,
655+ }),
656+ signal: abortStatusCheck.signal,
657+ });
658+
659+ const data = await response.json();
660+
661+ if (settings.type === textgen_types.MANCER) {
662+ loadMancerModels(data?.data);
663+ setOnlineStatus(settings.mancer_model);
664+ } else if (settings.type === textgen_types.TOGETHERAI) {
665+ loadTogetherAIModels(data?.data);
666+ setOnlineStatus(settings.togetherai_model);
667+ } else if (settings.type === textgen_types.OLLAMA) {
668+ loadOllamaModels(data?.data);
669+ setOnlineStatus(settings.ollama_model || t`Connected`);
670+ } else if (settings.type === textgen_types.INFERMATICAI) {
671+ loadInfermaticAIModels(data?.data);
672+ setOnlineStatus(settings.infermaticai_model);
673+ } else if (settings.type === textgen_types.DREAMGEN) {
674+ loadDreamGenModels(data?.data);
675+ setOnlineStatus(settings.dreamgen_model);
676+ } else if (settings.type === textgen_types.OPENROUTER) {
677+ loadOpenRouterModels(data?.data);
678+ setOnlineStatus(settings.openrouter_model);
679+ } else if (settings.type === textgen_types.VLLM) {
680+ loadVllmModels(data?.data);
681+ setOnlineStatus(settings.vllm_model);
682+ } else if (settings.type === textgen_types.APHRODITE) {
683+ loadAphroditeModels(data?.data);
684+ setOnlineStatus(settings.aphrodite_model);
685+ } else if (settings.type === textgen_types.FEATHERLESS) {
686+ loadFeatherlessModels(data?.data);
687+ setOnlineStatus(settings.featherless_model);
688+ } else if (settings.type === textgen_types.TABBY) {
689+ loadTabbyModels(data?.data);
690+ setOnlineStatus(settings.tabby_model || data?.result);
691+ } else if (settings.type === textgen_types.GENERIC) {
692+ loadGenericModels(data?.data);
693+ setOnlineStatus(settings.generic_model || data?.result || t`Connected`);
694+ } else {
695+ setOnlineStatus(data?.result);
696+ }
697+
698+ if (!online_status) {
699+ setOnlineStatus('no_connection');
700+ }
701+
702+ power_user.chat_template_hash = '';
703+
704+ // Determine instruct mode preset
705+ const autoSelected = autoSelectInstructPreset(online_status);
706+
707+ const supportsTokenization = response.headers.get('x-supports-tokenization') === 'true';
708+ supportsTokenization ? sessionStorage.setItem(TOKENIZER_SUPPORTED_KEY, 'true') : sessionStorage.removeItem(TOKENIZER_SUPPORTED_KEY);
709+
710+ const wantsInstructDerivation = !autoSelected && (power_user.instruct.enabled && power_user.instruct_derived);
711+ const wantsContextDerivation = !autoSelected && power_user.context_derived;
712+ const wantsContextSize = power_user.context_size_derived;
713+ const supportsChatTemplate = [textgen_types.KOBOLDCPP, textgen_types.LLAMACPP].includes(settings.type);
714+
715+ if (supportsChatTemplate && (wantsInstructDerivation || wantsContextDerivation || wantsContextSize)) {
716+ const response = await fetch('/api/backends/text-completions/props', {
717+ method: 'POST',
718+ headers: getRequestHeaders(),
719+ body: JSON.stringify({
720+ api_server: endpoint,
721+ api_type: settings.type,
722+ }),
723+ });
724+
725+ if (response.ok) {
726+ const data = await response.json();
727+ if (data) {
728+ const { chat_template, chat_template_hash } = data;
729+ power_user.chat_template_hash = chat_template_hash;
730+
731+ if (wantsContextSize && 'default_generation_settings' in data) {
732+ const backend_max_context = data['default_generation_settings']['n_ctx'];
733+ const old_value = max_context;
734+ if (max_context !== backend_max_context) {
735+ setGenerationParamsFromPreset({ max_length: backend_max_context });
736+ }
737+ if (old_value !== max_context) {
738+ console.log(`Auto-switched max context from ${old_value} to ${max_context}`);
739+ toastr.info(`${old_value} ⇒ ${max_context}`, 'Context Size Changed');
740+ }
741+ }
742+ console.log(`We have chat template ${chat_template.split('\n')[0]}...`);
743+ const { context, instruct } = await deriveTemplatesFromChatTemplate(chat_template, chat_template_hash);
744+ if (wantsContextDerivation && context) {
745+ selectContextPreset(context, { isAuto: true });
746+ }
747+ if (wantsInstructDerivation && power_user.instruct.enabled && instruct) {
748+ selectInstructPreset(instruct, { isAuto: true });
749+ }
750+ }
751+ }
752+ }
753+
754+ // We didn't get a 200 status code, but the endpoint has an explanation. Which means it DID connect, but I digress.
755+ if (online_status === 'no_connection' && data.response) {
756+ toastr.error(data.response, t`API Error`, { timeOut: 5000, preventDuplicates: true });
757+ }
758+ } catch (err) {
759+ if (err instanceof AbortReason) {
760+ console.info('Status check aborted.', err.reason);
761+ } else {
762+ console.error('Error getting status', err);
763+
764+ }
765+ setOnlineStatus('no_connection');
766+ }
767+
768+ return resultCheckStatus();
769+}
770+
771+export function initTextGenSettings() {
625772 $('#send_banned_tokens_textgenerationwebui').on('change', function () {
626773 const checked = !!$(this).prop('checked');
627774 toggleBannedStringsKillSwitch(checked,
@@ -877,7 +1024,38 @@ jQuery(function () {
8771024
8781025 saveSettingsDebounced();
8791026 });
880-});
1027+
1028+ $('#api_button_textgenerationwebui').on('click', async function (e) {
1029+ const keys = [
1030+ { id: 'api_key_mancer', secret: SECRET_KEYS.MANCER },
1031+ { id: 'api_key_vllm', secret: SECRET_KEYS.VLLM },
1032+ { id: 'api_key_aphrodite', secret: SECRET_KEYS.APHRODITE },
1033+ { id: 'api_key_tabby', secret: SECRET_KEYS.TABBY },
1034+ { id: 'api_key_togetherai', secret: SECRET_KEYS.TOGETHERAI },
1035+ { id: 'api_key_ooba', secret: SECRET_KEYS.OOBA },
1036+ { id: 'api_key_infermaticai', secret: SECRET_KEYS.INFERMATICAI },
1037+ { id: 'api_key_dreamgen', secret: SECRET_KEYS.DREAMGEN },
1038+ { id: 'api_key_openrouter-tg', secret: SECRET_KEYS.OPENROUTER },
1039+ { id: 'api_key_koboldcpp', secret: SECRET_KEYS.KOBOLDCPP },
1040+ { id: 'api_key_llamacpp', secret: SECRET_KEYS.LLAMACPP },
1041+ { id: 'api_key_featherless', secret: SECRET_KEYS.FEATHERLESS },
1042+ { id: 'api_key_huggingface', secret: SECRET_KEYS.HUGGINGFACE },
1043+ { id: 'api_key_generic', secret: SECRET_KEYS.GENERIC },
1044+ ];
1045+
1046+ for (const key of keys) {
1047+ const keyValue = String($(`#${key.id}`).val()).trim();
1048+ if (keyValue.length) {
1049+ await writeSecret(key.secret, keyValue);
1050+ }
1051+ }
1052+
1053+ validateTextGenUrl();
1054+ startStatusLoading();
1055+ saveSettingsDebounced();
1056+ getStatusTextgen();
1057+ });
1058+}
8811059
8821060function showTypeSpecificControls(type) {
8831061 $('[data-tg-type]').each(function () {