feature: allow auto-use of max context size given by backend

4988f22e94b5ac4506032213ef594245fb40de4b

Karl-Johan Alm <karljohan-alm@garage.co.jp>

Signed
Parents: 0143ce199ae7e0ed687821d3b20804f9241e0133
3 files changed, +28 -1Ignore whitespace
public/index.html+4 -0
@@ -220,6 +220,10 @@
220 <div data-preset-manager-delete="textgenerationwebui" class="margin0 menu_button_icon menu_button" title="Delete the preset" data-i18n="[title]Delete the preset">220 <div data-preset-manager-delete="textgenerationwebui" class="margin0 menu_button_icon menu_button" title="Delete the preset" data-i18n="[title]Delete the preset">
221 <i class="fa-fw fa-solid fa-trash-can"></i>221 <i class="fa-fw fa-solid fa-trash-can"></i>
222 </div>222 </div>
223 <label for="context_size_derived" class="checkbox_label flex1" title="Use backend provided context size, when available." data-i18n="[title]context_size_derived">
224 <input id="context_size_derived" type="checkbox" style="display:none;" />
225 <small><i class="fa-solid fa-bolt menu_button margin0"></i></small>
226 </label>
223 </div>227 </div>
224 </div>228 </div>
225 <div class="flex-container flexNoGap">229 <div class="flex-container flexNoGap">
public/script.js+12 -1
@@ -1238,8 +1238,9 @@ async function getStatusTextgen() {
12381238
1239 const wantsInstructDerivation = (power_user.instruct.enabled && power_user.instruct.derived);1239 const wantsInstructDerivation = (power_user.instruct.enabled && power_user.instruct.derived);
1240 const wantsContextDerivation = power_user.context_derived;1240 const wantsContextDerivation = power_user.context_derived;
1241 const wantsContextSize = power_user.context_size_derived;
1241 const supportsChatTemplate = [textgen_types.KOBOLDCPP, textgen_types.LLAMACPP].includes(textgen_settings.type);1242 const supportsChatTemplate = [textgen_types.KOBOLDCPP, textgen_types.LLAMACPP].includes(textgen_settings.type);
1242 if (supportsChatTemplate && (wantsInstructDerivation || wantsContextDerivation)) {1243 if (supportsChatTemplate && (wantsInstructDerivation || wantsContextDerivation || wantsContextSize)) {
1243 const response = await fetch('/api/backends/text-completions/props', {1244 const response = await fetch('/api/backends/text-completions/props', {
1244 method: 'POST',1245 method: 'POST',
1245 headers: getRequestHeaders(),1246 headers: getRequestHeaders(),
@@ -1253,6 +1254,16 @@ async function getStatusTextgen() {
1253 const data = await response.json();1254 const data = await response.json();
1254 if (data) {1255 if (data) {
1255 const { chat_template, chat_template_hash } = data;1256 const { chat_template, chat_template_hash } = data;
1257 if (wantsContextSize && "default_generation_settings" in data) {
1258 const backend_max_context = data["default_generation_settings"]["n_ctx"];
1259 if (max_context !== backend_max_context) {
1260 console.log(`Auto-switching max context from ${max_context} to ${backend_max_context}`);
1261 toastr.info(`Context Size Changed: ${max_context} ⇒ ${backend_max_context}`);
1262 max_context = backend_max_context;
1263 $('#max_context').val(max_context);
1264 $('#max_context_counter').val(max_context);
1265 }
1266 }
1256 console.log(`We have chat template ${chat_template.split('\n')[0]}...`);1267 console.log(`We have chat template ${chat_template.split('\n')[0]}...`);
1257 const templates = await deriveTemplatesFromChatTemplate(chat_template, chat_template_hash);1268 const templates = await deriveTemplatesFromChatTemplate(chat_template, chat_template_hash);
1258 if (templates) {1269 if (templates) {
public/scripts/power-user.js+12 -0
@@ -245,6 +245,7 @@ let power_user = {
245 },245 },
246246
247 context_derived: false,247 context_derived: false,
248 context_size_derived: false,
248249
249 sysprompt: {250 sysprompt: {
250 enabled: true,251 enabled: true,
@@ -1481,6 +1482,7 @@ async function loadPowerUserSettings(settings, data) {
1481 $('#example_messages_behavior').val(getExampleMessagesBehavior());1482 $('#example_messages_behavior').val(getExampleMessagesBehavior());
1482 $(`#example_messages_behavior option[value="${getExampleMessagesBehavior()}"]`).prop('selected', true);1483 $(`#example_messages_behavior option[value="${getExampleMessagesBehavior()}"]`).prop('selected', true);
1483 $('#context_derived').parent().find('i').toggleClass('toggleEnabled', !!power_user.context_derived);1484 $('#context_derived').parent().find('i').toggleClass('toggleEnabled', !!power_user.context_derived);
1485 $('#context_size_derived').parent().find('i').toggleClass('toggleEnabled', !!power_user.context_size_derived);
14841486
1485 $('#console_log_prompts').prop('checked', power_user.console_log_prompts);1487 $('#console_log_prompts').prop('checked', power_user.console_log_prompts);
1486 $('#request_token_probabilities').prop('checked', power_user.request_token_probabilities);1488 $('#request_token_probabilities').prop('checked', power_user.request_token_probabilities);
@@ -3076,6 +3078,16 @@ $(document).ready(() => {
3076 $('#context_derived').parent().find('i').toggleClass('toggleEnabled', !!power_user.context_derived);3078 $('#context_derived').parent().find('i').toggleClass('toggleEnabled', !!power_user.context_derived);
3077 });3079 });
30783080
3081 $('#context_size_derived').on('input', function () {
3082 const value = !!$(this).prop('checked');
3083 power_user.context_size_derived = value;
3084 saveSettingsDebounced();
3085 });
3086
3087 $('#context_size_derived').on('change', function () {
3088 $('#context_size_derived').parent().find('i').toggleClass('toggleEnabled', !!power_user.context_size_derived);
3089 });
3090
3079 $('#always-force-name2-checkbox').change(function () {3091 $('#always-force-name2-checkbox').change(function () {
3080 power_user.always_force_name2 = !!$(this).prop('checked');3092 power_user.always_force_name2 = !!$(this).prop('checked');
3081 saveSettingsDebounced();3093 saveSettingsDebounced();