Merge pull request #3112 from kallewoof/202411-backend-maxctx feature: allow auto-use of max context size given by backend

39c3924b3faa9ed92b2dd35f67c133e54b81db2d

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

Parents: 66a862f797d0a86291925f1238941dbf3b845bea 8e94589f837b51e0399528977c1ec07998c7e34e

public/index.html+10 -4 Viewed

                                     <input id="koboldcpp_api_url_text" class="text_pole wide100p" value="" autocomplete="off" data-server-history="koboldcpp">
                                 </div>
                             </div>
 +                            <div class="flex-container flexFlowColumn marginTopBot5">
 +                                <label data-tg-type="ooba" class="checkbox_label" for="bypass_status_check_textgenerationwebui">
 +                                    <input type="checkbox" id="bypass_status_check_textgenerationwebui" />
 +                                    <span data-i18n="Bypass status check">Bypass status check</span>
 +                                </label>
 +                                <label data-tg-type="koboldcpp, llamacpp" class="checkbox_label" for="context_size_derived">
 +                                    <input type="checkbox" id="context_size_derived" />
 +                                    <span data-i18n="Derive context size from backend">Derive context size from backend</span>
 +                                </label>
 +                            </div>
                             <div class="flex-container">
                                 <div id="api_button_textgenerationwebui" class="api_button menu_button menu_button_icon" type="submit" data-i18n="Connect" data-server-connect="ooba_blocking,vllm,aphrodite,tabby,koboldcpp,ollama,llamacpp,huggingface">Connect</div>
                                 <div data-tg-type="openrouter" class="menu_button menu_button_icon openrouter_authorize" title="Get your OpenRouter API token using OAuth flow. You will be redirected to openrouter.ai" data-i18n="Authorize;[title]Get your OpenRouter API token using OAuth flow. You will be redirected to openrouter.ai">Authorize</div>
                                 <div class="api_loading menu_button menu_button_icon" data-i18n="Cancel">Cancel</div>
                             </div>
 -                            <label data-tg-type="ooba" class="checkbox_label margin-bot-10px" for="bypass_status_check_textgenerationwebui">
 -                                <input type="checkbox" id="bypass_status_check_textgenerationwebui" />
 -                                <span data-i18n="Bypass status check">Bypass status check</span>
 -                            </label>
                         </form>
                         <div class="online_status">
                             <div class="online_status_indicator"></div>

public/script.js+17 -1 Viewed

         const wantsInstructDerivation = (power_user.instruct.enabled && power_user.instruct.derived);
         const wantsContextDerivation = power_user.context_derived;
 +        const wantsContextSize = power_user.context_size_derived;
         const supportsChatTemplate = [textgen_types.KOBOLDCPP, textgen_types.LLAMACPP].includes(textgen_settings.type);
         if (supportsChatTemplate && (wantsInstructDerivation || wantsContextDerivation || wantsContextSize)) {
             const response = await fetch('/api/backends/text-completions/props', {
                 method: 'POST',
                 headers: getRequestHeaders(),

                 const data = await response.json();
                 if (data) {
                     const { chat_template, chat_template_hash } = data;
 +                    if (wantsContextSize && 'default_generation_settings' in data) {
 +                        const backend_max_context = data['default_generation_settings']['n_ctx'];
 +                        const old_value = max_context;
 +                        if (max_context !== backend_max_context) {
 +                            setGenerationParamsFromPreset({ max_length: backend_max_context });
 +                        }
 +                        if (old_value !== max_context) {
 +                            console.log(`Auto-switched max context from ${old_value} to ${max_context}`);
 +                            toastr.info(`${old_value} ⇒ ${max_context}`, 'Context Size Changed');
 +                        }
 +                    }
                     console.log(`We have chat template ${chat_template.split('\n')[0]}...`);
                     const templates = await deriveTemplatesFromChatTemplate(chat_template, chat_template_hash);
                     if (templates) {

     });
+}
 +/**
 + * Sets the generation parameters from a preset object.
 + * @param {{ genamt?: number, max_length?: number }} preset Preset object
 + */
 export function setGenerationParamsFromPreset(preset) {
     const needsUnlock = (preset.max_length ?? max_context) > MAX_CONTEXT_DEFAULT || (preset.genamt ?? amount_gen) > MAX_RESPONSE_DEFAULT;
     $('#max_context_unlocked').prop('checked', needsUnlock).trigger('change');

public/scripts/power-user.js+12 -0 Viewed

     },
     context_derived: false,
 +    context_size_derived: false,
     sysprompt: {
         enabled: true,

     $('#example_messages_behavior').val(getExampleMessagesBehavior());
     $(`#example_messages_behavior option[value="${getExampleMessagesBehavior()}"]`).prop('selected', true);
     $('#context_derived').parent().find('i').toggleClass('toggleEnabled', !!power_user.context_derived);
 +    $('#context_size_derived').prop('checked', !!power_user.context_size_derived);
     $('#console_log_prompts').prop('checked', power_user.console_log_prompts);
     $('#request_token_probabilities').prop('checked', power_user.request_token_probabilities);

         $('#context_derived').parent().find('i').toggleClass('toggleEnabled', !!power_user.context_derived);
     });
 +    $('#context_size_derived').on('input', function () {
 +        const value = !!$(this).prop('checked');
 +        power_user.context_size_derived = value;
 +        saveSettingsDebounced();
 +    });
++
 +    $('#context_size_derived').on('change', function () {
 +        $('#context_size_derived').prop('checked', !!power_user.context_size_derived);
 +    });
++
     $('#always-force-name2-checkbox').change(function () {
         power_user.always_force_name2 = !!$(this).prop('checked');
         saveSettingsDebounced();

src/endpoints/backends/text-completions.js+11 -11 Viewed

         if (!modelsReply.ok) {
             console.log('Models endpoint is offline.');
             return response.statussendStatus(400);
+        }
         /** @type {any} */

         if (!Array.isArray(data.data)) {
             console.log('Models response is not an array.');
             return response.statussendStatus(400);
+        }
         const modelIds = data.data.map(x => x.id);

         return response.send({ result, data: data.data });
     } catch (error) {
         console.error(error);
         return response.statussendStatus(500);
+    }
 });

         const propsReply = await fetch(propsUrl, args);
         if (!propsReply.ok) {
             return response.statussendStatus(400);
+        }
         /** @type {any} */

         return response.send(props);
     } catch (error) {
         console.error(error);
         return response.statussendStatus(500);
+    }
 });

         return response.send({ ok: true });
     } catch (error) {
         console.error(error);
         return response.statussendStatus(500);
+    }
 });

         return response.send({ caption });
     } catch (error) {
         console.error(error);
         return response.statussendStatus(500);
+    }
 });

     } catch (error) {
         console.error(error);
         return response.statussendStatus(500);
+    }
 });

     } catch (error) {
         console.error(error);
         return response.statussendStatus(500);
+    }
 });

     } catch (error) {
         console.error(error);
         return response.statussendStatus(500);
+    }
 });

         return response.send({ ok: true });
     } catch (error) {
         console.error(error);
         return response.statussendStatus(500);
+    }
 });