OpenRouter: Add control for middle-out transform Closes #3033

7adc6d38e29672ac27acebf1d588f61be97d68d1

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: 09dd9762f78d4758ed89a5dfa525d2f4471d8fc3
3 files changed, +50 -1Showing whitespace changes
public/index.html+13 -0
@@ -651,6 +651,19 @@
651651 <input type="number" id="n_openai" name="n_openai" class="text_pole" min="1" value="1">
652652 </div>
653653 </div>
654+ <div data-source="openrouter" class="range-block">
655+ <div class="range-block-title" title="Allow compressing requests by removing messages from the middle of the prompt.">
656+ <span data-i18n="Middle-out Transform">Middle-out Transform</span>
657+ <a href="https://openrouter.ai/docs/transforms" target="_blank" rel="noreferrer noopener" class="note-link-span fa-solid fa-circle-question"></a>
658+ </div>
659+ <div class="wide100p">
660+ <select id="openrouter_middleout" class="text_pole">
661+ <option value="auto" data-i18n="Auto">Auto</option>
662+ <option value="on" data-i18n="Allow">Allow</option>
663+ <option value="off" data-i18n="Forbid">Forbid</option>
664+ </select>
665+ </div>
666+ </div>
654667 <div data-source="openrouter">
655668 Max prompt cost: <span id="openrouter_max_prompt_cost">Unknown</span>
656669 </div>
public/scripts/openai.js+18 -0
@@ -207,6 +207,12 @@ const custom_prompt_post_processing_types = {
207207 STRICT: 'strict',
208208};
209209
210+const openrouter_middleout_types = {
211+ AUTO: 'auto',
212+ ON: 'on',
213+ OFF: 'off',
214+};
215+
210216const sensitiveFields = [
211217 'reverse_proxy',
212218 'proxy_password',
@@ -267,6 +273,7 @@ const default_settings = {
267273 openrouter_sort_models: 'alphabetically',
268274 openrouter_providers: [],
269275 openrouter_allow_fallbacks: true,
276+ openrouter_middleout: openrouter_middleout_types.ON,
270277 jailbreak_system: false,
271278 reverse_proxy: '',
272279 chat_completion_source: chat_completion_sources.OPENAI,
@@ -343,6 +350,7 @@ const oai_settings = {
343350 openrouter_sort_models: 'alphabetically',
344351 openrouter_providers: [],
345352 openrouter_allow_fallbacks: true,
353+ openrouter_middleout: openrouter_middleout_types.ON,
346354 jailbreak_system: false,
347355 reverse_proxy: '',
348356 chat_completion_source: chat_completion_sources.OPENAI,
@@ -1920,6 +1928,7 @@ async function sendOpenAIRequest(type, messages, signal) {
19201928 generate_data['use_fallback'] = oai_settings.openrouter_use_fallback;
19211929 generate_data['provider'] = oai_settings.openrouter_providers;
19221930 generate_data['allow_fallbacks'] = oai_settings.openrouter_allow_fallbacks;
1931+ generate_data['middleout'] = oai_settings.openrouter_middleout;
19231932
19241933 if (isTextCompletion) {
19251934 generate_data['stop'] = getStoppingStrings(isImpersonate, isContinue);
@@ -3022,6 +3031,7 @@ function loadOpenAISettings(data, settings) {
30223031 oai_settings.openrouter_sort_models = settings.openrouter_sort_models ?? default_settings.openrouter_sort_models;
30233032 oai_settings.openrouter_use_fallback = settings.openrouter_use_fallback ?? default_settings.openrouter_use_fallback;
30243033 oai_settings.openrouter_allow_fallbacks = settings.openrouter_allow_fallbacks ?? default_settings.openrouter_allow_fallbacks;
3034+ oai_settings.openrouter_middleout = settings.openrouter_middleout ?? default_settings.openrouter_middleout;
30253035 oai_settings.ai21_model = settings.ai21_model ?? default_settings.ai21_model;
30263036 oai_settings.mistralai_model = settings.mistralai_model ?? default_settings.mistralai_model;
30273037 oai_settings.cohere_model = settings.cohere_model ?? default_settings.cohere_model;
@@ -3130,6 +3140,7 @@ function loadOpenAISettings(data, settings) {
31303140 $('#openrouter_group_models').prop('checked', oai_settings.openrouter_group_models);
31313141 $('#openrouter_allow_fallbacks').prop('checked', oai_settings.openrouter_allow_fallbacks);
31323142 $('#openrouter_providers_chat').val(oai_settings.openrouter_providers).trigger('change');
3143+ $('#openrouter_middleout').val(oai_settings.openrouter_middleout);
31333144 $('#squash_system_messages').prop('checked', oai_settings.squash_system_messages);
31343145 $('#continue_prefill').prop('checked', oai_settings.continue_prefill);
31353146 $('#openai_function_calling').prop('checked', oai_settings.function_calling);
@@ -3370,6 +3381,7 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) {
33703381 openrouter_sort_models: settings.openrouter_sort_models,
33713382 openrouter_providers: settings.openrouter_providers,
33723383 openrouter_allow_fallbacks: settings.openrouter_allow_fallbacks,
3384+ openrouter_middleout: settings.openrouter_middleout,
33733385 ai21_model: settings.ai21_model,
33743386 mistralai_model: settings.mistralai_model,
33753387 cohere_model: settings.cohere_model,
@@ -3836,6 +3848,7 @@ function onSettingsPresetChange() {
38363848 openrouter_sort_models: ['#openrouter_sort_models', 'openrouter_sort_models', false],
38373849 openrouter_providers: ['#openrouter_providers_chat', 'openrouter_providers', false],
38383850 openrouter_allow_fallbacks: ['#openrouter_allow_fallbacks', 'openrouter_allow_fallbacks', true],
3851+ openrouter_middleout: ['#openrouter_middleout', 'openrouter_middleout', false],
38393852 ai21_model: ['#model_ai21_select', 'ai21_model', false],
38403853 mistralai_model: ['#model_mistralai_select', 'mistralai_model', false],
38413854 cohere_model: ['#model_cohere_select', 'cohere_model', false],
@@ -5261,6 +5274,11 @@ export function initOpenAI() {
52615274 saveSettingsDebounced();
52625275 });
52635276
5277+ $('#openrouter_middleout').on('input', function () {
5278+ oai_settings.openrouter_middleout = String($(this).val());
5279+ saveSettingsDebounced();
5280+ });
5281+
52645282 $('#squash_system_messages').on('input', function () {
52655283 oai_settings.squash_system_messages = !!$(this).prop('checked');
52665284 saveSettingsDebounced();
src/endpoints/backends/chat-completions.js+19 -1
@@ -72,6 +72,22 @@ function postProcessPrompt(messages, type, names) {
7272}
7373
7474/**
75+ * Gets OpenRouter transforms based on the request.
76+ * @param {import('express').Request} request Express request
77+ * @returns {string[] | undefined} OpenRouter transforms
78+ */
79+function getOpenRouterTransforms(request) {
80+ switch (request.body.middleout) {
81+ case 'on':
82+ return ['middle-out'];
83+ case 'off':
84+ return [];
85+ case 'auto':
86+ return undefined;
87+ }
88+}
89+
90+/**
7591 * Sends a request to Claude API.
7692 * @param {express.Request} request Express request
7793 * @param {express.Response} response Express response
@@ -834,7 +850,9 @@ router.post('/generate', jsonParser, function (request, response) {
834850 apiKey = readSecret(request.user.directories, SECRET_KEYS.OPENROUTER);
835851 // OpenRouter needs to pass the Referer and X-Title: https://openrouter.ai/docs#requests
836852 headers = { ...OPENROUTER_HEADERS };
837- bodyParams = { 'transforms': ['middle-out'] };
853+ bodyParams = {
854+ 'transforms': getOpenRouterTransforms(request),
855+ };
838856
839857 if (request.body.min_p !== undefined) {
840858 bodyParams['min_p'] = request.body.min_p;