feat(openrouter): add model quantizations setting (#5080) * feat(openrouter): add model quantizations setting * Remove bogus setting * Simplify nullish coalescing assignment --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>

6c864e8bb20c312eea05e2313ced9c91e145e96a

Brioch <134280928+Brioch@users.noreply.github.com>

Signed
Parents: bba43f33219e41de7331b61f6872f5c7227503a3
8 files changed, +83 -0Showing whitespace changes
public/index.html+28 -0
@@ -2441,6 +2441,20 @@
2441 <span data-i18n="Allow fallback providers">Allow fallback providers</span>2441 <span data-i18n="Allow fallback providers">Allow fallback providers</span>
2442 </label>2442 </label>
2443 </div>2443 </div>
2444 <div>
2445 <h4 data-i18n="Model Quantizations">Model Quantizations</h4>
2446 <select id="openrouter_quantizations_text" class="openrouter_quantizations" multiple>
2447 <option data-i18n="Integer (4 bit)" value="int4">Integer (4 bit)</option>
2448 <option data-i18n="Integer (8 bit)" value="int8">Integer (8 bit)</option>
2449 <option data-i18n="Floating point (4 bit)" value="fp4">Floating point (4 bit)</option>
2450 <option data-i18n="Floating point (6 bit)" value="fp6">Floating point (6 bit)</option>
2451 <option data-i18n="Floating point (8 bit)" value="fp8">Floating point (8 bit)</option>
2452 <option data-i18n="Floating point (16 bit)" value="fp16">Floating point (16 bit)</option>
2453 <option data-i18n="Brain floating point (16 bit)" value="bf16">Brain floating point (16 bit)</option>
2454 <option data-i18n="Floating point (32 bit)" value="fp32">Floating point (32 bit)</option>
2455 <option data-i18n="Unknown" value="unknown">Unknown</option>
2456 </select>
2457 </div>
2444 </div>2458 </div>
2445 <div data-tg-type="infermaticai" class="flex-container flexFlowColumn">2459 <div data-tg-type="infermaticai" class="flex-container flexFlowColumn">
2446 <h4 data-i18n="InfermaticAI API Key">InfermaticAI API Key</h4>2460 <h4 data-i18n="InfermaticAI API Key">InfermaticAI API Key</h4>
@@ -3183,6 +3197,20 @@
3183 <i class="fa-solid fa-lightbulb"></i>3197 <i class="fa-solid fa-lightbulb"></i>
3184 <span data-i18n="To use instruct formatting, switch to OpenRouter under Text Completion API.">To use instruct formatting, switch to OpenRouter under Text Completion API.</span>3198 <span data-i18n="To use instruct formatting, switch to OpenRouter under Text Completion API.">To use instruct formatting, switch to OpenRouter under Text Completion API.</span>
3185 </small>3199 </small>
3200 <div>
3201 <h4 data-i18n="Model Quantizations">Model Quantizations</h4>
3202 <select id="openrouter_quantizations_chat" class="openrouter_quantizations" multiple>
3203 <option data-i18n="Integer (4 bit)" value="int4">Integer (4 bit)</option>
3204 <option data-i18n="Integer (8 bit)" value="int8">Integer (8 bit)</option>
3205 <option data-i18n="Floating point (4 bit)" value="fp4">Floating point (4 bit)</option>
3206 <option data-i18n="Floating point (6 bit)" value="fp6">Floating point (6 bit)</option>
3207 <option data-i18n="Floating point (8 bit)" value="fp8">Floating point (8 bit)</option>
3208 <option data-i18n="Floating point (16 bit)" value="fp16">Floating point (16 bit)</option>
3209 <option data-i18n="Brain floating point (16 bit)" value="bf16">Brain floating point (16 bit)</option>
3210 <option data-i18n="Floating point (32 bit)" value="fp32">Floating point (32 bit)</option>
3211 <option data-i18n="Unknown" value="unknown">Unknown</option>
3212 </select>
3213 </div>
3186 </form>3214 </form>
3187 <form id="ai21_form" data-source="ai21" action="javascript:void(null);" method="post" enctype="multipart/form-data">3215 <form id="ai21_form" data-source="ai21" action="javascript:void(null);" method="post" enctype="multipart/form-data">
3188 <h4 data-i18n="AI21 API Key">AI21 API Key</h4>3216 <h4 data-i18n="AI21 API Key">AI21 API Key</h4>
public/locales/fr-fr.json+2 -0
@@ -290,6 +290,8 @@
290 "View Remaining Credits": "Afficher les crédits restants",290 "View Remaining Credits": "Afficher les crédits restants",
291 "OpenRouter Model": "Modèle OpenRouter",291 "OpenRouter Model": "Modèle OpenRouter",
292 "Model Providers": "Fournisseurs de modèles",292 "Model Providers": "Fournisseurs de modèles",
293 "Model Quantizations": "Quantifications du modèle",
294 "Select quantizations. No selection = all quantizations.": "Sélectionnez les quantifications. Aucune sélection = toutes les quantifications.",
293 "InfermaticAI API Key": "Clé API InfermaticAI",295 "InfermaticAI API Key": "Clé API InfermaticAI",
294 "InfermaticAI Model": "Modèle InfermaticAI",296 "InfermaticAI Model": "Modèle InfermaticAI",
295 "DreamGen API key": "Clé API DreamGen",297 "DreamGen API key": "Clé API DreamGen",
public/scripts/openai.js+18 -0
@@ -288,6 +288,7 @@ export const settingsToUpdate = {
288 openrouter_group_models: ['#openrouter_group_models', 'openrouter_group_models', false, true],288 openrouter_group_models: ['#openrouter_group_models', 'openrouter_group_models', false, true],
289 openrouter_sort_models: ['#openrouter_sort_models', 'openrouter_sort_models', false, true],289 openrouter_sort_models: ['#openrouter_sort_models', 'openrouter_sort_models', false, true],
290 openrouter_providers: ['#openrouter_providers_chat', 'openrouter_providers', false, true],290 openrouter_providers: ['#openrouter_providers_chat', 'openrouter_providers', false, true],
291 openrouter_quantizations: ['#openrouter_quantizations_chat', 'openrouter_quantizations', false, true],
291 openrouter_allow_fallbacks: ['#openrouter_allow_fallbacks', 'openrouter_allow_fallbacks', true, true],292 openrouter_allow_fallbacks: ['#openrouter_allow_fallbacks', 'openrouter_allow_fallbacks', true, true],
292 openrouter_middleout: ['#openrouter_middleout', 'openrouter_middleout', false, true],293 openrouter_middleout: ['#openrouter_middleout', 'openrouter_middleout', false, true],
293 ai21_model: ['#model_ai21_select', 'ai21_model', false, true],294 ai21_model: ['#model_ai21_select', 'ai21_model', false, true],
@@ -434,6 +435,7 @@ const default_settings = {
434 openrouter_group_models: false,435 openrouter_group_models: false,
435 openrouter_sort_models: 'alphabetically',436 openrouter_sort_models: 'alphabetically',
436 openrouter_providers: [],437 openrouter_providers: [],
438 openrouter_quantizations: [],
437 openrouter_allow_fallbacks: true,439 openrouter_allow_fallbacks: true,
438 openrouter_middleout: openrouter_middleout_types.ON,440 openrouter_middleout: openrouter_middleout_types.ON,
439 reverse_proxy: '',441 reverse_proxy: '',
@@ -2628,6 +2630,7 @@ export async function createGenerationParameters(settings, model, type, messages
2628 generate_data.top_a = Number(settings.top_a_openai);2630 generate_data.top_a = Number(settings.top_a_openai);
2629 generate_data.use_fallback = settings.openrouter_use_fallback;2631 generate_data.use_fallback = settings.openrouter_use_fallback;
2630 generate_data.provider = settings.openrouter_providers;2632 generate_data.provider = settings.openrouter_providers;
2633 generate_data.quantizations = settings.openrouter_quantizations;
2631 generate_data.allow_fallbacks = settings.openrouter_allow_fallbacks;2634 generate_data.allow_fallbacks = settings.openrouter_allow_fallbacks;
2632 generate_data.middleout = settings.openrouter_middleout;2635 generate_data.middleout = settings.openrouter_middleout;
2633 }2636 }
@@ -4048,6 +4051,7 @@ function loadOpenAISettings(data, settings) {
4048 setContinuePostfixControls();4051 setContinuePostfixControls();
40494052
4050 $('#openrouter_providers_chat').trigger('change');4053 $('#openrouter_providers_chat').trigger('change');
4054 $('#openrouter_quantizations_chat').trigger('change');
4051 $('#chat_completion_source').trigger('change');4055 $('#chat_completion_source').trigger('change');
4052}4056}
40534057
@@ -4676,6 +4680,7 @@ function onSettingsPresetChange() {
4676 if (oai_settings.bind_preset_to_connection) {4680 if (oai_settings.bind_preset_to_connection) {
4677 $('#chat_completion_source').trigger('change');4681 $('#chat_completion_source').trigger('change');
4678 $('#openrouter_providers_chat').trigger('change');4682 $('#openrouter_providers_chat').trigger('change');
4683 $('#openrouter_quantizations_chat').trigger('change');
4679 }4684 }
46804685
4681 $('#openai_logit_bias_preset').trigger('change');4686 $('#openai_logit_bias_preset').trigger('change');
@@ -6805,6 +6810,19 @@ export function initOpenAI() {
6805 saveSettingsDebounced();6810 saveSettingsDebounced();
6806 });6811 });
68076812
6813 $('#openrouter_quantizations_chat').on('change', function () {
6814 const selectedQuantizations = $(this).val();
6815
6816 // Not a multiple select?
6817 if (!Array.isArray(selectedQuantizations)) {
6818 return;
6819 }
6820
6821 oai_settings.openrouter_quantizations = selectedQuantizations;
6822
6823 saveSettingsDebounced();
6824 });
6825
6808 $('#bind_preset_to_connection').on('input', function () {6826 $('#bind_preset_to_connection').on('input', function () {
6809 oai_settings.bind_preset_to_connection = !!$(this).prop('checked');6827 oai_settings.bind_preset_to_connection = !!$(this).prop('checked');
6810 saveSettingsDebounced();6828 saveSettingsDebounced();
public/scripts/preset-manager.js+1 -0
@@ -712,6 +712,7 @@ class PresetManager {
712 'featherless_model',712 'featherless_model',
713 'max_tokens_second',713 'max_tokens_second',
714 'openrouter_providers',714 'openrouter_providers',
715 'openrouter_quantizations',
715 'openrouter_allow_fallbacks',716 'openrouter_allow_fallbacks',
716 'tabby_model',717 'tabby_model',
717 'derived',718 'derived',
public/scripts/textgen-models.js+7 -0
@@ -1065,6 +1065,13 @@ export function initTextGenModels() {
1065 width: '100%',1065 width: '100%',
1066 templateResult: getAphroditeModelTemplate,1066 templateResult: getAphroditeModelTemplate,
1067 });1067 });
1068 $('.openrouter_quantizations').select2({
1069 closeOnSelect: false,
1070 placeholder: t`Select quantizations. No selection = all quantizations.`,
1071 searchInputCssClass: 'text_pole',
1072 searchInputPlaceholder: t`Search quantizations...`,
1073 width: '100%',
1074 });
1068 providersSelect.select2({1075 providersSelect.select2({
1069 sorter: data => data.sort((a, b) => a.text.localeCompare(b.text)),1076 sorter: data => data.sort((a, b) => a.text.localeCompare(b.text)),
1070 placeholder: t`Select providers. No selection = all providers.`,1077 placeholder: t`Select providers. No selection = all providers.`,
public/scripts/textgen-settings.js+16 -0
@@ -211,6 +211,7 @@ export const textgenerationwebui_settings = {
211 ollama_model: '',211 ollama_model: '',
212 openrouter_model: 'openrouter/auto',212 openrouter_model: 'openrouter/auto',
213 openrouter_providers: [],213 openrouter_providers: [],
214 openrouter_quantizations: [],
214 vllm_model: '',215 vllm_model: '',
215 aphrodite_model: '',216 aphrodite_model: '',
216 dreamgen_model: 'lucid-v1-extra-large/text',217 dreamgen_model: 'lucid-v1-extra-large/text',
@@ -590,6 +591,7 @@ export async function loadTextGenSettings(data, loadedSettings) {
590591
591 $('#textgen_type').val(textgenerationwebui_settings.type);592 $('#textgen_type').val(textgenerationwebui_settings.type);
592 $('#openrouter_providers_text').val(textgenerationwebui_settings.openrouter_providers).trigger('change');593 $('#openrouter_providers_text').val(textgenerationwebui_settings.openrouter_providers).trigger('change');
594 $('#openrouter_quantizations_text').val(textgenerationwebui_settings.openrouter_quantizations).trigger('change');
593 showSamplerControls(textgenerationwebui_settings.type);595 showSamplerControls(textgenerationwebui_settings.type);
594 BIAS_CACHE.delete(BIAS_KEY);596 BIAS_CACHE.delete(BIAS_KEY);
595 displayLogitBias(textgenerationwebui_settings.logit_bias, BIAS_KEY);597 displayLogitBias(textgenerationwebui_settings.logit_bias, BIAS_KEY);
@@ -1073,6 +1075,19 @@ export function initTextGenSettings() {
1073 saveSettingsDebounced();1075 saveSettingsDebounced();
1074 });1076 });
10751077
1078 $('#openrouter_quantizations_text').on('change', function () {
1079 const selectedQuantizations = $(this).val();
1080
1081 // Not a multiple select?
1082 if (!Array.isArray(selectedQuantizations)) {
1083 return;
1084 }
1085
1086 textgenerationwebui_settings.openrouter_quantizations = selectedQuantizations;
1087
1088 saveSettingsDebounced();
1089 });
1090
1076 $('#api_button_textgenerationwebui').on('click', async function (e) {1091 $('#api_button_textgenerationwebui').on('click', async function (e) {
1077 const keys = [1092 const keys = [
1078 { id: 'api_key_mancer', secret: SECRET_KEYS.MANCER },1093 { id: 'api_key_mancer', secret: SECRET_KEYS.MANCER },
@@ -1735,6 +1750,7 @@ export function createTextGenGenerationData(settings, model, finalPrompt = null,
17351750
1736 if (settings.type === OPENROUTER) {1751 if (settings.type === OPENROUTER) {
1737 params.provider = settings.openrouter_providers;1752 params.provider = settings.openrouter_providers;
1753 params.quantizations = settings.openrouter_quantizations;
1738 params.allow_fallbacks = settings.openrouter_allow_fallbacks;1754 params.allow_fallbacks = settings.openrouter_allow_fallbacks;
1739 }1755 }
17401756
src/endpoints/backends/chat-completions.js+5 -0
@@ -2097,6 +2097,11 @@ router.post('/generate', async function (request, response) {
2097 };2097 };
2098 }2098 }
20992099
2100 if (Array.isArray(request.body.quantizations) && request.body.quantizations.length > 0) {
2101 bodyParams['provider'] ??= {};
2102 bodyParams['provider']['quantizations'] = request.body.quantizations;
2103 }
2104
2100 if (request.body.use_fallback) {2105 if (request.body.use_fallback) {
2101 bodyParams['route'] = 'fallback';2106 bodyParams['route'] = 'fallback';
2102 }2107 }
src/endpoints/backends/text-completions.js+6 -0
@@ -367,6 +367,12 @@ router.post('/generate', async function (request, response) {
367 } else {367 } else {
368 delete request.body.provider;368 delete request.body.provider;
369 }369 }
370
371 if (Array.isArray(request.body.quantizations) && request.body.quantizations.length > 0) {
372 request.body.provider ??= {};
373 request.body.provider.quantizations = request.body.quantizations;
374 }
375
370 request.body = _.pickBy(request.body, (_, key) => OPENROUTER_KEYS.includes(key));376 request.body = _.pickBy(request.body, (_, key) => OPENROUTER_KEYS.includes(key));
371 args.body = JSON.stringify(request.body);377 args.body = JSON.stringify(request.body);
372 }378 }