feat(openrouter): add model quantizations setting (#5080) * feat(openrouter): add model quantizations setting * Remove bogus setting * Simplify nullish coalescing assignment --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>

6c864e8bb20c312eea05e2313ced9c91e145e96a

Brioch <134280928+Brioch@users.noreply.github.com>

Signed
Parents: bba43f33219e41de7331b61f6872f5c7227503a3
8 files changed, +83 -0Showing whitespace changes
public/index.html+28 -0
@@ -2441,6 +2441,20 @@
24412441 <span data-i18n="Allow fallback providers">Allow fallback providers</span>
24422442 </label>
24432443 </div>
2444+ <div>
2445+ <h4 data-i18n="Model Quantizations">Model Quantizations</h4>
2446+ <select id="openrouter_quantizations_text" class="openrouter_quantizations" multiple>
2447+ <option data-i18n="Integer (4 bit)" value="int4">Integer (4 bit)</option>
2448+ <option data-i18n="Integer (8 bit)" value="int8">Integer (8 bit)</option>
2449+ <option data-i18n="Floating point (4 bit)" value="fp4">Floating point (4 bit)</option>
2450+ <option data-i18n="Floating point (6 bit)" value="fp6">Floating point (6 bit)</option>
2451+ <option data-i18n="Floating point (8 bit)" value="fp8">Floating point (8 bit)</option>
2452+ <option data-i18n="Floating point (16 bit)" value="fp16">Floating point (16 bit)</option>
2453+ <option data-i18n="Brain floating point (16 bit)" value="bf16">Brain floating point (16 bit)</option>
2454+ <option data-i18n="Floating point (32 bit)" value="fp32">Floating point (32 bit)</option>
2455+ <option data-i18n="Unknown" value="unknown">Unknown</option>
2456+ </select>
2457+ </div>
24442458 </div>
24452459 <div data-tg-type="infermaticai" class="flex-container flexFlowColumn">
24462460 <h4 data-i18n="InfermaticAI API Key">InfermaticAI API Key</h4>
@@ -3183,6 +3197,20 @@
31833197 <i class="fa-solid fa-lightbulb"></i>
31843198 <span data-i18n="To use instruct formatting, switch to OpenRouter under Text Completion API.">To use instruct formatting, switch to OpenRouter under Text Completion API.</span>
31853199 </small>
3200+ <div>
3201+ <h4 data-i18n="Model Quantizations">Model Quantizations</h4>
3202+ <select id="openrouter_quantizations_chat" class="openrouter_quantizations" multiple>
3203+ <option data-i18n="Integer (4 bit)" value="int4">Integer (4 bit)</option>
3204+ <option data-i18n="Integer (8 bit)" value="int8">Integer (8 bit)</option>
3205+ <option data-i18n="Floating point (4 bit)" value="fp4">Floating point (4 bit)</option>
3206+ <option data-i18n="Floating point (6 bit)" value="fp6">Floating point (6 bit)</option>
3207+ <option data-i18n="Floating point (8 bit)" value="fp8">Floating point (8 bit)</option>
3208+ <option data-i18n="Floating point (16 bit)" value="fp16">Floating point (16 bit)</option>
3209+ <option data-i18n="Brain floating point (16 bit)" value="bf16">Brain floating point (16 bit)</option>
3210+ <option data-i18n="Floating point (32 bit)" value="fp32">Floating point (32 bit)</option>
3211+ <option data-i18n="Unknown" value="unknown">Unknown</option>
3212+ </select>
3213+ </div>
31863214 </form>
31873215 <form id="ai21_form" data-source="ai21" action="javascript:void(null);" method="post" enctype="multipart/form-data">
31883216 <h4 data-i18n="AI21 API Key">AI21 API Key</h4>
public/locales/fr-fr.json+2 -0
@@ -290,6 +290,8 @@
290290 "View Remaining Credits": "Afficher les crédits restants",
291291 "OpenRouter Model": "Modèle OpenRouter",
292292 "Model Providers": "Fournisseurs de modèles",
293+ "Model Quantizations": "Quantifications du modèle",
294+ "Select quantizations. No selection = all quantizations.": "Sélectionnez les quantifications. Aucune sélection = toutes les quantifications.",
293295 "InfermaticAI API Key": "Clé API InfermaticAI",
294296 "InfermaticAI Model": "Modèle InfermaticAI",
295297 "DreamGen API key": "Clé API DreamGen",
public/scripts/openai.js+18 -0
@@ -288,6 +288,7 @@ export const settingsToUpdate = {
288288 openrouter_group_models: ['#openrouter_group_models', 'openrouter_group_models', false, true],
289289 openrouter_sort_models: ['#openrouter_sort_models', 'openrouter_sort_models', false, true],
290290 openrouter_providers: ['#openrouter_providers_chat', 'openrouter_providers', false, true],
291+ openrouter_quantizations: ['#openrouter_quantizations_chat', 'openrouter_quantizations', false, true],
291292 openrouter_allow_fallbacks: ['#openrouter_allow_fallbacks', 'openrouter_allow_fallbacks', true, true],
292293 openrouter_middleout: ['#openrouter_middleout', 'openrouter_middleout', false, true],
293294 ai21_model: ['#model_ai21_select', 'ai21_model', false, true],
@@ -434,6 +435,7 @@ const default_settings = {
434435 openrouter_group_models: false,
435436 openrouter_sort_models: 'alphabetically',
436437 openrouter_providers: [],
438+ openrouter_quantizations: [],
437439 openrouter_allow_fallbacks: true,
438440 openrouter_middleout: openrouter_middleout_types.ON,
439441 reverse_proxy: '',
@@ -2628,6 +2630,7 @@ export async function createGenerationParameters(settings, model, type, messages
26282630 generate_data.top_a = Number(settings.top_a_openai);
26292631 generate_data.use_fallback = settings.openrouter_use_fallback;
26302632 generate_data.provider = settings.openrouter_providers;
2633+ generate_data.quantizations = settings.openrouter_quantizations;
26312634 generate_data.allow_fallbacks = settings.openrouter_allow_fallbacks;
26322635 generate_data.middleout = settings.openrouter_middleout;
26332636 }
@@ -4048,6 +4051,7 @@ function loadOpenAISettings(data, settings) {
40484051 setContinuePostfixControls();
40494052
40504053 $('#openrouter_providers_chat').trigger('change');
4054+ $('#openrouter_quantizations_chat').trigger('change');
40514055 $('#chat_completion_source').trigger('change');
40524056}
40534057
@@ -4676,6 +4680,7 @@ function onSettingsPresetChange() {
46764680 if (oai_settings.bind_preset_to_connection) {
46774681 $('#chat_completion_source').trigger('change');
46784682 $('#openrouter_providers_chat').trigger('change');
4683+ $('#openrouter_quantizations_chat').trigger('change');
46794684 }
46804685
46814686 $('#openai_logit_bias_preset').trigger('change');
@@ -6805,6 +6810,19 @@ export function initOpenAI() {
68056810 saveSettingsDebounced();
68066811 });
68076812
6813+ $('#openrouter_quantizations_chat').on('change', function () {
6814+ const selectedQuantizations = $(this).val();
6815+
6816+ // Not a multiple select?
6817+ if (!Array.isArray(selectedQuantizations)) {
6818+ return;
6819+ }
6820+
6821+ oai_settings.openrouter_quantizations = selectedQuantizations;
6822+
6823+ saveSettingsDebounced();
6824+ });
6825+
68086826 $('#bind_preset_to_connection').on('input', function () {
68096827 oai_settings.bind_preset_to_connection = !!$(this).prop('checked');
68106828 saveSettingsDebounced();
public/scripts/preset-manager.js+1 -0
@@ -712,6 +712,7 @@ class PresetManager {
712712 'featherless_model',
713713 'max_tokens_second',
714714 'openrouter_providers',
715+ 'openrouter_quantizations',
715716 'openrouter_allow_fallbacks',
716717 'tabby_model',
717718 'derived',
public/scripts/textgen-models.js+7 -0
@@ -1065,6 +1065,13 @@ export function initTextGenModels() {
10651065 width: '100%',
10661066 templateResult: getAphroditeModelTemplate,
10671067 });
1068+ $('.openrouter_quantizations').select2({
1069+ closeOnSelect: false,
1070+ placeholder: t`Select quantizations. No selection = all quantizations.`,
1071+ searchInputCssClass: 'text_pole',
1072+ searchInputPlaceholder: t`Search quantizations...`,
1073+ width: '100%',
1074+ });
10681075 providersSelect.select2({
10691076 sorter: data => data.sort((a, b) => a.text.localeCompare(b.text)),
10701077 placeholder: t`Select providers. No selection = all providers.`,
public/scripts/textgen-settings.js+16 -0
@@ -211,6 +211,7 @@ export const textgenerationwebui_settings = {
211211 ollama_model: '',
212212 openrouter_model: 'openrouter/auto',
213213 openrouter_providers: [],
214+ openrouter_quantizations: [],
214215 vllm_model: '',
215216 aphrodite_model: '',
216217 dreamgen_model: 'lucid-v1-extra-large/text',
@@ -590,6 +591,7 @@ export async function loadTextGenSettings(data, loadedSettings) {
590591
591592 $('#textgen_type').val(textgenerationwebui_settings.type);
592593 $('#openrouter_providers_text').val(textgenerationwebui_settings.openrouter_providers).trigger('change');
594+ $('#openrouter_quantizations_text').val(textgenerationwebui_settings.openrouter_quantizations).trigger('change');
593595 showSamplerControls(textgenerationwebui_settings.type);
594596 BIAS_CACHE.delete(BIAS_KEY);
595597 displayLogitBias(textgenerationwebui_settings.logit_bias, BIAS_KEY);
@@ -1073,6 +1075,19 @@ export function initTextGenSettings() {
10731075 saveSettingsDebounced();
10741076 });
10751077
1078+ $('#openrouter_quantizations_text').on('change', function () {
1079+ const selectedQuantizations = $(this).val();
1080+
1081+ // Not a multiple select?
1082+ if (!Array.isArray(selectedQuantizations)) {
1083+ return;
1084+ }
1085+
1086+ textgenerationwebui_settings.openrouter_quantizations = selectedQuantizations;
1087+
1088+ saveSettingsDebounced();
1089+ });
1090+
10761091 $('#api_button_textgenerationwebui').on('click', async function (e) {
10771092 const keys = [
10781093 { id: 'api_key_mancer', secret: SECRET_KEYS.MANCER },
@@ -1735,6 +1750,7 @@ export function createTextGenGenerationData(settings, model, finalPrompt = null,
17351750
17361751 if (settings.type === OPENROUTER) {
17371752 params.provider = settings.openrouter_providers;
1753+ params.quantizations = settings.openrouter_quantizations;
17381754 params.allow_fallbacks = settings.openrouter_allow_fallbacks;
17391755 }
17401756
src/endpoints/backends/chat-completions.js+5 -0
@@ -2097,6 +2097,11 @@ router.post('/generate', async function (request, response) {
20972097 };
20982098 }
20992099
2100+ if (Array.isArray(request.body.quantizations) && request.body.quantizations.length > 0) {
2101+ bodyParams['provider'] ??= {};
2102+ bodyParams['provider']['quantizations'] = request.body.quantizations;
2103+ }
2104+
21002105 if (request.body.use_fallback) {
21012106 bodyParams['route'] = 'fallback';
21022107 }
src/endpoints/backends/text-completions.js+6 -0
@@ -367,6 +367,12 @@ router.post('/generate', async function (request, response) {
367367 } else {
368368 delete request.body.provider;
369369 }
370+
371+ if (Array.isArray(request.body.quantizations) && request.body.quantizations.length > 0) {
372+ request.body.provider ??= {};
373+ request.body.provider.quantizations = request.body.quantizations;
374+ }
375+
370376 request.body = _.pickBy(request.body, (_, key) => OPENROUTER_KEYS.includes(key));
371377 args.body = JSON.stringify(request.body);
372378 }