Add model selection support for llama.cpp router mode (#4910) * Add model selection support for llama.cpp router mode - Add llamacpp_model setting to textgen-settings.js - Implement loadLlamaCppModels() function to fetch and populate models - Add onLlamaCppModelSelect() handler for model selection - Update status check to load llama.cpp models when connecting - Update getTextGenModel() to return selected llama.cpp model - Add model dropdown to HTML UI in llama.cpp section - Initialize event handlers and Select2 for better UX - Add llamacpp_model to preset manager for save/load support - Add llamacpp_model to slash commands support This implements model selection for llama.cpp router mode, allowing users to select from multiple models without restarting the server. Follows the same pattern as Ollama, Tabby, and vLLM implementations. * Correct spelling * Fix clear selection position --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>
Signed| @@ -16,6 +16,11 @@ | ||
| 16 | 16 | user-select: none; |
| 17 | 17 | } |
| 18 | 18 | |
| 19 | +.select2-container .select2-selection.select2-selection--single.select2-selection--clearable .select2-selection__clear { | |
| 20 | + top: 0; | |
| 21 | + right: 25px; | |
| 22 | +} | |
| 23 | + | |
| 19 | 24 | .select2-container .select2-selection .select2-selection__clear { |
| 20 | 25 | color: var(--SmartThemeBodyColor); |
| 21 | 26 | font-size: 20px; |
| @@ -2679,6 +2679,16 @@ | ||
| 2679 | 2679 | <small data-i18n="Example: http://127.0.0.1:8080">Example: http://127.0.0.1:8080</small> |
| 2680 | 2680 | <input id="llamacpp_api_url_text" class="text_pole wide100p" value="" autocomplete="off" data-server-history="llamacpp"> |
| 2681 | 2681 | </div> |
| 2682 | + <div> | |
| 2683 | + <h4> | |
| 2684 | + <span data-i18n="llama.cpp Model">llama.cpp Model</span> | |
| 2685 | + </h4> | |
| 2686 | + <select id="llamacpp_model"> | |
| 2687 | + <option value="" data-i18n="-- Connect to the API --"> | |
| 2688 | + -- Connect to the API -- | |
| 2689 | + </option> | |
| 2690 | + </select> | |
| 2691 | + </div> | |
| 2682 | 2692 | </div> |
| 2683 | 2693 | <div data-tg-type="ollama"> |
| 2684 | 2694 | <div class="flex-container flexFlowColumn"> |
| @@ -701,6 +701,7 @@ class PresetManager { | ||
| 701 | 701 | 'ollama_model', |
| 702 | 702 | 'vllm_model', |
| 703 | 703 | 'aphrodite_model', |
| 704 | + 'llamacpp_model', | |
| 704 | 705 | 'server_urls', |
| 705 | 706 | 'type', |
| 706 | 707 | 'custom_model', |
| @@ -4834,6 +4834,7 @@ function getModelOptions(quiet) { | ||
| 4834 | 4834 | { id: 'aphrodite_model', api: 'textgenerationwebui', type: textgen_types.APHRODITE }, |
| 4835 | 4835 | { id: 'ollama_model', api: 'textgenerationwebui', type: textgen_types.OLLAMA }, |
| 4836 | 4836 | { id: 'tabby_model', api: 'textgenerationwebui', type: textgen_types.TABBY }, |
| 4837 | + { id: 'llamacpp_model', api: 'textgenerationwebui', type: textgen_types.LLAMACPP }, | |
| 4837 | 4838 | { id: 'featherless_model', api: 'textgenerationwebui', type: textgen_types.FEATHERLESS }, |
| 4838 | 4839 | { id: 'model_openai_select', api: 'openai', type: chat_completion_sources.OPENAI }, |
| 4839 | 4840 | { id: 'model_claude_select', api: 'openai', type: chat_completion_sources.CLAUDE }, |
| @@ -17,6 +17,7 @@ let vllmModels = []; | ||
| 17 | 17 | let aphroditeModels = []; |
| 18 | 18 | let featherlessModels = []; |
| 19 | 19 | let tabbyModels = []; |
| 20 | +let llamacppModels = []; | |
| 20 | 21 | export let openRouterModels = []; |
| 21 | 22 | |
| 22 | 23 | /** |
| @@ -139,6 +140,30 @@ export async function loadTabbyModels(data) { | ||
| 139 | 140 | } |
| 140 | 141 | } |
| 141 | 142 | |
| 143 | +export async function loadLlamaCppModels(data) { | |
| 144 | + if (!Array.isArray(data)) { | |
| 145 | + console.error('Invalid llama.cpp models data', data); | |
| 146 | + return; | |
| 147 | + } | |
| 148 | + | |
| 149 | + llamacppModels = data; | |
| 150 | + llamacppModels.sort((a, b) => a.id.localeCompare(b.id)); | |
| 151 | + llamacppModels.unshift({ id: '' }); | |
| 152 | + | |
| 153 | + if (!llamacppModels.find(x => x.id === textgen_settings.llamacpp_model)) { | |
| 154 | + textgen_settings.llamacpp_model = llamacppModels[0]?.id || ''; | |
| 155 | + } | |
| 156 | + | |
| 157 | + $('#llamacpp_model').empty(); | |
| 158 | + for (const model of llamacppModels) { | |
| 159 | + const option = document.createElement('option'); | |
| 160 | + option.value = model.id; | |
| 161 | + option.text = model.id; | |
| 162 | + option.selected = model.id === textgen_settings.llamacpp_model; | |
| 163 | + $('#llamacpp_model').append(option); | |
| 164 | + } | |
| 165 | +} | |
| 166 | + | |
| 142 | 167 | export async function loadTogetherAIModels(data) { |
| 143 | 168 | if (!Array.isArray(data)) { |
| 144 | 169 | console.error('Invalid Together AI models data', data); |
| @@ -637,6 +662,12 @@ function onTabbyModelSelect() { | ||
| 637 | 662 | $('#api_button_textgenerationwebui').trigger('click'); |
| 638 | 663 | } |
| 639 | 664 | |
| 665 | +function onLlamaCppModelSelect() { | |
| 666 | + const modelId = String($('#llamacpp_model').val()); | |
| 667 | + textgen_settings.llamacpp_model = modelId; | |
| 668 | + $('#api_button_textgenerationwebui').trigger('click'); | |
| 669 | +} | |
| 670 | + | |
| 640 | 671 | function onOpenRouterModelSelect() { |
| 641 | 672 | const modelId = String($('#openrouter_model').val()); |
| 642 | 673 | textgen_settings.openrouter_model = modelId; |
| @@ -952,6 +983,7 @@ export function initTextGenModels() { | ||
| 952 | 983 | $('#aphrodite_model').on('change', onAphroditeModelSelect); |
| 953 | 984 | $('#tabby_download_model').on('click', downloadTabbyModel); |
| 954 | 985 | $('#tabby_model').on('change', onTabbyModelSelect); |
| 986 | + $('#llamacpp_model').on('change', onLlamaCppModelSelect); | |
| 955 | 987 | $('#featherless_model').on('change', () => onFeatherlessModelSelect(String($('#featherless_model').val()))); |
| 956 | 988 | |
| 957 | 989 | const providersSelect = $('.openrouter_providers'); |
| @@ -990,6 +1022,13 @@ export function initTextGenModels() { | ||
| 990 | 1022 | width: '100%', |
| 991 | 1023 | allowClear: true, |
| 992 | 1024 | }); |
| 1025 | + $('#llamacpp_model').select2({ | |
| 1026 | + placeholder: t`[Currently loaded]`, | |
| 1027 | + searchInputPlaceholder: t`Search models...`, | |
| 1028 | + searchInputCssClass: 'text_pole', | |
| 1029 | + width: '100%', | |
| 1030 | + allowClear: true, | |
| 1031 | + }); | |
| 993 | 1032 | $('#model_infermaticai_select').select2({ |
| 994 | 1033 | placeholder: t`Select a model`, |
| 995 | 1034 | searchInputPlaceholder: t`Search models...`, |
| @@ -23,7 +23,7 @@ import { power_user, registerDebugFunction } from './power-user.js'; | ||
| 23 | 23 | import { getActiveManualApiSamplers, loadApiSelectedSamplers, isSamplerManualPriorityEnabled } from './samplerSelect.js'; |
| 24 | 24 | import { SECRET_KEYS, writeSecret } from './secrets.js'; |
| 25 | 25 | import { getEventSourceStream } from './sse-stream.js'; |
| 26 | 26 | import { getCurrentDreamGenModelTokenizer, getCurrentOpenRouterModelTokenizer, loadAphroditeModels, loadDreamGenModels, loadFeatherlessModels, loadGenericModels, loadInfermaticAIModels, loadLlamaCppModels, loadMancerModels, loadOllamaModels, loadOpenRouterModels, loadTabbyModels, loadTogetherAIModels, loadVllmModels } from './textgen-models.js'; |
| 27 | 27 | import { ENCODE_TOKENIZERS, TEXTGEN_TOKENIZERS, TOKENIZER_SUPPORTED_KEY, getTextTokens, tokenizers } from './tokenizers.js'; |
| 28 | 28 | import { AbortReason } from './util/AbortReason.js'; |
| 29 | 29 | import { getSortableDelay, onlyUnique, arraysEqual, isObject } from './utils.js'; |
| @@ -214,6 +214,7 @@ export const textgenerationwebui_settings = { | ||
| 214 | 214 | aphrodite_model: '', |
| 215 | 215 | dreamgen_model: 'lucid-v1-extra-large/text', |
| 216 | 216 | tabby_model: '', |
| 217 | + llamacpp_model: '', | |
| 217 | 218 | sampler_order: KOBOLDCPP_ORDER, |
| 218 | 219 | logit_bias: [], |
| 219 | 220 | n: 1, |
| @@ -701,6 +702,9 @@ async function getStatusTextgen() { | ||
| 701 | 702 | } else if (textgenerationwebui_settings.type === textgen_types.TABBY) { |
| 702 | 703 | loadTabbyModels(data?.data); |
| 703 | 704 | setOnlineStatus(textgenerationwebui_settings.tabby_model || data?.result); |
| 705 | + } else if (textgenerationwebui_settings.type === textgen_types.LLAMACPP) { | |
| 706 | + loadLlamaCppModels(data?.data); | |
| 707 | + setOnlineStatus(textgenerationwebui_settings.llamacpp_model || data?.result || t`Connected`); | |
| 704 | 708 | } else if (textgenerationwebui_settings.type === textgen_types.GENERIC) { |
| 705 | 709 | loadGenericModels(data?.data); |
| 706 | 710 | setOnlineStatus(textgenerationwebui_settings.generic_model || data?.result || t`Connected`); |
| @@ -1462,6 +1466,11 @@ export function getTextGenModel(settings = null) { | ||
| 1462 | 1466 | return settings.tabby_model; |
| 1463 | 1467 | } |
| 1464 | 1468 | break; |
| 1469 | + case LLAMACPP: | |
| 1470 | + if (settings.llamacpp_model) { | |
| 1471 | + return settings.llamacpp_model; | |
| 1472 | + } | |
| 1473 | + break; | |
| 1465 | 1474 | default: |
| 1466 | 1475 | return undefined; |
| 1467 | 1476 | } |