custom endpoint handling (#4031)

e4217dbeba8891436e494847f3b514c9ed99a946

Aykut Akgün <68244453+Autumnlight02@users.noreply.github.com>

Signed
Parents: 2a7a8cab11bd49facda59f6fbc3a7af431e297dc
4 files changed, +71 -29Showing whitespace changes
public/scripts/extensions/caption/index.js+15 -1
@@ -408,6 +408,8 @@ jQuery(async function () {
408408 // Handle multimodal sources
409409 if (settings.source === 'multimodal') {
410410 const api = settings.multimodal_api;
411+ const altEndpointEnabled = settings.alt_endpoint_enabled;
412+ const altEndpointUrl = settings.alt_endpoint_url;
411413
412414 // APIs that support reverse proxy
413415 const reverseProxyApis = {
@@ -444,7 +446,11 @@ jQuery(async function () {
444446 'vllm': textgen_types.VLLM,
445447 };
446448
447449 if (textCompletionApis[api] && textgenerationwebui_settings.server_urls[textCompletionApis[api]]altEndpointEnabled && altEndpointUrl) {
450+ return true;
451+ }
452+
453+ if (textCompletionApis[api] && !altEndpointEnabled && textgenerationwebui_settings.server_urls[textCompletionApis[api]]) {
448454 return true;
449455 }
450456
@@ -580,6 +586,14 @@ jQuery(async function () {
580586 extension_settings.caption.multimodal_model = String($('#caption_multimodal_model').val());
581587 saveSettingsDebounced();
582588 });
589+ $('#caption_altEndpoint_url').val(extension_settings.caption.alt_endpoint_url).on('input', () => {
590+ extension_settings.caption.alt_endpoint_url = String($('#caption_altEndpoint_url').val());
591+ saveSettingsDebounced();
592+ });
593+ $('#caption_altEndpoint_enabled').prop('checked', !!(extension_settings.caption.alt_endpoint_enabled)).on('input', () => {
594+ extension_settings.caption.alt_endpoint_enabled = !!$('#caption_altEndpoint_enabled').prop('checked');
595+ saveSettingsDebounced();
596+ });
583597
584598 const onMessageEvent = async (index) => {
585599 if (!extension_settings.caption.auto_mode) {
public/scripts/extensions/caption/settings.html+10 -0
@@ -189,6 +189,16 @@
189189 <small><b data-i18n="Hint:">Hint:</b> <span data-i18n="Set your API keys and endpoints in the 'API Connections' tab first.">Set your API keys and endpoints in the 'API Connections' tab first.</span></small>
190190 </div>
191191 </div>
192+ <div data-type="koboldcpp,ollama,vllm,llamacpp,ooba" class="flex-container flexFlowColumn">
193+ <label for="caption_altEndpoint_enabled" class="checkbox_label">
194+ <input id="caption_altEndpoint_enabled" type="checkbox">
195+ <span data-i18n="Use secondary URL">Use secondary URL</span>
196+ </label>
197+ <label for="caption_altEndpoint_url" data-i18n="Secondary captioning endpoint URL">
198+ Secondary captioning endpoint URL
199+ </label>
200+ <input id="caption_altEndpoint_url" class="text_pole" type="text" placeholder="e.g. http://localhost:5001" />
201+ </div>
192202 <div id="caption_prompt_block">
193203 <label for="caption_prompt" data-i18n="Caption Prompt">Caption Prompt</label>
194204 <textarea id="caption_prompt" class="text_pole" rows="1" placeholder="&lt; Use default &gt;">{{PROMPT_DEFAULT}}</textarea>
public/scripts/extensions/shared.js+42 -23
@@ -61,7 +61,9 @@ export async function getMultimodalCaption(base64Img, prompt) {
6161 requestBody.model = textgenerationwebui_settings.ollama_model;
6262 }
6363
6464 requestBody.server_url = textgenerationwebui_settingsextension_settings.server_urls[textgen_typescaption.OLLAMA];alt_endpoint_enabled
65+ ? extension_settings.caption.alt_endpoint_url
66+ : textgenerationwebui_settings.server_urls[textgen_types.OLLAMA];
6567 }
6668
6769 if (isVllm) {
@@ -69,19 +71,27 @@ export async function getMultimodalCaption(base64Img, prompt) {
6971 requestBody.model = textgenerationwebui_settings.vllm_model;
7072 }
7173
7274 requestBody.server_url = textgenerationwebui_settingsextension_settings.server_urls[textgen_typescaption.VLLM];alt_endpoint_enabled
75+ ? extension_settings.caption.alt_endpoint_url
76+ : textgenerationwebui_settings.server_urls[textgen_types.VLLM];
7377 }
7478
7579 if (isLlamaCpp) {
7680 requestBody.server_url = textgenerationwebui_settingsextension_settings.server_urls[textgen_typescaption.LLAMACPP];alt_endpoint_enabled
81+ ? extension_settings.caption.alt_endpoint_url
82+ : textgenerationwebui_settings.server_urls[textgen_types.LLAMACPP];
7783 }
7884
7985 if (isOoba) {
8086 requestBody.server_url = textgenerationwebui_settingsextension_settings.server_urls[textgen_typescaption.OOBA];alt_endpoint_enabled
87+ ? extension_settings.caption.alt_endpoint_url
88+ : textgenerationwebui_settings.server_urls[textgen_types.OOBA];
8189 }
8290
8391 if (isKoboldCpp) {
8492 requestBody.server_url = textgenerationwebui_settingsextension_settings.server_urls[textgen_typescaption.KOBOLDCPP];alt_endpoint_enabled
93+ ? extension_settings.caption.alt_endpoint_url
94+ : textgenerationwebui_settings.server_urls[textgen_types.KOBOLDCPP];
8595 }
8696
8797 if (isCustom) {
@@ -121,75 +131,84 @@ export async function getMultimodalCaption(base64Img, prompt) {
121131}
122132
123133function throwIfInvalidModel(useReverseProxy) {
124- if (extension_settings.caption.multimodal_api === 'openai' && !secret_state[SECRET_KEYS.OPENAI] && !useReverseProxy) {
134+ const altEndpointEnabled = extension_settings.caption.alt_endpoint_enabled;
135+ const altEndpointUrl = extension_settings.caption.alt_endpoint_url;
136+ const multimodalModel = extension_settings.caption.multimodal_model;
137+ const multimodalApi = extension_settings.caption.multimodal_api;
138+
139+ if (altEndpointEnabled && ['llamacpp', 'ooba', 'koboldcpp', 'vllm', 'ollama'].includes(multimodalApi) && !altEndpointUrl) {
140+ throw new Error('Secondary endpoint URL is not set.');
141+ }
142+
143+ if (multimodalApi === 'openai' && !secret_state[SECRET_KEYS.OPENAI] && !useReverseProxy) {
125144 throw new Error('OpenAI API key is not set.');
126145 }
127146
128147 if (extension_settings.caption.multimodal_apimultimodalApi === 'openrouter' && !secret_state[SECRET_KEYS.OPENROUTER]) {
129148 throw new Error('OpenRouter API key is not set.');
130149 }
131150
132151 if (extension_settings.caption.multimodal_apimultimodalApi === 'anthropic' && !secret_state[SECRET_KEYS.CLAUDE] && !useReverseProxy) {
133152 throw new Error('Anthropic (Claude) API key is not set.');
134153 }
135154
136155 if (extension_settings.caption.multimodal_apimultimodalApi === 'zerooneai' && !secret_state[SECRET_KEYS.ZEROONEAI]) {
137156 throw new Error('01.AI API key is not set.');
138157 }
139158
140159 if (extension_settings.caption.multimodal_apimultimodalApi === 'groq' && !secret_state[SECRET_KEYS.GROQ]) {
141160 throw new Error('Groq API key is not set.');
142161 }
143162
144163 if (extension_settings.caption.multimodal_apimultimodalApi === 'google' && !secret_state[SECRET_KEYS.MAKERSUITE] && !useReverseProxy) {
145164 throw new Error('Google AI Studio API key is not set.');
146165 }
147166
148167 if (extension_settings.caption.multimodal_apimultimodalApi === 'vertexai' && !secret_state[SECRET_KEYS.VERTEXAI] && !useReverseProxy) {
149168 throw new Error('Google Vertex AI API key is not set.');
150169 }
151170
152171 if (extension_settings.caption.multimodal_apimultimodalApi === 'mistral' && !secret_state[SECRET_KEYS.MISTRALAI] && !useReverseProxy) {
153172 throw new Error('Mistral AI API key is not set.');
154173 }
155174
156175 if (extension_settings.caption.multimodal_apimultimodalApi === 'cohere' && !secret_state[SECRET_KEYS.COHERE]) {
157176 throw new Error('Cohere API key is not set.');
158177 }
159178
160179 if (extension_settings.caption.multimodal_apimultimodalApi === 'xai' && !secret_state[SECRET_KEYS.XAI] && !useReverseProxy) {
161180 throw new Error('xAI API key is not set.');
162181 }
163182
164183 if (extension_settings.caption.multimodal_apimultimodalApi === 'ollama' && !textgenerationwebui_settings.server_urls[textgen_types.OLLAMA] && !altEndpointEnabled) {
165184 throw new Error('Ollama server URL is not set.');
166185 }
167186
168187 if (extension_settings.caption.multimodal_apimultimodalApi === 'ollama' && extension_settings.caption.multimodal_modelmultimodalModel === 'ollama_current' && !textgenerationwebui_settings.ollama_model) {
169188 throw new Error('Ollama model is not set.');
170189 }
171190
172191 if (extension_settings.caption.multimodal_apimultimodalApi === 'llamacpp' && !textgenerationwebui_settings.server_urls[textgen_types.LLAMACPP] && !altEndpointEnabled) {
173192 throw new Error('LlamaCPP server URL is not set.');
174193 }
175194
176195 if (extension_settings.caption.multimodal_apimultimodalApi === 'ooba' && !textgenerationwebui_settings.server_urls[textgen_types.OOBA] && !altEndpointEnabled) {
177196 throw new Error('Text Generation WebUI server URL is not set.');
178197 }
179198
180199 if (extension_settings.caption.multimodal_apimultimodalApi === 'koboldcpp' && !textgenerationwebui_settings.server_urls[textgen_types.KOBOLDCPP] && !altEndpointEnabled) {
181200 throw new Error('KoboldCpp server URL is not set.');
182201 }
183202
184203 if (extension_settings.caption.multimodal_apimultimodalApi === 'vllm' && !textgenerationwebui_settings.server_urls[textgen_types.VLLM] && !altEndpointEnabled) {
185204 throw new Error('vLLM server URL is not set.');
186205 }
187206
188207 if (extension_settings.caption.multimodal_apimultimodalApi === 'vllm' && extension_settings.caption.multimodal_modelmultimodalModel === 'vllm_current' && !textgenerationwebui_settings.vllm_model) {
189208 throw new Error('vLLM model is not set.');
190209 }
191210
192211 if (extension_settings.caption.multimodal_apimultimodalApi === 'custom' && !oai_settings.custom_url) {
193212 throw new Error('Custom API URL is not set.');
194213 }
195214}
src/endpoints/openai.js+4 -5
@@ -151,8 +151,11 @@ router.post('/caption-image', async (request, response) => {
151151 apiUrl = 'https://text.pollinations.ai/openai/chat/completions';
152152 }
153153
154- if (request.body.api === 'ooba') {
154+ if (['koboldcpp', 'vllm', 'llamacpp', 'ooba'].includes(request.body.api)) {
155155 apiUrl = `${trimV1(request.body.server_url)}/v1/chat/completions`;
156+ }
157+
158+ if (request.body.api === 'ooba') {
156159 const imgMessage = body.messages.pop();
157160 body.messages.push({
158161 role: 'user',
@@ -165,10 +168,6 @@ router.post('/caption-image', async (request, response) => {
165168 });
166169 }
167170
168- if (['koboldcpp', 'vllm', 'llamacpp'].includes(request.body.api)) {
169- apiUrl = `${trimV1(request.body.server_url)}/v1/chat/completions`;
170- }
171-
172171 setAdditionalHeaders(request, { headers }, apiUrl);
173172 console.debug('Multimodal captioning request', body);
174173