custom endpoint handling (#4031)

e4217dbeba8891436e494847f3b514c9ed99a946

Aykut Akgün <68244453+Autumnlight02@users.noreply.github.com>

Signed
Parents: 2a7a8cab11bd49facda59f6fbc3a7af431e297dc
4 files changed, +71 -29Ignore whitespace
public/scripts/extensions/caption/index.js+15 -1
@@ -408,6 +408,8 @@ jQuery(async function () {
408 // Handle multimodal sources408 // Handle multimodal sources
409 if (settings.source === 'multimodal') {409 if (settings.source === 'multimodal') {
410 const api = settings.multimodal_api;410 const api = settings.multimodal_api;
411 const altEndpointEnabled = settings.alt_endpoint_enabled;
412 const altEndpointUrl = settings.alt_endpoint_url;
411413
412 // APIs that support reverse proxy414 // APIs that support reverse proxy
413 const reverseProxyApis = {415 const reverseProxyApis = {
@@ -444,7 +446,11 @@ jQuery(async function () {
444 'vllm': textgen_types.VLLM,446 'vllm': textgen_types.VLLM,
445 };447 };
446448
447 if (textCompletionApis[api] && textgenerationwebui_settings.server_urls[textCompletionApis[api]]) {449 if (textCompletionApis[api] && altEndpointEnabled && altEndpointUrl) {
450 return true;
451 }
452
453 if (textCompletionApis[api] && !altEndpointEnabled && textgenerationwebui_settings.server_urls[textCompletionApis[api]]) {
448 return true;454 return true;
449 }455 }
450456
@@ -580,6 +586,14 @@ jQuery(async function () {
580 extension_settings.caption.multimodal_model = String($('#caption_multimodal_model').val());586 extension_settings.caption.multimodal_model = String($('#caption_multimodal_model').val());
581 saveSettingsDebounced();587 saveSettingsDebounced();
582 });588 });
589 $('#caption_altEndpoint_url').val(extension_settings.caption.alt_endpoint_url).on('input', () => {
590 extension_settings.caption.alt_endpoint_url = String($('#caption_altEndpoint_url').val());
591 saveSettingsDebounced();
592 });
593 $('#caption_altEndpoint_enabled').prop('checked', !!(extension_settings.caption.alt_endpoint_enabled)).on('input', () => {
594 extension_settings.caption.alt_endpoint_enabled = !!$('#caption_altEndpoint_enabled').prop('checked');
595 saveSettingsDebounced();
596 });
583597
584 const onMessageEvent = async (index) => {598 const onMessageEvent = async (index) => {
585 if (!extension_settings.caption.auto_mode) {599 if (!extension_settings.caption.auto_mode) {
public/scripts/extensions/caption/settings.html+10 -0
@@ -189,6 +189,16 @@
189 <small><b data-i18n="Hint:">Hint:</b> <span data-i18n="Set your API keys and endpoints in the 'API Connections' tab first.">Set your API keys and endpoints in the 'API Connections' tab first.</span></small>189 <small><b data-i18n="Hint:">Hint:</b> <span data-i18n="Set your API keys and endpoints in the 'API Connections' tab first.">Set your API keys and endpoints in the 'API Connections' tab first.</span></small>
190 </div>190 </div>
191 </div>191 </div>
192 <div data-type="koboldcpp,ollama,vllm,llamacpp,ooba" class="flex-container flexFlowColumn">
193 <label for="caption_altEndpoint_enabled" class="checkbox_label">
194 <input id="caption_altEndpoint_enabled" type="checkbox">
195 <span data-i18n="Use secondary URL">Use secondary URL</span>
196 </label>
197 <label for="caption_altEndpoint_url" data-i18n="Secondary captioning endpoint URL">
198 Secondary captioning endpoint URL
199 </label>
200 <input id="caption_altEndpoint_url" class="text_pole" type="text" placeholder="e.g. http://localhost:5001" />
201 </div>
192 <div id="caption_prompt_block">202 <div id="caption_prompt_block">
193 <label for="caption_prompt" data-i18n="Caption Prompt">Caption Prompt</label>203 <label for="caption_prompt" data-i18n="Caption Prompt">Caption Prompt</label>
194 <textarea id="caption_prompt" class="text_pole" rows="1" placeholder="&lt; Use default &gt;">{{PROMPT_DEFAULT}}</textarea>204 <textarea id="caption_prompt" class="text_pole" rows="1" placeholder="&lt; Use default &gt;">{{PROMPT_DEFAULT}}</textarea>
public/scripts/extensions/shared.js+42 -23
@@ -61,7 +61,9 @@ export async function getMultimodalCaption(base64Img, prompt) {
61 requestBody.model = textgenerationwebui_settings.ollama_model;61 requestBody.model = textgenerationwebui_settings.ollama_model;
62 }62 }
6363
64 requestBody.server_url = textgenerationwebui_settings.server_urls[textgen_types.OLLAMA];64 requestBody.server_url = extension_settings.caption.alt_endpoint_enabled
65 ? extension_settings.caption.alt_endpoint_url
66 : textgenerationwebui_settings.server_urls[textgen_types.OLLAMA];
65 }67 }
6668
67 if (isVllm) {69 if (isVllm) {
@@ -69,19 +71,27 @@ export async function getMultimodalCaption(base64Img, prompt) {
69 requestBody.model = textgenerationwebui_settings.vllm_model;71 requestBody.model = textgenerationwebui_settings.vllm_model;
70 }72 }
7173
72 requestBody.server_url = textgenerationwebui_settings.server_urls[textgen_types.VLLM];74 requestBody.server_url = extension_settings.caption.alt_endpoint_enabled
75 ? extension_settings.caption.alt_endpoint_url
76 : textgenerationwebui_settings.server_urls[textgen_types.VLLM];
73 }77 }
7478
75 if (isLlamaCpp) {79 if (isLlamaCpp) {
76 requestBody.server_url = textgenerationwebui_settings.server_urls[textgen_types.LLAMACPP];80 requestBody.server_url = extension_settings.caption.alt_endpoint_enabled
81 ? extension_settings.caption.alt_endpoint_url
82 : textgenerationwebui_settings.server_urls[textgen_types.LLAMACPP];
77 }83 }
7884
79 if (isOoba) {85 if (isOoba) {
80 requestBody.server_url = textgenerationwebui_settings.server_urls[textgen_types.OOBA];86 requestBody.server_url = extension_settings.caption.alt_endpoint_enabled
87 ? extension_settings.caption.alt_endpoint_url
88 : textgenerationwebui_settings.server_urls[textgen_types.OOBA];
81 }89 }
8290
83 if (isKoboldCpp) {91 if (isKoboldCpp) {
84 requestBody.server_url = textgenerationwebui_settings.server_urls[textgen_types.KOBOLDCPP];92 requestBody.server_url = extension_settings.caption.alt_endpoint_enabled
93 ? extension_settings.caption.alt_endpoint_url
94 : textgenerationwebui_settings.server_urls[textgen_types.KOBOLDCPP];
85 }95 }
8696
87 if (isCustom) {97 if (isCustom) {
@@ -121,75 +131,84 @@ export async function getMultimodalCaption(base64Img, prompt) {
121}131}
122132
123function throwIfInvalidModel(useReverseProxy) {133function throwIfInvalidModel(useReverseProxy) {
124 if (extension_settings.caption.multimodal_api === 'openai' && !secret_state[SECRET_KEYS.OPENAI] && !useReverseProxy) {134 const altEndpointEnabled = extension_settings.caption.alt_endpoint_enabled;
135 const altEndpointUrl = extension_settings.caption.alt_endpoint_url;
136 const multimodalModel = extension_settings.caption.multimodal_model;
137 const multimodalApi = extension_settings.caption.multimodal_api;
138
139 if (altEndpointEnabled && ['llamacpp', 'ooba', 'koboldcpp', 'vllm', 'ollama'].includes(multimodalApi) && !altEndpointUrl) {
140 throw new Error('Secondary endpoint URL is not set.');
141 }
142
143 if (multimodalApi === 'openai' && !secret_state[SECRET_KEYS.OPENAI] && !useReverseProxy) {
125 throw new Error('OpenAI API key is not set.');144 throw new Error('OpenAI API key is not set.');
126 }145 }
127146
128 if (extension_settings.caption.multimodal_api === 'openrouter' && !secret_state[SECRET_KEYS.OPENROUTER]) {147 if (multimodalApi === 'openrouter' && !secret_state[SECRET_KEYS.OPENROUTER]) {
129 throw new Error('OpenRouter API key is not set.');148 throw new Error('OpenRouter API key is not set.');
130 }149 }
131150
132 if (extension_settings.caption.multimodal_api === 'anthropic' && !secret_state[SECRET_KEYS.CLAUDE] && !useReverseProxy) {151 if (multimodalApi === 'anthropic' && !secret_state[SECRET_KEYS.CLAUDE] && !useReverseProxy) {
133 throw new Error('Anthropic (Claude) API key is not set.');152 throw new Error('Anthropic (Claude) API key is not set.');
134 }153 }
135154
136 if (extension_settings.caption.multimodal_api === 'zerooneai' && !secret_state[SECRET_KEYS.ZEROONEAI]) {155 if (multimodalApi === 'zerooneai' && !secret_state[SECRET_KEYS.ZEROONEAI]) {
137 throw new Error('01.AI API key is not set.');156 throw new Error('01.AI API key is not set.');
138 }157 }
139158
140 if (extension_settings.caption.multimodal_api === 'groq' && !secret_state[SECRET_KEYS.GROQ]) {159 if (multimodalApi === 'groq' && !secret_state[SECRET_KEYS.GROQ]) {
141 throw new Error('Groq API key is not set.');160 throw new Error('Groq API key is not set.');
142 }161 }
143162
144 if (extension_settings.caption.multimodal_api === 'google' && !secret_state[SECRET_KEYS.MAKERSUITE] && !useReverseProxy) {163 if (multimodalApi === 'google' && !secret_state[SECRET_KEYS.MAKERSUITE] && !useReverseProxy) {
145 throw new Error('Google AI Studio API key is not set.');164 throw new Error('Google AI Studio API key is not set.');
146 }165 }
147166
148 if (extension_settings.caption.multimodal_api === 'vertexai' && !secret_state[SECRET_KEYS.VERTEXAI] && !useReverseProxy) {167 if (multimodalApi === 'vertexai' && !secret_state[SECRET_KEYS.VERTEXAI] && !useReverseProxy) {
149 throw new Error('Google Vertex AI API key is not set.');168 throw new Error('Google Vertex AI API key is not set.');
150 }169 }
151170
152 if (extension_settings.caption.multimodal_api === 'mistral' && !secret_state[SECRET_KEYS.MISTRALAI] && !useReverseProxy) {171 if (multimodalApi === 'mistral' && !secret_state[SECRET_KEYS.MISTRALAI] && !useReverseProxy) {
153 throw new Error('Mistral AI API key is not set.');172 throw new Error('Mistral AI API key is not set.');
154 }173 }
155174
156 if (extension_settings.caption.multimodal_api === 'cohere' && !secret_state[SECRET_KEYS.COHERE]) {175 if (multimodalApi === 'cohere' && !secret_state[SECRET_KEYS.COHERE]) {
157 throw new Error('Cohere API key is not set.');176 throw new Error('Cohere API key is not set.');
158 }177 }
159178
160 if (extension_settings.caption.multimodal_api === 'xai' && !secret_state[SECRET_KEYS.XAI] && !useReverseProxy) {179 if (multimodalApi === 'xai' && !secret_state[SECRET_KEYS.XAI] && !useReverseProxy) {
161 throw new Error('xAI API key is not set.');180 throw new Error('xAI API key is not set.');
162 }181 }
163182
164 if (extension_settings.caption.multimodal_api === 'ollama' && !textgenerationwebui_settings.server_urls[textgen_types.OLLAMA]) {183 if (multimodalApi === 'ollama' && !textgenerationwebui_settings.server_urls[textgen_types.OLLAMA] && !altEndpointEnabled) {
165 throw new Error('Ollama server URL is not set.');184 throw new Error('Ollama server URL is not set.');
166 }185 }
167186
168 if (extension_settings.caption.multimodal_api === 'ollama' && extension_settings.caption.multimodal_model === 'ollama_current' && !textgenerationwebui_settings.ollama_model) {187 if (multimodalApi === 'ollama' && multimodalModel === 'ollama_current' && !textgenerationwebui_settings.ollama_model) {
169 throw new Error('Ollama model is not set.');188 throw new Error('Ollama model is not set.');
170 }189 }
171190
172 if (extension_settings.caption.multimodal_api === 'llamacpp' && !textgenerationwebui_settings.server_urls[textgen_types.LLAMACPP]) {191 if (multimodalApi === 'llamacpp' && !textgenerationwebui_settings.server_urls[textgen_types.LLAMACPP] && !altEndpointEnabled) {
173 throw new Error('LlamaCPP server URL is not set.');192 throw new Error('LlamaCPP server URL is not set.');
174 }193 }
175194
176 if (extension_settings.caption.multimodal_api === 'ooba' && !textgenerationwebui_settings.server_urls[textgen_types.OOBA]) {195 if (multimodalApi === 'ooba' && !textgenerationwebui_settings.server_urls[textgen_types.OOBA] && !altEndpointEnabled) {
177 throw new Error('Text Generation WebUI server URL is not set.');196 throw new Error('Text Generation WebUI server URL is not set.');
178 }197 }
179198
180 if (extension_settings.caption.multimodal_api === 'koboldcpp' && !textgenerationwebui_settings.server_urls[textgen_types.KOBOLDCPP]) {199 if (multimodalApi === 'koboldcpp' && !textgenerationwebui_settings.server_urls[textgen_types.KOBOLDCPP] && !altEndpointEnabled) {
181 throw new Error('KoboldCpp server URL is not set.');200 throw new Error('KoboldCpp server URL is not set.');
182 }201 }
183202
184 if (extension_settings.caption.multimodal_api === 'vllm' && !textgenerationwebui_settings.server_urls[textgen_types.VLLM]) {203 if (multimodalApi === 'vllm' && !textgenerationwebui_settings.server_urls[textgen_types.VLLM] && !altEndpointEnabled) {
185 throw new Error('vLLM server URL is not set.');204 throw new Error('vLLM server URL is not set.');
186 }205 }
187206
188 if (extension_settings.caption.multimodal_api === 'vllm' && extension_settings.caption.multimodal_model === 'vllm_current' && !textgenerationwebui_settings.vllm_model) {207 if (multimodalApi === 'vllm' && multimodalModel === 'vllm_current' && !textgenerationwebui_settings.vllm_model) {
189 throw new Error('vLLM model is not set.');208 throw new Error('vLLM model is not set.');
190 }209 }
191210
192 if (extension_settings.caption.multimodal_api === 'custom' && !oai_settings.custom_url) {211 if (multimodalApi === 'custom' && !oai_settings.custom_url) {
193 throw new Error('Custom API URL is not set.');212 throw new Error('Custom API URL is not set.');
194 }213 }
195}214}
src/endpoints/openai.js+4 -5
@@ -151,8 +151,11 @@ router.post('/caption-image', async (request, response) => {
151 apiUrl = 'https://text.pollinations.ai/openai/chat/completions';151 apiUrl = 'https://text.pollinations.ai/openai/chat/completions';
152 }152 }
153153
154 if (request.body.api === 'ooba') {154 if (['koboldcpp', 'vllm', 'llamacpp', 'ooba'].includes(request.body.api)) {
155 apiUrl = `${trimV1(request.body.server_url)}/v1/chat/completions`;155 apiUrl = `${trimV1(request.body.server_url)}/v1/chat/completions`;
156 }
157
158 if (request.body.api === 'ooba') {
156 const imgMessage = body.messages.pop();159 const imgMessage = body.messages.pop();
157 body.messages.push({160 body.messages.push({
158 role: 'user',161 role: 'user',
@@ -165,10 +168,6 @@ router.post('/caption-image', async (request, response) => {
165 });168 });
166 }169 }
167170
168 if (['koboldcpp', 'vllm', 'llamacpp'].includes(request.body.api)) {
169 apiUrl = `${trimV1(request.body.server_url)}/v1/chat/completions`;
170 }
171
172 setAdditionalHeaders(request, { headers }, apiUrl);171 setAdditionalHeaders(request, { headers }, apiUrl);
173 console.debug('Multimodal captioning request', body);172 console.debug('Multimodal captioning request', body);
174173