Caption: Improve Ollama models handling

80fa7f96f160105717606536f0e7002ac3a61e33

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: 5e4bb0415e2898560a904f46cce29c6765fc2f71
4 files changed, +42 -12Ignore whitespace
public/scripts/extensions/caption/index.js+8 -2
@@ -588,10 +588,12 @@ jQuery(async function () {
588588 saveSettingsDebounced();
589589 });
590590 $('#caption_ollama_pull').on('click', (e) => {
591591 const presetModel = extension_settings.caption.multimodal_modelselectedModel !== 'ollama_current' ? extension_settings.caption.multimodal_model : '';
592+ const staticModels = { 'ollama_current': textgenerationwebui_settings.ollama_model, 'ollama_custom': extension_settings.caption.ollama_custom_model };
593+ const presetModel = staticModels[selectedModel] || selectedModel;
592594 e.preventDefault();
593595 $('#ollama_download_model').trigger('click');
594596 $('#dialogue_popup_input.popup .popup-input').val(presetModel);
595597 });
596598 $('#caption_multimodal_api').on('change', async () => {
597599 const api = String($('#caption_multimodal_api').val());
@@ -616,6 +618,10 @@ jQuery(async function () {
616618 extension_settings.caption.show_in_chat = !!$('#caption_show_in_chat').prop('checked');
617619 saveSettingsDebounced();
618620 });
621+ $('#caption_ollama_custom_model').val(extension_settings.caption.ollama_custom_model || '').on('input', () => {
622+ extension_settings.caption.ollama_custom_model = String($('#caption_ollama_custom_model').val()).trim();
623+ saveSettingsDebounced();
624+ });
619625
620626 const onMessageEvent = async (index) => {
621627 if (!extension_settings.caption.auto_mode) {
public/scripts/extensions/caption/settings.html+21 -3
@@ -154,11 +154,20 @@
154154 <option data-type="groq" value="llama-3.2-90b-vision-preview">llama-3.2-90b-vision-preview</option>
155155 <option data-type="groq" value="llava-v1.5-7b-4096-preview">llava-v1.5-7b-4096-preview</option>
156156 <option data-type="ollama" value="ollama_current" data-i18n="currently_selected">[Currently selected]</option>
157+ <option data-type="ollama" value="ollama_custom" data-i18n="[Custom model]">[Custom model]</option>
157158 <option data-type="ollama" value="bakllava">bakllava</option>
158159 <option data-type="ollama" value="llava">llava</option>
159160 <option data-type="ollama" value="llava-llama3">llava-llama3</option>
160161 <option data-type="ollama" value="llava-phi3">llava-phi3</option>
161162 <option data-type="ollama" value="moondream">moondream</option>
163+ <option data-type="ollama" value="gemma3">gemma3</option>
164+ <option data-type="ollama" value="minicpm-v">minicpm-v</option>
165+ <option data-type="ollama" value="qwen2.5vl">qwen2.5vl</option>
166+ <option data-type="ollama" value="granite3.2-vision">granite3.2-vision</option>
167+ <option data-type="ollama" value="mistral-small3.1">mistral-small3.1</option>
168+ <option data-type="ollama" value="mistral-small3.2">mistral-small3.2</option>
169+ <option data-type="ollama" value="llama3.2-vision">llama3.2-vision</option>
170+ <option data-type="ollama" value="llama4">llama4</option>
162171 <option data-type="llamacpp" value="llamacpp_current" data-i18n="currently_loaded">[Currently loaded]</option>
163172 <option data-type="ooba" value="ooba_current" data-i18n="currently_loaded">[Currently loaded]</option>
164173 <option data-type="koboldcpp" value="koboldcpp_current" data-i18n="currently_loaded">[Currently loaded]</option>
@@ -170,16 +179,25 @@
170179 </select>
171180 </div>
172181 <div data-type="ollama">
173- The model must be downloaded first! Do it with the <code>ollama pull</code> command or <a href="#" id="caption_ollama_pull">click here</a>.
182+ <div>
183+ The model must be downloaded first! Do it with the <code>ollama pull</code> command or <a href="#" id="caption_ollama_pull">click here</a>.
184+ </div>
185+ <div class="marginTop5">
186+ <label for="caption_ollama_custom_model">
187+ <span data-i18n="Custom Model Tag">Custom Model Tag</span>
188+ <small data-i18n="(for [Custom model] option)">(for [Custom model] option)</small>
189+ </label>
190+ <input id="caption_ollama_custom_model" class="text_pole" type="text" placeholder="e.g. gemma3:latest" />
191+ </div>
174192 </div>
175193 <label data-type="openai,anthropic,google,vertexai,mistral,xai" class="checkbox_label flexBasis100p" for="caption_allow_reverse_proxy" title="Allow using reverse proxy if defined and valid.">
176194 <input id="caption_allow_reverse_proxy" type="checkbox" class="checkbox">
177195 <span data-i18n="Allow reverse proxy">Allow reverse proxy</span>
178196 </label>
179197 <div class="flexBasis100p m-b-1marginBot10">
180198 <small><b data-i18n="Hint:">Hint:</b> <span data-i18n="Set your API keys and endpoints in the 'API Connections' tab first.">Set your API keys and endpoints in the 'API Connections' tab first.</span></small>
181199 </div>
182200 <div data-type="koboldcpp,ollama,vllm,llamacpp,ooba" class="flex-container flexFlowColumn wide100p">
183201 <label for="caption_altEndpoint_enabled" class="checkbox_label">
184202 <input id="caption_altEndpoint_enabled" type="checkbox">
185203 <span data-i18n="Use secondary URL">Use secondary URL</span>
public/scripts/extensions/shared.js+11 -6
@@ -22,12 +22,6 @@ export async function getMultimodalCaption(base64Img, prompt) {
2222
2323 throwIfInvalidModel(useReverseProxy);
2424
25- const noPrefix = ['ollama'].includes(extension_settings.caption.multimodal_api);
26-
27- if (noPrefix && base64Img.startsWith('data:image/')) {
28- base64Img = base64Img.split(',')[1];
29- }
30-
3125 // OpenRouter has a payload limit of ~2MB. Google is 4MB, but we love democracy.
3226 // Ooba requires all images to be JPEGs. Koboldcpp just asked nicely.
3327 const isOllama = extension_settings.caption.multimodal_api === 'ollama';
@@ -47,6 +41,9 @@ export async function getMultimodalCaption(base64Img, prompt) {
4741 } else if (!safeMimeTypes.includes(mimeType)) {
4842 base64Img = await createThumbnail(base64Img, null, null);
4943 }
44+ if (isOllama && base64Img.startsWith('data:image/')) {
45+ base64Img = base64Img.split(',')[1];
46+ }
5047
5148 const proxyUrl = useReverseProxy ? oai_settings.reverse_proxy : '';
5249 const proxyPassword = useReverseProxy ? oai_settings.proxy_password : '';
@@ -72,6 +69,10 @@ export async function getMultimodalCaption(base64Img, prompt) {
7269 requestBody.model = textgenerationwebui_settings.ollama_model;
7370 }
7471
72+ if (extension_settings.caption.multimodal_model === 'ollama_custom') {
73+ requestBody.model = extension_settings.caption.ollama_custom_model;
74+ }
75+
7576 requestBody.server_url = extension_settings.caption.alt_endpoint_enabled
7677 ? extension_settings.caption.alt_endpoint_url
7778 : textgenerationwebui_settings.server_urls[textgen_types.OLLAMA];
@@ -211,6 +212,10 @@ function throwIfInvalidModel(useReverseProxy) {
211212 throw new Error('Ollama model is not set.');
212213 }
213214
215+ if (multimodalApi === 'ollama' && multimodalModel === 'ollama_custom' && !extension_settings.caption.ollama_custom_model) {
216+ throw new Error('Ollama custom model tag is not set.');
217+ }
218+
214219 if (multimodalApi === 'llamacpp' && !textgenerationwebui_settings.server_urls[textgen_types.LLAMACPP] && !altEndpointEnabled) {
215220 throw new Error('LlamaCPP server URL is not set.');
216221 }
src/endpoints/backends/text-completions.js+2 -1
@@ -478,7 +478,8 @@ ollama.post('/caption-image', async function (request, response) {
478478 });
479479
480480 if (!fetchResponse.ok) {
481- console.error('Ollama caption error:', fetchResponse.status, fetchResponse.statusText);
481+ const errorText = await fetchResponse.text();
482+ console.error('Ollama caption error:', fetchResponse.status, fetchResponse.statusText, errorText);
482483 return response.status(500).send({ error: true });
483484 }
484485