Caption: Improve Ollama models handling

80fa7f96f160105717606536f0e7002ac3a61e33

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: 5e4bb0415e2898560a904f46cce29c6765fc2f71
4 files changed, +42 -12Ignore whitespace
public/scripts/extensions/caption/index.js+8 -2
@@ -588,10 +588,12 @@ jQuery(async function () {
588 saveSettingsDebounced();588 saveSettingsDebounced();
589 });589 });
590 $('#caption_ollama_pull').on('click', (e) => {590 $('#caption_ollama_pull').on('click', (e) => {
591 const presetModel = extension_settings.caption.multimodal_model !== 'ollama_current' ? extension_settings.caption.multimodal_model : '';591 const selectedModel = extension_settings.caption.multimodal_model;
592 const staticModels = { 'ollama_current': textgenerationwebui_settings.ollama_model, 'ollama_custom': extension_settings.caption.ollama_custom_model };
593 const presetModel = staticModels[selectedModel] || selectedModel;
592 e.preventDefault();594 e.preventDefault();
593 $('#ollama_download_model').trigger('click');595 $('#ollama_download_model').trigger('click');
594 $('#dialogue_popup_input').val(presetModel);596 $('.popup .popup-input').val(presetModel);
595 });597 });
596 $('#caption_multimodal_api').on('change', async () => {598 $('#caption_multimodal_api').on('change', async () => {
597 const api = String($('#caption_multimodal_api').val());599 const api = String($('#caption_multimodal_api').val());
@@ -616,6 +618,10 @@ jQuery(async function () {
616 extension_settings.caption.show_in_chat = !!$('#caption_show_in_chat').prop('checked');618 extension_settings.caption.show_in_chat = !!$('#caption_show_in_chat').prop('checked');
617 saveSettingsDebounced();619 saveSettingsDebounced();
618 });620 });
621 $('#caption_ollama_custom_model').val(extension_settings.caption.ollama_custom_model || '').on('input', () => {
622 extension_settings.caption.ollama_custom_model = String($('#caption_ollama_custom_model').val()).trim();
623 saveSettingsDebounced();
624 });
619625
620 const onMessageEvent = async (index) => {626 const onMessageEvent = async (index) => {
621 if (!extension_settings.caption.auto_mode) {627 if (!extension_settings.caption.auto_mode) {
public/scripts/extensions/caption/settings.html+21 -3
@@ -154,11 +154,20 @@
154 <option data-type="groq" value="llama-3.2-90b-vision-preview">llama-3.2-90b-vision-preview</option>154 <option data-type="groq" value="llama-3.2-90b-vision-preview">llama-3.2-90b-vision-preview</option>
155 <option data-type="groq" value="llava-v1.5-7b-4096-preview">llava-v1.5-7b-4096-preview</option>155 <option data-type="groq" value="llava-v1.5-7b-4096-preview">llava-v1.5-7b-4096-preview</option>
156 <option data-type="ollama" value="ollama_current" data-i18n="currently_selected">[Currently selected]</option>156 <option data-type="ollama" value="ollama_current" data-i18n="currently_selected">[Currently selected]</option>
157 <option data-type="ollama" value="ollama_custom" data-i18n="[Custom model]">[Custom model]</option>
157 <option data-type="ollama" value="bakllava">bakllava</option>158 <option data-type="ollama" value="bakllava">bakllava</option>
158 <option data-type="ollama" value="llava">llava</option>159 <option data-type="ollama" value="llava">llava</option>
159 <option data-type="ollama" value="llava-llama3">llava-llama3</option>160 <option data-type="ollama" value="llava-llama3">llava-llama3</option>
160 <option data-type="ollama" value="llava-phi3">llava-phi3</option>161 <option data-type="ollama" value="llava-phi3">llava-phi3</option>
161 <option data-type="ollama" value="moondream">moondream</option>162 <option data-type="ollama" value="moondream">moondream</option>
163 <option data-type="ollama" value="gemma3">gemma3</option>
164 <option data-type="ollama" value="minicpm-v">minicpm-v</option>
165 <option data-type="ollama" value="qwen2.5vl">qwen2.5vl</option>
166 <option data-type="ollama" value="granite3.2-vision">granite3.2-vision</option>
167 <option data-type="ollama" value="mistral-small3.1">mistral-small3.1</option>
168 <option data-type="ollama" value="mistral-small3.2">mistral-small3.2</option>
169 <option data-type="ollama" value="llama3.2-vision">llama3.2-vision</option>
170 <option data-type="ollama" value="llama4">llama4</option>
162 <option data-type="llamacpp" value="llamacpp_current" data-i18n="currently_loaded">[Currently loaded]</option>171 <option data-type="llamacpp" value="llamacpp_current" data-i18n="currently_loaded">[Currently loaded]</option>
163 <option data-type="ooba" value="ooba_current" data-i18n="currently_loaded">[Currently loaded]</option>172 <option data-type="ooba" value="ooba_current" data-i18n="currently_loaded">[Currently loaded]</option>
164 <option data-type="koboldcpp" value="koboldcpp_current" data-i18n="currently_loaded">[Currently loaded]</option>173 <option data-type="koboldcpp" value="koboldcpp_current" data-i18n="currently_loaded">[Currently loaded]</option>
@@ -170,16 +179,25 @@
170 </select>179 </select>
171 </div>180 </div>
172 <div data-type="ollama">181 <div data-type="ollama">
173 The model must be downloaded first! Do it with the <code>ollama pull</code> command or <a href="#" id="caption_ollama_pull">click here</a>.182 <div>
183 The model must be downloaded first! Do it with the <code>ollama pull</code> command or <a href="#" id="caption_ollama_pull">click here</a>.
184 </div>
185 <div class="marginTop5">
186 <label for="caption_ollama_custom_model">
187 <span data-i18n="Custom Model Tag">Custom Model Tag</span>
188 <small data-i18n="(for [Custom model] option)">(for [Custom model] option)</small>
189 </label>
190 <input id="caption_ollama_custom_model" class="text_pole" type="text" placeholder="e.g. gemma3:latest" />
191 </div>
174 </div>192 </div>
175 <label data-type="openai,anthropic,google,vertexai,mistral,xai" class="checkbox_label flexBasis100p" for="caption_allow_reverse_proxy" title="Allow using reverse proxy if defined and valid.">193 <label data-type="openai,anthropic,google,vertexai,mistral,xai" class="checkbox_label flexBasis100p" for="caption_allow_reverse_proxy" title="Allow using reverse proxy if defined and valid.">
176 <input id="caption_allow_reverse_proxy" type="checkbox" class="checkbox">194 <input id="caption_allow_reverse_proxy" type="checkbox" class="checkbox">
177 <span data-i18n="Allow reverse proxy">Allow reverse proxy</span>195 <span data-i18n="Allow reverse proxy">Allow reverse proxy</span>
178 </label>196 </label>
179 <div class="flexBasis100p m-b-1">197 <div class="flexBasis100p marginBot10">
180 <small><b data-i18n="Hint:">Hint:</b> <span data-i18n="Set your API keys and endpoints in the 'API Connections' tab first.">Set your API keys and endpoints in the 'API Connections' tab first.</span></small>198 <small><b data-i18n="Hint:">Hint:</b> <span data-i18n="Set your API keys and endpoints in the 'API Connections' tab first.">Set your API keys and endpoints in the 'API Connections' tab first.</span></small>
181 </div>199 </div>
182 <div data-type="koboldcpp,ollama,vllm,llamacpp,ooba" class="flex-container flexFlowColumn">200 <div data-type="koboldcpp,ollama,vllm,llamacpp,ooba" class="flex-container flexFlowColumn wide100p">
183 <label for="caption_altEndpoint_enabled" class="checkbox_label">201 <label for="caption_altEndpoint_enabled" class="checkbox_label">
184 <input id="caption_altEndpoint_enabled" type="checkbox">202 <input id="caption_altEndpoint_enabled" type="checkbox">
185 <span data-i18n="Use secondary URL">Use secondary URL</span>203 <span data-i18n="Use secondary URL">Use secondary URL</span>
public/scripts/extensions/shared.js+11 -6
@@ -22,12 +22,6 @@ export async function getMultimodalCaption(base64Img, prompt) {
2222
23 throwIfInvalidModel(useReverseProxy);23 throwIfInvalidModel(useReverseProxy);
2424
25 const noPrefix = ['ollama'].includes(extension_settings.caption.multimodal_api);
26
27 if (noPrefix && base64Img.startsWith('data:image/')) {
28 base64Img = base64Img.split(',')[1];
29 }
30
31 // OpenRouter has a payload limit of ~2MB. Google is 4MB, but we love democracy.25 // OpenRouter has a payload limit of ~2MB. Google is 4MB, but we love democracy.
32 // Ooba requires all images to be JPEGs. Koboldcpp just asked nicely.26 // Ooba requires all images to be JPEGs. Koboldcpp just asked nicely.
33 const isOllama = extension_settings.caption.multimodal_api === 'ollama';27 const isOllama = extension_settings.caption.multimodal_api === 'ollama';
@@ -47,6 +41,9 @@ export async function getMultimodalCaption(base64Img, prompt) {
47 } else if (!safeMimeTypes.includes(mimeType)) {41 } else if (!safeMimeTypes.includes(mimeType)) {
48 base64Img = await createThumbnail(base64Img, null, null);42 base64Img = await createThumbnail(base64Img, null, null);
49 }43 }
44 if (isOllama && base64Img.startsWith('data:image/')) {
45 base64Img = base64Img.split(',')[1];
46 }
5047
51 const proxyUrl = useReverseProxy ? oai_settings.reverse_proxy : '';48 const proxyUrl = useReverseProxy ? oai_settings.reverse_proxy : '';
52 const proxyPassword = useReverseProxy ? oai_settings.proxy_password : '';49 const proxyPassword = useReverseProxy ? oai_settings.proxy_password : '';
@@ -72,6 +69,10 @@ export async function getMultimodalCaption(base64Img, prompt) {
72 requestBody.model = textgenerationwebui_settings.ollama_model;69 requestBody.model = textgenerationwebui_settings.ollama_model;
73 }70 }
7471
72 if (extension_settings.caption.multimodal_model === 'ollama_custom') {
73 requestBody.model = extension_settings.caption.ollama_custom_model;
74 }
75
75 requestBody.server_url = extension_settings.caption.alt_endpoint_enabled76 requestBody.server_url = extension_settings.caption.alt_endpoint_enabled
76 ? extension_settings.caption.alt_endpoint_url77 ? extension_settings.caption.alt_endpoint_url
77 : textgenerationwebui_settings.server_urls[textgen_types.OLLAMA];78 : textgenerationwebui_settings.server_urls[textgen_types.OLLAMA];
@@ -211,6 +212,10 @@ function throwIfInvalidModel(useReverseProxy) {
211 throw new Error('Ollama model is not set.');212 throw new Error('Ollama model is not set.');
212 }213 }
213214
215 if (multimodalApi === 'ollama' && multimodalModel === 'ollama_custom' && !extension_settings.caption.ollama_custom_model) {
216 throw new Error('Ollama custom model tag is not set.');
217 }
218
214 if (multimodalApi === 'llamacpp' && !textgenerationwebui_settings.server_urls[textgen_types.LLAMACPP] && !altEndpointEnabled) {219 if (multimodalApi === 'llamacpp' && !textgenerationwebui_settings.server_urls[textgen_types.LLAMACPP] && !altEndpointEnabled) {
215 throw new Error('LlamaCPP server URL is not set.');220 throw new Error('LlamaCPP server URL is not set.');
216 }221 }
src/endpoints/backends/text-completions.js+2 -1
@@ -478,7 +478,8 @@ ollama.post('/caption-image', async function (request, response) {
478 });478 });
479479
480 if (!fetchResponse.ok) {480 if (!fetchResponse.ok) {
481 console.error('Ollama caption error:', fetchResponse.status, fetchResponse.statusText);481 const errorText = await fetchResponse.text();
482 console.error('Ollama caption error:', fetchResponse.status, fetchResponse.statusText, errorText);
482 return response.status(500).send({ error: true });483 return response.status(500).send({ error: true });
483 }484 }
484485