Add multimodal captioning for Cohere

c167890d266e88b93343ada6734d0886b94757de

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: a09b9fa74640494e0a053b665158240184dc3b1c
4 files changed, +73 -19Showing whitespace changes
public/scripts/extensions/caption/index.js+56 -17
@@ -398,23 +398,62 @@ jQuery(async function () {
398398
399 $('#caption_wand_container').append(sendButton);399 $('#caption_wand_container').append(sendButton);
400 $(sendButton).on('click', () => {400 $(sendButton).on('click', () => {
401 const hasCaptionModule =401 const hasCaptionModule = (() => {
402 (modules.includes('caption') && extension_settings.caption.source === 'extras') ||402 const settings = extension_settings.caption;
403 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'openai' && (secret_state[SECRET_KEYS.OPENAI] || extension_settings.caption.allow_reverse_proxy)) ||403
404 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'openrouter' && secret_state[SECRET_KEYS.OPENROUTER]) ||404 // Handle non-multimodal sources
405 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'zerooneai' && secret_state[SECRET_KEYS.ZEROONEAI]) ||405 if (settings.source === 'extras' && modules.includes('caption')) return true;
406 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'groq' && secret_state[SECRET_KEYS.GROQ]) ||406 if (settings.source === 'local' || settings.source === 'horde') return true;
407 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'mistral' && (secret_state[SECRET_KEYS.MISTRALAI] || extension_settings.caption.allow_reverse_proxy)) ||407
408 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'google' && (secret_state[SECRET_KEYS.MAKERSUITE] || extension_settings.caption.allow_reverse_proxy)) ||408 // Handle multimodal sources
409 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'anthropic' && (secret_state[SECRET_KEYS.CLAUDE] || extension_settings.caption.allow_reverse_proxy)) ||409 if (settings.source === 'multimodal') {
410 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'ollama' && textgenerationwebui_settings.server_urls[textgen_types.OLLAMA]) ||410 const api = settings.multimodal_api;
411 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'llamacpp' && textgenerationwebui_settings.server_urls[textgen_types.LLAMACPP]) ||411
412 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'ooba' && textgenerationwebui_settings.server_urls[textgen_types.OOBA]) ||412 // APIs that support reverse proxy
413 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'koboldcpp' && textgenerationwebui_settings.server_urls[textgen_types.KOBOLDCPP]) ||413 const reverseProxyApis = {
414 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'vllm' && textgenerationwebui_settings.server_urls[textgen_types.VLLM]) ||414 'openai': SECRET_KEYS.OPENAI,
415 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'custom') ||415 'mistral': SECRET_KEYS.MISTRALAI,
416 extension_settings.caption.source === 'local' ||416 'google': SECRET_KEYS.MAKERSUITE,
417 extension_settings.caption.source === 'horde';417 'anthropic': SECRET_KEYS.CLAUDE,
418 };
419
420 if (reverseProxyApis[api]) {
421 if (secret_state[reverseProxyApis[api]] || settings.allow_reverse_proxy) {
422 return true;
423 }
424 }
425
426 const chatCompletionApis = {
427 'openrouter': SECRET_KEYS.OPENROUTER,
428 'zerooneai': SECRET_KEYS.ZEROONEAI,
429 'groq': SECRET_KEYS.GROQ,
430 'cohere': SECRET_KEYS.COHERE,
431 };
432
433 if (chatCompletionApis[api] && secret_state[chatCompletionApis[api]]) {
434 return true;
435 }
436
437 const textCompletionApis = {
438 'ollama': textgen_types.OLLAMA,
439 'llamacpp': textgen_types.LLAMACPP,
440 'ooba': textgen_types.OOBA,
441 'koboldcpp': textgen_types.KOBOLDCPP,
442 'vllm': textgen_types.VLLM,
443 };
444
445 if (textCompletionApis[api] && textgenerationwebui_settings.server_urls[textCompletionApis[api]]) {
446 return true;
447 }
448
449 // Custom API doesn't need additional checks
450 if (api === 'custom') {
451 return true;
452 }
453 }
454
455 return false;
456 })();
418457
419 if (!hasCaptionModule) {458 if (!hasCaptionModule) {
420 toastr.error('Choose other captioning source in the extension settings.', 'Captioning is not available');459 toastr.error('Choose other captioning source in the extension settings.', 'Captioning is not available');
public/scripts/extensions/caption/settings.html+3 -0
@@ -19,6 +19,7 @@
19 <select id="caption_multimodal_api" class="flex1 text_pole">19 <select id="caption_multimodal_api" class="flex1 text_pole">
20 <option value="zerooneai">01.AI (Yi)</option>20 <option value="zerooneai">01.AI (Yi)</option>
21 <option value="anthropic">Anthropic</option>21 <option value="anthropic">Anthropic</option>
22 <option value="cohere">Cohere</option>
22 <option value="custom" data-i18n="Custom (OpenAI-compatible)">Custom (OpenAI-compatible)</option>23 <option value="custom" data-i18n="Custom (OpenAI-compatible)">Custom (OpenAI-compatible)</option>
23 <option value="google">Google AI Studio</option>24 <option value="google">Google AI Studio</option>
24 <option value="groq">Groq</option>25 <option value="groq">Groq</option>
@@ -35,6 +36,8 @@
35 <div class="flex1 flex-container flexFlowColumn flexNoGap">36 <div class="flex1 flex-container flexFlowColumn flexNoGap">
36 <label for="caption_multimodal_model" data-i18n="Model">Model</label>37 <label for="caption_multimodal_model" data-i18n="Model">Model</label>
37 <select id="caption_multimodal_model" class="flex1 text_pole">38 <select id="caption_multimodal_model" class="flex1 text_pole">
39 <option data-type="cohere" value="c4ai-aya-vision-8b">c4ai-aya-vision-8b</option>
40 <option data-type="cohere" value="c4ai-aya-vision-32b">c4ai-aya-vision-32b</option>
38 <option data-type="mistral" value="pixtral-12b-latest">pixtral-12b-latest</option>41 <option data-type="mistral" value="pixtral-12b-latest">pixtral-12b-latest</option>
39 <option data-type="mistral" value="pixtral-12b-2409">pixtral-12b-2409</option>42 <option data-type="mistral" value="pixtral-12b-2409">pixtral-12b-2409</option>
40 <option data-type="mistral" value="pixtral-large-latest">pixtral-large-latest</option>43 <option data-type="mistral" value="pixtral-large-latest">pixtral-large-latest</option>
public/scripts/extensions/shared.js+5 -1
@@ -144,10 +144,14 @@ function throwIfInvalidModel(useReverseProxy) {
144 throw new Error('Google AI Studio API key is not set.');144 throw new Error('Google AI Studio API key is not set.');
145 }145 }
146146
147 if (extension_settings.caption.multi_modal_api === 'mistral' && !secret_state[SECRET_KEYS.MISTRALAI] && !useReverseProxy) {147 if (extension_settings.caption.multimodal_api === 'mistral' && !secret_state[SECRET_KEYS.MISTRALAI] && !useReverseProxy) {
148 throw new Error('Mistral AI API key is not set.');148 throw new Error('Mistral AI API key is not set.');
149 }149 }
150150
151 if (extension_settings.caption.multimodal_api === 'cohere' && !secret_state[SECRET_KEYS.COHERE]) {
152 throw new Error('Cohere API key is not set.');
153 }
154
151 if (extension_settings.caption.multimodal_api === 'ollama' && !textgenerationwebui_settings.server_urls[textgen_types.OLLAMA]) {155 if (extension_settings.caption.multimodal_api === 'ollama' && !textgenerationwebui_settings.server_urls[textgen_types.OLLAMA]) {
152 throw new Error('Ollama server URL is not set.');156 throw new Error('Ollama server URL is not set.');
153 }157 }
src/endpoints/openai.js+9 -1
@@ -62,6 +62,10 @@ router.post('/caption-image', jsonParser, async (request, response) => {
62 key = readSecret(request.user.directories, SECRET_KEYS.GROQ);62 key = readSecret(request.user.directories, SECRET_KEYS.GROQ);
63 }63 }
6464
65 if (request.body.api === 'cohere') {
66 key = readSecret(request.user.directories, SECRET_KEYS.COHERE);
67 }
68
65 if (!key && !request.body.reverse_proxy && ['custom', 'ooba', 'koboldcpp', 'vllm'].includes(request.body.api) === false) {69 if (!key && !request.body.reverse_proxy && ['custom', 'ooba', 'koboldcpp', 'vllm'].includes(request.body.api) === false) {
66 console.warn('No key found for API', request.body.api);70 console.warn('No key found for API', request.body.api);
67 return response.sendStatus(400);71 return response.sendStatus(400);
@@ -126,6 +130,10 @@ router.post('/caption-image', jsonParser, async (request, response) => {
126 apiUrl = 'https://api.mistral.ai/v1/chat/completions';130 apiUrl = 'https://api.mistral.ai/v1/chat/completions';
127 }131 }
128132
133 if (request.body.api === 'cohere') {
134 apiUrl = 'https://api.cohere.ai/v2/chat';
135 }
136
129 if (request.body.api === 'ooba') {137 if (request.body.api === 'ooba') {
130 apiUrl = `${trimV1(request.body.server_url)}/v1/chat/completions`;138 apiUrl = `${trimV1(request.body.server_url)}/v1/chat/completions`;
131 const imgMessage = body.messages.pop();139 const imgMessage = body.messages.pop();
@@ -165,7 +173,7 @@ router.post('/caption-image', jsonParser, async (request, response) => {
165 /** @type {any} */173 /** @type {any} */
166 const data = await result.json();174 const data = await result.json();
167 console.info('Multimodal captioning response', data);175 console.info('Multimodal captioning response', data);
168 const caption = data?.choices[0]?.message?.content;176 const caption = data?.choices?.[0]?.message?.content ?? data?.message?.content?.[0]?.text;
169177
170 if (!caption) {178 if (!caption) {
171 return response.status(500).send('No caption found');179 return response.status(500).send('No caption found');