Add multimodal captioning for Cohere

c167890d266e88b93343ada6734d0886b94757de

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: a09b9fa74640494e0a053b665158240184dc3b1c
4 files changed, +73 -19Showing whitespace changes
public/scripts/extensions/caption/index.js+56 -17
@@ -398,23 +398,62 @@ jQuery(async function () {
398398
399399 $('#caption_wand_container').append(sendButton);
400400 $(sendButton).on('click', () => {
401401 const hasCaptionModule = (() => {
402- (modules.includes('caption') && extension_settings.caption.source === 'extras') ||
402+ const settings = extension_settings.caption;
403- (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'openai' && (secret_state[SECRET_KEYS.OPENAI] || extension_settings.caption.allow_reverse_proxy)) ||
403+
404- (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'openrouter' && secret_state[SECRET_KEYS.OPENROUTER]) ||
404+ // Handle non-multimodal sources
405405 if (extension_settings.captionsettings.source === 'multimodalextras' && extension_settings.captionmodules.multimodal_api === includes('zerooneaicaption' && secret_state[SECRET_KEYS.ZEROONEAI])) ||return true;
406406 if (extension_settings.captionsettings.source === 'multimodallocal' &&|| extension_settings.captionsettings.multimodal_apisource === 'groqhorde' && secret_state[SECRET_KEYS.GROQ]) ||return true;
407- (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'mistral' && (secret_state[SECRET_KEYS.MISTRALAI] || extension_settings.caption.allow_reverse_proxy)) ||
407+
408- (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'google' && (secret_state[SECRET_KEYS.MAKERSUITE] || extension_settings.caption.allow_reverse_proxy)) ||
408+ // Handle multimodal sources
409- (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'anthropic' && (secret_state[SECRET_KEYS.CLAUDE] || extension_settings.caption.allow_reverse_proxy)) ||
409+ if (settings.source === 'multimodal') {
410- (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'ollama' && textgenerationwebui_settings.server_urls[textgen_types.OLLAMA]) ||
410+ const api = settings.multimodal_api;
411- (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'llamacpp' && textgenerationwebui_settings.server_urls[textgen_types.LLAMACPP]) ||
411+
412- (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'ooba' && textgenerationwebui_settings.server_urls[textgen_types.OOBA]) ||
412+ // APIs that support reverse proxy
413- (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'koboldcpp' && textgenerationwebui_settings.server_urls[textgen_types.KOBOLDCPP]) ||
413+ const reverseProxyApis = {
414- (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'vllm' && textgenerationwebui_settings.server_urls[textgen_types.VLLM]) ||
414+ 'openai': SECRET_KEYS.OPENAI,
415- (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'custom') ||
415+ 'mistral': SECRET_KEYS.MISTRALAI,
416- extension_settings.caption.source === 'local' ||
416+ 'google': SECRET_KEYS.MAKERSUITE,
417- extension_settings.caption.source === 'horde';
417+ 'anthropic': SECRET_KEYS.CLAUDE,
418+ };
419+
420+ if (reverseProxyApis[api]) {
421+ if (secret_state[reverseProxyApis[api]] || settings.allow_reverse_proxy) {
422+ return true;
423+ }
424+ }
425+
426+ const chatCompletionApis = {
427+ 'openrouter': SECRET_KEYS.OPENROUTER,
428+ 'zerooneai': SECRET_KEYS.ZEROONEAI,
429+ 'groq': SECRET_KEYS.GROQ,
430+ 'cohere': SECRET_KEYS.COHERE,
431+ };
432+
433+ if (chatCompletionApis[api] && secret_state[chatCompletionApis[api]]) {
434+ return true;
435+ }
436+
437+ const textCompletionApis = {
438+ 'ollama': textgen_types.OLLAMA,
439+ 'llamacpp': textgen_types.LLAMACPP,
440+ 'ooba': textgen_types.OOBA,
441+ 'koboldcpp': textgen_types.KOBOLDCPP,
442+ 'vllm': textgen_types.VLLM,
443+ };
444+
445+ if (textCompletionApis[api] && textgenerationwebui_settings.server_urls[textCompletionApis[api]]) {
446+ return true;
447+ }
448+
449+ // Custom API doesn't need additional checks
450+ if (api === 'custom') {
451+ return true;
452+ }
453+ }
454+
455+ return false;
456+ })();
418457
419458 if (!hasCaptionModule) {
420459 toastr.error('Choose other captioning source in the extension settings.', 'Captioning is not available');
public/scripts/extensions/caption/settings.html+3 -0
@@ -19,6 +19,7 @@
1919 <select id="caption_multimodal_api" class="flex1 text_pole">
2020 <option value="zerooneai">01.AI (Yi)</option>
2121 <option value="anthropic">Anthropic</option>
22+ <option value="cohere">Cohere</option>
2223 <option value="custom" data-i18n="Custom (OpenAI-compatible)">Custom (OpenAI-compatible)</option>
2324 <option value="google">Google AI Studio</option>
2425 <option value="groq">Groq</option>
@@ -35,6 +36,8 @@
3536 <div class="flex1 flex-container flexFlowColumn flexNoGap">
3637 <label for="caption_multimodal_model" data-i18n="Model">Model</label>
3738 <select id="caption_multimodal_model" class="flex1 text_pole">
39+ <option data-type="cohere" value="c4ai-aya-vision-8b">c4ai-aya-vision-8b</option>
40+ <option data-type="cohere" value="c4ai-aya-vision-32b">c4ai-aya-vision-32b</option>
3841 <option data-type="mistral" value="pixtral-12b-latest">pixtral-12b-latest</option>
3942 <option data-type="mistral" value="pixtral-12b-2409">pixtral-12b-2409</option>
4043 <option data-type="mistral" value="pixtral-large-latest">pixtral-large-latest</option>
public/scripts/extensions/shared.js+5 -1
@@ -144,10 +144,14 @@ function throwIfInvalidModel(useReverseProxy) {
144144 throw new Error('Google AI Studio API key is not set.');
145145 }
146146
147147 if (extension_settings.caption.multi_modal_apimultimodal_api === 'mistral' && !secret_state[SECRET_KEYS.MISTRALAI] && !useReverseProxy) {
148148 throw new Error('Mistral AI API key is not set.');
149149 }
150150
151+ if (extension_settings.caption.multimodal_api === 'cohere' && !secret_state[SECRET_KEYS.COHERE]) {
152+ throw new Error('Cohere API key is not set.');
153+ }
154+
151155 if (extension_settings.caption.multimodal_api === 'ollama' && !textgenerationwebui_settings.server_urls[textgen_types.OLLAMA]) {
152156 throw new Error('Ollama server URL is not set.');
153157 }
src/endpoints/openai.js+9 -1
@@ -62,6 +62,10 @@ router.post('/caption-image', jsonParser, async (request, response) => {
6262 key = readSecret(request.user.directories, SECRET_KEYS.GROQ);
6363 }
6464
65+ if (request.body.api === 'cohere') {
66+ key = readSecret(request.user.directories, SECRET_KEYS.COHERE);
67+ }
68+
6569 if (!key && !request.body.reverse_proxy && ['custom', 'ooba', 'koboldcpp', 'vllm'].includes(request.body.api) === false) {
6670 console.warn('No key found for API', request.body.api);
6771 return response.sendStatus(400);
@@ -126,6 +130,10 @@ router.post('/caption-image', jsonParser, async (request, response) => {
126130 apiUrl = 'https://api.mistral.ai/v1/chat/completions';
127131 }
128132
133+ if (request.body.api === 'cohere') {
134+ apiUrl = 'https://api.cohere.ai/v2/chat';
135+ }
136+
129137 if (request.body.api === 'ooba') {
130138 apiUrl = `${trimV1(request.body.server_url)}/v1/chat/completions`;
131139 const imgMessage = body.messages.pop();
@@ -165,7 +173,7 @@ router.post('/caption-image', jsonParser, async (request, response) => {
165173 /** @type {any} */
166174 const data = await result.json();
167175 console.info('Multimodal captioning response', data);
168176 const caption = data?.choices?.[0]?.message?.content ?? data?.message?.content?.[0]?.text;
169177
170178 if (!caption) {
171179 return response.status(500).send('No caption found');