Add xAI for image captioning

17cdc78a9194450902101b0f8dcd38d7fc5eb677

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: 1c52099ed6d8f964bfcc69a3571ccdb22ba10e78
5 files changed, +17 -1Showing whitespace changes
public/index.html+1 -1
@@ -1987,7 +1987,7 @@
1987 <code><i class="fa-solid fa-wand-magic-sparkles"></i></code>1987 <code><i class="fa-solid fa-wand-magic-sparkles"></i></code>
1988 <span data-i18n="image_inlining_hint_3">menu to attach an image file to the chat.</span>1988 <span data-i18n="image_inlining_hint_3">menu to attach an image file to the chat.</span>
1989 </div>1989 </div>
1990 <div class="flex-container flexFlowColumn wide100p textAlignCenter marginTop10" data-source="openai,custom">1990 <div class="flex-container flexFlowColumn wide100p textAlignCenter marginTop10" data-source="openai,custom,xai">
1991 <div class="flex-container oneline-dropdown">1991 <div class="flex-container oneline-dropdown">
1992 <label for="openai_inline_image_quality" data-i18n="Inline Image Quality">1992 <label for="openai_inline_image_quality" data-i18n="Inline Image Quality">
1993 Inline Image Quality1993 Inline Image Quality
public/scripts/extensions/caption/index.js+1 -0
@@ -428,6 +428,7 @@ jQuery(async function () {
428 'zerooneai': SECRET_KEYS.ZEROONEAI,428 'zerooneai': SECRET_KEYS.ZEROONEAI,
429 'groq': SECRET_KEYS.GROQ,429 'groq': SECRET_KEYS.GROQ,
430 'cohere': SECRET_KEYS.COHERE,430 'cohere': SECRET_KEYS.COHERE,
431 'xai': SECRET_KEYS.XAI,
431 };432 };
432433
433 if (chatCompletionApis[api] && secret_state[chatCompletionApis[api]]) {434 if (chatCompletionApis[api] && secret_state[chatCompletionApis[api]]) {
public/scripts/extensions/caption/settings.html+3 -0
@@ -31,6 +31,7 @@
31 <option value="openrouter">OpenRouter</option>31 <option value="openrouter">OpenRouter</option>
32 <option value="ooba" data-i18n="Text Generation WebUI (oobabooga)">Text Generation WebUI (oobabooga)</option>32 <option value="ooba" data-i18n="Text Generation WebUI (oobabooga)">Text Generation WebUI (oobabooga)</option>
33 <option value="vllm">vLLM</option>33 <option value="vllm">vLLM</option>
34 <option value="xai">xAI (Grok)</option>
34 </select>35 </select>
35 </div>36 </div>
36 <div class="flex1 flex-container flexFlowColumn flexNoGap">37 <div class="flex1 flex-container flexFlowColumn flexNoGap">
@@ -134,6 +135,8 @@
134 <option data-type="koboldcpp" value="koboldcpp_current" data-i18n="currently_loaded">[Currently loaded]</option>135 <option data-type="koboldcpp" value="koboldcpp_current" data-i18n="currently_loaded">[Currently loaded]</option>
135 <option data-type="vllm" value="vllm_current" data-i18n="currently_selected">[Currently selected]</option>136 <option data-type="vllm" value="vllm_current" data-i18n="currently_selected">[Currently selected]</option>
136 <option data-type="custom" value="custom_current" data-i18n="currently_selected">[Currently selected]</option>137 <option data-type="custom" value="custom_current" data-i18n="currently_selected">[Currently selected]</option>
138 <option data-type="xai" value="grok-2-vision-1212">grok-2-vision-1212</option>
139 <option data-type="xai" value="grok-vision-beta">grok-vision-beta</option>
137 </select>140 </select>
138 </div>141 </div>
139 <div data-type="ollama">142 <div data-type="ollama">
public/scripts/extensions/shared.js+4 -0
@@ -153,6 +153,10 @@ function throwIfInvalidModel(useReverseProxy) {
153 throw new Error('Cohere API key is not set.');153 throw new Error('Cohere API key is not set.');
154 }154 }
155155
156 if (extension_settings.caption.multimodal_api === 'xai' && !secret_state[SECRET_KEYS.XAI]) {
157 throw new Error('xAI API key is not set.');
158 }
159
156 if (extension_settings.caption.multimodal_api === 'ollama' && !textgenerationwebui_settings.server_urls[textgen_types.OLLAMA]) {160 if (extension_settings.caption.multimodal_api === 'ollama' && !textgenerationwebui_settings.server_urls[textgen_types.OLLAMA]) {
157 throw new Error('Ollama server URL is not set.');161 throw new Error('Ollama server URL is not set.');
158 }162 }
src/endpoints/openai.js+8 -0
@@ -65,6 +65,10 @@ router.post('/caption-image', async (request, response) => {
65 key = readSecret(request.user.directories, SECRET_KEYS.COHERE);65 key = readSecret(request.user.directories, SECRET_KEYS.COHERE);
66 }66 }
6767
68 if (request.body.api === 'xai') {
69 key = readSecret(request.user.directories, SECRET_KEYS.XAI);
70 }
71
68 if (!key && !request.body.reverse_proxy && ['custom', 'ooba', 'koboldcpp', 'vllm'].includes(request.body.api) === false) {72 if (!key && !request.body.reverse_proxy && ['custom', 'ooba', 'koboldcpp', 'vllm'].includes(request.body.api) === false) {
69 console.warn('No key found for API', request.body.api);73 console.warn('No key found for API', request.body.api);
70 return response.sendStatus(400);74 return response.sendStatus(400);
@@ -134,6 +138,10 @@ router.post('/caption-image', async (request, response) => {
134 apiUrl = 'https://api.cohere.ai/v2/chat';138 apiUrl = 'https://api.cohere.ai/v2/chat';
135 }139 }
136140
141 if (request.body.api === 'xai') {
142 apiUrl = 'https://api.x.ai/v1/chat/completions';
143 }
144
137 if (request.body.api === 'ooba') {145 if (request.body.api === 'ooba') {
138 apiUrl = `${trimV1(request.body.server_url)}/v1/chat/completions`;146 apiUrl = `${trimV1(request.body.server_url)}/v1/chat/completions`;
139 const imgMessage = body.messages.pop();147 const imgMessage = body.messages.pop();