MistralAI: Add Pixtral to models and captioning

60df924bec5b32a6c089c6c19764d41a1e9ef14b

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: 1366c2741d35aaaa4680425b8c8775c381edfba4
6 files changed, +29 -4Ignore whitespace
public/index.html+3 -0
@@ -2896,6 +2896,7 @@
2896 <option value="mistral-large-latest">mistral-large-latest</option>2896 <option value="mistral-large-latest">mistral-large-latest</option>
2897 <option value="codestral-latest">codestral-latest</option>2897 <option value="codestral-latest">codestral-latest</option>
2898 <option value="codestral-mamba-latest">codestral-mamba-latest</option>2898 <option value="codestral-mamba-latest">codestral-mamba-latest</option>
2899 <option value="pixtral-latest">pixtral-latest</option>
2899 </optgroup>2900 </optgroup>
2900 <optgroup label="Sub-versions">2901 <optgroup label="Sub-versions">
2901 <option value="open-mistral-nemo-2407">open-mistral-nemo-2407</option>2902 <option value="open-mistral-nemo-2407">open-mistral-nemo-2407</option>
@@ -2903,11 +2904,13 @@
2903 <option value="mistral-tiny-2312">mistral-tiny-2312</option>2904 <option value="mistral-tiny-2312">mistral-tiny-2312</option>
2904 <option value="mistral-small-2312">mistral-small-2312</option>2905 <option value="mistral-small-2312">mistral-small-2312</option>
2905 <option value="mistral-small-2402">mistral-small-2402</option>2906 <option value="mistral-small-2402">mistral-small-2402</option>
2907 <option value="mistral-small-2409">mistral-small-2409</option>
2906 <option value="mistral-medium-2312">mistral-medium-2312</option>2908 <option value="mistral-medium-2312">mistral-medium-2312</option>
2907 <option value="mistral-large-2402">mistral-large-2402</option>2909 <option value="mistral-large-2402">mistral-large-2402</option>
2908 <option value="mistral-large-2407">mistral-large-2407</option>2910 <option value="mistral-large-2407">mistral-large-2407</option>
2909 <option value="codestral-2405">codestral-2405</option>2911 <option value="codestral-2405">codestral-2405</option>
2910 <option value="codestral-mamba-2407">codestral-mamba-2407</option>2912 <option value="codestral-mamba-2407">codestral-mamba-2407</option>
2913 <option value="pixtral-12b-2409">pixtral-12b-2409</option>
2911 </optgroup>2914 </optgroup>
2912 </select>2915 </select>
2913 </div>2916 </div>
public/scripts/extensions/caption/index.js+1 -0
@@ -403,6 +403,7 @@ jQuery(async function () {
403 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'openai' && (secret_state[SECRET_KEYS.OPENAI] || extension_settings.caption.allow_reverse_proxy)) ||403 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'openai' && (secret_state[SECRET_KEYS.OPENAI] || extension_settings.caption.allow_reverse_proxy)) ||
404 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'openrouter' && secret_state[SECRET_KEYS.OPENROUTER]) ||404 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'openrouter' && secret_state[SECRET_KEYS.OPENROUTER]) ||
405 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'zerooneai' && secret_state[SECRET_KEYS.ZEROONEAI]) ||405 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'zerooneai' && secret_state[SECRET_KEYS.ZEROONEAI]) ||
406 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'mistral' && (secret_state[SECRET_KEYS.MISTRALAI] || extension_settings.caption.allow_reverse_proxy)) ||
406 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'google' && (secret_state[SECRET_KEYS.MAKERSUITE] || extension_settings.caption.allow_reverse_proxy)) ||407 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'google' && (secret_state[SECRET_KEYS.MAKERSUITE] || extension_settings.caption.allow_reverse_proxy)) ||
407 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'anthropic' && (secret_state[SECRET_KEYS.CLAUDE] || extension_settings.caption.allow_reverse_proxy)) ||408 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'anthropic' && (secret_state[SECRET_KEYS.CLAUDE] || extension_settings.caption.allow_reverse_proxy)) ||
408 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'ollama' && textgenerationwebui_settings.server_urls[textgen_types.OLLAMA]) ||409 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'ollama' && textgenerationwebui_settings.server_urls[textgen_types.OLLAMA]) ||
public/scripts/extensions/caption/settings.html+4 -1
@@ -23,6 +23,7 @@
23 <option value="google">Google AI Studio</option>23 <option value="google">Google AI Studio</option>
24 <option value="koboldcpp">KoboldCpp</option>24 <option value="koboldcpp">KoboldCpp</option>
25 <option value="llamacpp">llama.cpp</option>25 <option value="llamacpp">llama.cpp</option>
26 <option value="mistral">MistralAI</option>
26 <option value="ollama">Ollama</option>27 <option value="ollama">Ollama</option>
27 <option value="openai">OpenAI</option>28 <option value="openai">OpenAI</option>
28 <option value="openrouter">OpenRouter</option>29 <option value="openrouter">OpenRouter</option>
@@ -33,6 +34,8 @@
33 <div class="flex1 flex-container flexFlowColumn flexNoGap">34 <div class="flex1 flex-container flexFlowColumn flexNoGap">
34 <label for="caption_multimodal_model" data-i18n="Model">Model</label>35 <label for="caption_multimodal_model" data-i18n="Model">Model</label>
35 <select id="caption_multimodal_model" class="flex1 text_pole">36 <select id="caption_multimodal_model" class="flex1 text_pole">
37 <option data-type="mistral" value="pixtral-latest">pixtral-latest</option>
38 <option data-type="mistral" value="pixtral-12b-2409">pixtral-12b-2409</option>
36 <option data-type="zerooneai" value="yi-vision">yi-vision</option>39 <option data-type="zerooneai" value="yi-vision">yi-vision</option>
37 <option data-type="openai" value="gpt-4-vision-preview">gpt-4-vision-preview</option>40 <option data-type="openai" value="gpt-4-vision-preview">gpt-4-vision-preview</option>
38 <option data-type="openai" value="gpt-4-turbo">gpt-4-turbo</option>41 <option data-type="openai" value="gpt-4-turbo">gpt-4-turbo</option>
@@ -96,7 +99,7 @@
96 <div data-type="ollama">99 <div data-type="ollama">
97 The model must be downloaded first! Do it with the <code>ollama pull</code> command or <a href="#" id="caption_ollama_pull">click here</a>.100 The model must be downloaded first! Do it with the <code>ollama pull</code> command or <a href="#" id="caption_ollama_pull">click here</a>.
98 </div>101 </div>
99 <label data-type="openai,anthropic,google" class="checkbox_label flexBasis100p" for="caption_allow_reverse_proxy" title="Allow using reverse proxy if defined and valid.">102 <label data-type="openai,anthropic,google,mistral" class="checkbox_label flexBasis100p" for="caption_allow_reverse_proxy" title="Allow using reverse proxy if defined and valid.">
100 <input id="caption_allow_reverse_proxy" type="checkbox" class="checkbox">103 <input id="caption_allow_reverse_proxy" type="checkbox" class="checkbox">
101 <span data-i18n="Allow reverse proxy">Allow reverse proxy</span>104 <span data-i18n="Allow reverse proxy">Allow reverse proxy</span>
102 </label>105 </label>
public/scripts/extensions/shared.js+6 -2
@@ -13,7 +13,7 @@ import { createThumbnail, isValidUrl } from '../utils.js';
13 */13 */
14export async function getMultimodalCaption(base64Img, prompt) {14export async function getMultimodalCaption(base64Img, prompt) {
15 const useReverseProxy =15 const useReverseProxy =
16 (['openai', 'anthropic', 'google'].includes(extension_settings.caption.multimodal_api))16 (['openai', 'anthropic', 'google', 'mistral'].includes(extension_settings.caption.multimodal_api))
17 && extension_settings.caption.allow_reverse_proxy17 && extension_settings.caption.allow_reverse_proxy
18 && oai_settings.reverse_proxy18 && oai_settings.reverse_proxy
19 && isValidUrl(oai_settings.reverse_proxy);19 && isValidUrl(oai_settings.reverse_proxy);
@@ -36,7 +36,7 @@ export async function getMultimodalCaption(base64Img, prompt) {
36 const isVllm = extension_settings.caption.multimodal_api === 'vllm';36 const isVllm = extension_settings.caption.multimodal_api === 'vllm';
37 const base64Bytes = base64Img.length * 0.75;37 const base64Bytes = base64Img.length * 0.75;
38 const compressionLimit = 2 * 1024 * 1024;38 const compressionLimit = 2 * 1024 * 1024;
39 if ((['google', 'openrouter'].includes(extension_settings.caption.multimodal_api) && base64Bytes > compressionLimit) || isOoba || isKoboldCpp) {39 if ((['google', 'openrouter', 'mistral'].includes(extension_settings.caption.multimodal_api) && base64Bytes > compressionLimit) || isOoba || isKoboldCpp) {
40 const maxSide = 1024;40 const maxSide = 1024;
41 base64Img = await createThumbnail(base64Img, maxSide, maxSide, 'image/jpeg');41 base64Img = await createThumbnail(base64Img, maxSide, maxSide, 'image/jpeg');
42 }42 }
@@ -139,6 +139,10 @@ function throwIfInvalidModel(useReverseProxy) {
139 throw new Error('Google AI Studio API key is not set.');139 throw new Error('Google AI Studio API key is not set.');
140 }140 }
141141
142 if (extension_settings.caption.multi_modal_api === 'mistral' && !secret_state[SECRET_KEYS.MISTRALAI] && !useReverseProxy) {
143 throw new Error('Mistral AI API key is not set.');
144 }
145
142 if (extension_settings.caption.multimodal_api === 'ollama' && !textgenerationwebui_settings.server_urls[textgen_types.OLLAMA]) {146 if (extension_settings.caption.multimodal_api === 'ollama' && !textgenerationwebui_settings.server_urls[textgen_types.OLLAMA]) {
143 throw new Error('Ollama server URL is not set.');147 throw new Error('Ollama server URL is not set.');
144 }148 }
public/scripts/openai.js+7 -1
@@ -2490,7 +2490,7 @@ class Message {
2490 * @returns {Promise<string>} Compressed image as a Data URL.2490 * @returns {Promise<string>} Compressed image as a Data URL.
2491 */2491 */
2492 async compressImage(image) {2492 async compressImage(image) {
2493 if ([chat_completion_sources.OPENROUTER, chat_completion_sources.MAKERSUITE].includes(oai_settings.chat_completion_source)) {2493 if ([chat_completion_sources.OPENROUTER, chat_completion_sources.MAKERSUITE, chat_completion_sources.MISTRALAI].includes(oai_settings.chat_completion_source)) {
2494 const sizeThreshold = 2 * 1024 * 1024;2494 const sizeThreshold = 2 * 1024 * 1024;
2495 const dataSize = image.length * 0.75;2495 const dataSize = image.length * 0.75;
2496 const maxSide = 1024;2496 const maxSide = 1024;
@@ -4221,6 +4221,8 @@ async function onModelChange() {
4221 $('#openai_max_context').attr('max', max_128k);4221 $('#openai_max_context').attr('max', max_128k);
4222 } else if (oai_settings.mistralai_model.includes('mixtral-8x22b')) {4222 } else if (oai_settings.mistralai_model.includes('mixtral-8x22b')) {
4223 $('#openai_max_context').attr('max', max_64k);4223 $('#openai_max_context').attr('max', max_64k);
4224 } else if (oai_settings.mistralai_model.includes('pixtral')) {
4225 $('#openai_max_context').attr('max', max_128k);
4224 } else {4226 } else {
4225 $('#openai_max_context').attr('max', max_32k);4227 $('#openai_max_context').attr('max', max_32k);
4226 }4228 }
@@ -4770,6 +4772,8 @@ export function isImageInliningSupported() {
4770 'gpt-4o-mini',4772 'gpt-4o-mini',
4771 'chatgpt-4o-latest',4773 'chatgpt-4o-latest',
4772 'yi-vision',4774 'yi-vision',
4775 'pixtral-latest',
4776 'pixtral-12b-2409',
4773 ];4777 ];
47744778
4775 switch (oai_settings.chat_completion_source) {4779 switch (oai_settings.chat_completion_source) {
@@ -4785,6 +4789,8 @@ export function isImageInliningSupported() {
4785 return true;4789 return true;
4786 case chat_completion_sources.ZEROONEAI:4790 case chat_completion_sources.ZEROONEAI:
4787 return visionSupportedModels.some(model => oai_settings.zerooneai_model.includes(model));4791 return visionSupportedModels.some(model => oai_settings.zerooneai_model.includes(model));
4792 case chat_completion_sources.MISTRALAI:
4793 return visionSupportedModels.some(model => oai_settings.mistralai_model.includes(model));
4788 default:4794 default:
4789 return false;4795 return false;
4790 }4796 }
src/endpoints/openai.js+8 -0
@@ -51,6 +51,10 @@ router.post('/caption-image', jsonParser, async (request, response) => {
51 key = readSecret(request.user.directories, SECRET_KEYS.ZEROONEAI);51 key = readSecret(request.user.directories, SECRET_KEYS.ZEROONEAI);
52 }52 }
5353
54 if (request.body.api === 'mistral') {
55 key = readSecret(request.user.directories, SECRET_KEYS.MISTRALAI);
56 }
57
54 if (!key && !request.body.reverse_proxy && ['custom', 'ooba', 'koboldcpp', 'vllm'].includes(request.body.api) === false) {58 if (!key && !request.body.reverse_proxy && ['custom', 'ooba', 'koboldcpp', 'vllm'].includes(request.body.api) === false) {
55 console.log('No key found for API', request.body.api);59 console.log('No key found for API', request.body.api);
56 return response.sendStatus(400);60 return response.sendStatus(400);
@@ -107,6 +111,10 @@ router.post('/caption-image', jsonParser, async (request, response) => {
107 apiUrl = 'https://api.01.ai/v1/chat/completions';111 apiUrl = 'https://api.01.ai/v1/chat/completions';
108 }112 }
109113
114 if (request.body.api === 'mistral') {
115 apiUrl = 'https://api.mistral.ai/v1/chat/completions';
116 }
117
110 if (request.body.api === 'ooba') {118 if (request.body.api === 'ooba') {
111 apiUrl = `${trimV1(request.body.server_url)}/v1/chat/completions`;119 apiUrl = `${trimV1(request.body.server_url)}/v1/chat/completions`;
112 const imgMessage = body.messages.pop();120 const imgMessage = body.messages.pop();