Merge pull request #3900 from equal-l2/vision-cleanup Vision cleanup

6848b38bb7d99c27a4b85a2dd35ca8d64ebcda6d

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: 76aa17e08f7f6bf94744dc24a6bd05b7d19cb297 e621f0d967d94d8c2c984dd7261759fa1c96783d
2 files changed, +29 -93Showing whitespace changes
public/scripts/openai.js+26 -55
@@ -5132,73 +5132,44 @@ export function isImageInliningSupported() {
51325132
5133 // gultra just isn't being offered as multimodal, thanks google.5133 // gultra just isn't being offered as multimodal, thanks google.
5134 const visionSupportedModels = [5134 const visionSupportedModels = [
5135 'gpt-4-vision',5135 // OpenAI
5136 'gemini-2.5-pro-exp-03-25',5136 'chatgpt-4o-latest',
5137 'gemini-2.5-pro-preview-03-25',
5138 'gemini-2.0-pro-exp',
5139 'gemini-2.0-pro-exp-02-05',
5140 'gemini-2.5-flash-preview-04-17',
5141 'gemini-2.0-flash-lite-preview',
5142 'gemini-2.0-flash-lite-preview-02-05',
5143 'gemini-2.0-flash',
5144 'gemini-2.0-flash-001',
5145 'gemini-2.0-flash-thinking-exp-1219',
5146 'gemini-2.0-flash-thinking-exp-01-21',
5147 'gemini-2.0-flash-thinking-exp',
5148 'gemini-2.0-flash-exp',
5149 'gemini-2.0-flash-exp-image-generation',
5150 'gemini-1.5-flash',
5151 'gemini-1.5-flash-latest',
5152 'gemini-1.5-flash-001',
5153 'gemini-1.5-flash-002',
5154 'gemini-1.5-flash-exp-0827',
5155 'gemini-1.5-flash-8b',
5156 'gemini-1.5-flash-8b-exp-0827',
5157 'gemini-1.5-flash-8b-exp-0924',
5158 'gemini-exp-1114',
5159 'gemini-exp-1121',
5160 'gemini-exp-1206',
5161 'gemini-1.0-pro-vision-latest',
5162 'gemini-1.5-pro',
5163 'gemini-1.5-pro-latest',
5164 'gemini-1.5-pro-001',
5165 'gemini-1.5-pro-002',
5166 'gemini-1.5-pro-exp-0801',
5167 'gemini-1.5-pro-exp-0827',
5168 'claude-3',
5169 'claude-3-5',
5170 'claude-3-7',
5171 'gpt-4-turbo',5137 'gpt-4-turbo',
5172 'gpt-4o',5138 'gpt-4-vision',
5173 'gpt-4o-mini',5139 'gpt-4.1',
5174 'gpt-4.5-preview',5140 'gpt-4.5-preview',
5175 'gpt-4.5-preview-2025-02-27',5141 'gpt-4o',
5176 'o1',5142 'o1',
5177 'o1-2024-12-17',5143 'o3',
5178 'chatgpt-4o-latest',5144 'o4-mini',
5145 // 01.AI (Yi)
5179 'yi-vision',5146 'yi-vision',
5180 'mistral-large-pixtral-2411',5147 // Claude
5148 'claude-3',
5149 // Cohere
5150 'c4ai-aya-vision',
5151 // Google AI Studio
5152 'gemini-1.5',
5153 'gemini-2.0',
5154 'gemini-2.5',
5155 'gemini-exp-1114',
5156 'gemini-exp-1121',
5157 'gemini-exp-1206',
5158 // MistralAI
5181 'mistral-small-2503',5159 'mistral-small-2503',
5182 'mistral-small-latest',5160 'mistral-small-latest',
5183 'pixtral-12b-latest',5161 'pixtral',
5184 'pixtral-12b',5162 // xAI (Grok)
5185 'pixtral-12b-2409',
5186 'pixtral-large-latest',
5187 'pixtral-large-2411',
5188 'c4ai-aya-vision-8b',
5189 'c4ai-aya-vision-32b',
5190 'grok-2-vision',5163 'grok-2-vision',
5191 'grok-vision',5164 'grok-vision',
5192 'gpt-4.1',
5193 'o3',
5194 'o3-2025-04-16',
5195 'o4-mini',
5196 'o4-mini-2025-04-16',
5197 ];5165 ];
51985166
5199 switch (oai_settings.chat_completion_source) {5167 switch (oai_settings.chat_completion_source) {
5200 case chat_completion_sources.OPENAI:5168 case chat_completion_sources.OPENAI:
5201 return visionSupportedModels.some(model => oai_settings.openai_model.includes(model) && !oai_settings.openai_model.includes('gpt-4-turbo-preview') && !oai_settings.openai_model.includes('o3-mini'));5169 return visionSupportedModels.some(model =>
5170 oai_settings.openai_model.includes(model)
5171 && ['gpt-4-turbo-preview', 'o1-mini', 'o3-mini'].some(x => !oai_settings.openai_model.includes(x))
5172 );
5202 case chat_completion_sources.MAKERSUITE:5173 case chat_completion_sources.MAKERSUITE:
5203 return visionSupportedModels.some(model => oai_settings.google_model.includes(model));5174 return visionSupportedModels.some(model => oai_settings.google_model.includes(model));
5204 case chat_completion_sources.CLAUDE:5175 case chat_completion_sources.CLAUDE:
src/prompt-converters.js+3 -38
@@ -357,47 +357,12 @@ export function convertCohereMessages(messages, names) {
357/**357/**
358 * Convert a prompt from the ChatML objects to the format used by Google MakerSuite models.358 * Convert a prompt from the ChatML objects to the format used by Google MakerSuite models.
359 * @param {object[]} messages Array of messages359 * @param {object[]} messages Array of messages
360 * @param {string} model Model name360 * @param {string} _model Model name
361 * @param {boolean} useSysPrompt Use system prompt361 * @param {boolean} useSysPrompt Use system prompt
362 * @param {PromptNames} names Prompt names362 * @param {PromptNames} names Prompt names
363 * @returns {{contents: *[], system_instruction: {parts: {text: string}[]}}} Prompt for Google MakerSuite models363 * @returns {{contents: *[], system_instruction: {parts: {text: string}[]}}} Prompt for Google MakerSuite models
364 */364 */
365export function convertGooglePrompt(messages, model, useSysPrompt, names) {365export function convertGooglePrompt(messages, _model, useSysPrompt, names) {
366 const visionSupportedModels = [
367 'gemini-2.5-pro-preview-03-25',
368 'gemini-2.5-pro-exp-03-25',
369 'gemini-2.0-pro-exp',
370 'gemini-2.0-pro-exp-02-05',
371 'gemini-2.5-flash-preview-04-17',
372 'gemini-2.0-flash-lite-preview',
373 'gemini-2.0-flash-lite-preview-02-05',
374 'gemini-2.0-flash',
375 'gemini-2.0-flash-001',
376 'gemini-2.0-flash-thinking-exp',
377 'gemini-2.0-flash-thinking-exp-01-21',
378 'gemini-2.0-flash-thinking-exp-1219',
379 'gemini-2.0-flash-exp',
380 'gemini-2.0-flash-exp-image-generation',
381 'gemini-1.5-flash',
382 'gemini-1.5-flash-latest',
383 'gemini-1.5-flash-001',
384 'gemini-1.5-flash-002',
385 'gemini-1.5-flash-exp-0827',
386 'gemini-1.5-flash-8b',
387 'gemini-1.5-flash-8b-exp-0827',
388 'gemini-1.5-flash-8b-exp-0924',
389 'gemini-exp-1114',
390 'gemini-exp-1121',
391 'gemini-exp-1206',
392 'gemini-1.5-pro',
393 'gemini-1.5-pro-latest',
394 'gemini-1.5-pro-001',
395 'gemini-1.5-pro-002',
396 'gemini-1.5-pro-exp-0801',
397 'gemini-1.5-pro-exp-0827',
398 ];
399
400 const isMultimodal = visionSupportedModels.includes(model);
401 const sysPrompt = [];366 const sysPrompt = [];
402367
403 if (useSysPrompt) {368 if (useSysPrompt) {
@@ -497,7 +462,7 @@ export function convertGooglePrompt(messages, model, useSysPrompt, names) {
497462
498 toolNameMap[toolCall.id] = toolCall.function.name;463 toolNameMap[toolCall.id] = toolCall.function.name;
499 });464 });
500 } else if (part.type === 'image_url' && isMultimodal) {465 } else if (part.type === 'image_url') {
501 const mimeType = part.image_url.url.split(';')[0].split(':')[1];466 const mimeType = part.image_url.url.split(';')[0].split(':')[1];
502 const base64Data = part.image_url.url.split(',')[1];467 const base64Data = part.image_url.url.split(',')[1];
503 parts.push({468 parts.push({