Merge pull request #2805 from SillyTavern/gemini-vision-fix Fix Gemini multimodal with JPG images

5e522d6e35a5f838bf5530a2d07d2e3951cd2776

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

public/scripts/extensions/shared.js+1 -6 Viewed

     throwIfInvalidModel(useReverseProxy);
     const noPrefix = ['google', 'ollama', 'llamacpp'].includes(extension_settings.caption.multimodal_api);
     if (noPrefix && base64Img.startsWith('data:image/')) {
         base64Img = base64Img.split(',')[1];

     // OpenRouter has a payload limit of ~2MB. Google is 4MB, but we love democracy.
     // Ooba requires all images to be JPEGs. Koboldcpp just asked nicely.
 -    const isGoogle = extension_settings.caption.multimodal_api === 'google';
     const isOllama = extension_settings.caption.multimodal_api === 'ollama';
     const isLlamaCpp = extension_settings.caption.multimodal_api === 'llamacpp';
     const isCustom = extension_settings.caption.multimodal_api === 'custom';

     if ((['google', 'openrouter'].includes(extension_settings.caption.multimodal_api) && base64Bytes > compressionLimit) || isOoba || isKoboldCpp) {
         const maxSide = 1024;
         base64Img = await createThumbnail(base64Img, maxSide, maxSide, 'image/jpeg');
+-
 -        if (isGoogle) {
 -            base64Img = base64Img.split(',')[1];
 -        }
+    }
     const proxyUrl = useReverseProxy ? oai_settings.reverse_proxy : '';

public/scripts/openai.js+26 -3 Viewed

 import { getEventSourceStream } from './sse-stream.js';
 import {
 +    createThumbnail,
     delay,
     download,
     getBase64Async,

                 if (!response.ok) throw new Error('Failed to fetch image');
                 const blob = await response.blob();
                 image = await getBase64Async(blob);
 -                if (oai_settings.chat_completion_source === chat_completion_sources.MAKERSUITE) {
 -                    image = image.split(',')[1];
 -                }
             } catch (error) {
                 console.error('Image adding skipped', error);
                 return;
+            }
+        }
 +        image = await this.compressImage(image);
++
         const quality = oai_settings.inline_image_quality || default_settings.inline_image_quality;
         this.content = [
             { type: 'text', text: textContent },

+        }
+    }
 +    /**
 +     * Compress an image if it exceeds the size threshold for the current chat completion source.
 +     * @param {string} image Data URL of the image.
 +     * @returns {Promise<string>} Compressed image as a Data URL.
 +     */
 +    async compressImage(image) {
 +        if ([chat_completion_sources.OPENROUTER, chat_completion_sources.MAKERSUITE].includes(oai_settings.chat_completion_source)) {
 +            const sizeThreshold = 2 * 1024 * 1024;
 +            const dataSize = image.length * 0.75;
 +            const maxSide = 1024;
 +            if (dataSize > sizeThreshold) {
 +                image = await createThumbnail(image, maxSide);
 +            }
 +        }
 +        return image;
 +    }
++
 +    /**
 +     * Get the token cost of an image.
 +     * @param {string} dataUrl Data URL of the image.
 +     * @param {string} quality String representing the quality of the image. Can be 'low', 'auto', or 'high'.
 +     * @returns {Promise<number>} The token cost of the image.
 +     */
     async getImageTokenCost(dataUrl, quality) {
         if (quality === 'low') {
             return Message.tokensPerImage;

src/endpoints/google.js+2 -2 Viewed

                     { text: request.body.prompt },
+                    {
                         inlineData: {
 -                            mimeType: 'image/png', // It needs to specify a MIME type in data if it's not a PNG
 +                            mimeType: mimeType,
 -                            data: mimeType === 'image/png' ? base64Data : request.body.image,
 +                            data: base64Data,
                         },
                     }],
             }],

src/prompt-converters.js+4 -2 Viewed

                 if (part.type === 'text') {
                     parts.push({ text: part.text });
                 } else if (part.type === 'image_url' && isMultimodal) {
 +                    const mimeType = part.image_url.url.split(';')[0].split(':')[1];
 +                    const base64Data = part.image_url.url.split(',')[1];
                     parts.push({
                         inlineData: {
                             mimeType: 'image/png'mimeType,
                             data: part.image_url.urlbase64Data,
                         },
                     });
                     hasImage = true;