Gemini inline video (#4078) * Add inline video attachment support for Gemini 2.5 Pro * file formatting * removed redundant function for saving video to message * removed other redundant function for saving video to message * Seperate inlining check for video * Edit video token cost to be a conservative estimate of 10000 tokens * fixed missing semicolon * Adds seperate ui toggle for video inlining. * Move mes_video out of img_container * Remove title from video element for now * Better visibilty of video with controls --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>

c4d89b2067be362e6ad0085e7d3a1b54d36a257b

Nikolas Brown <nbrown72504@gmail.com>

Signed

Parents: 3ec9b1a09994e33d663e47e425339df39235c973

public/index.html+14 -0 Viewed

                                         </div>
                                     </div>
                                     <div class="range-block" data-source="makersuite,vertexai">
 +                                        <label for="openai_video_inlining" class="checkbox_label flexWrap widthFreeExpand">
 +                                            <input id="openai_video_inlining" type="checkbox" />
 +                                            <span data-i18n="Send inline videos">Send inline videos</span>
 +                                        </label>
 +                                        <div id="video_inlining_hint" class="flexBasis100p toggle-description justifyLeft">
 +                                            <span data-i18n="video_inlining_hint_1">Sends videos in prompts if the model supports it. Use the</span>
 +                                            <code><i class="fa-solid fa-paperclip"></i></code>
 +                                            <span data-i18n="video_inlining_hint_2">action on any message or the</span>
 +                                            <code><i class="fa-solid fa-wand-magic-sparkles"></i></code>
 +                                            <span data-i18n="video_inlining_hint_3">menu to attach a video file to the chat.</span>
 +                                            <strong data-i18n="video_inlining_hint_4">Videos must be less than 20 MB and under 1 minute long</strong>
 +                                        </div>
 +                                    </div>
 +                                    <div class="range-block" data-source="makersuite,vertexai">
                                         <label for="openai_request_images" class="checkbox_label widthFreeExpand">
                                             <input id="openai_request_images" type="checkbox" />
                                             <span>

public/script.js+25 -0 Viewed

+        }
+    }
 +    // Add video to message
 +    if (mes.extra?.video) {
 +        const container = messageElement.find('.mes_block');
 +        const chatHeight = $('#chat').prop('scrollHeight');
++
 +        // Create video element if it doesn't exist
 +        let video = messageElement.find('.mes_video');
 +        if (video.length === 0) {
 +            video = $('<video class="mes_video" controls preload="metadata"></video>');
 +            container.append(video);
 +        }
++
 +        video.off('loadedmetadata').on('loadedmetadata', function () {
 +            if (!adjustScroll) {
 +                return;
 +            }
 +            const scrollPosition = $('#chat').scrollTop();
 +            const newChatHeight = $('#chat').prop('scrollHeight');
 +            const diff = newChatHeight - chatHeight;
 +            $('#chat').scrollTop(scrollPosition + diff);
 +        });
++
 +        video.attr('src', mes.extra?.video);
 +    }
++
     // Add file to message
     if (mes.extra?.file) {
         messageElement.find('.mes_file_container').remove();

public/scripts/chats.js+6 -0 Viewed

             const imageUrl = await saveBase64AsFile(base64Data, name2, fileNamePrefix, extension);
             message.extra.image = imageUrl;
             message.extra.inline_image = true;
 +        }
 +        // If file is video
 +        else if (file.type.startsWith('video/')) {
 +            const extension = file.type.split('/')[1];
 +            const videoUrl = await saveBase64AsFile(base64Data, name2, fileNamePrefix, extension);
 +            message.extra.video = videoUrl;
         } else {
             const uniqueFileName = `${fileNamePrefix}.txt`;

public/scripts/openai.js+81 -1 Viewed

     squash_system_messages: ['#squash_system_messages', 'squash_system_messages', true, false],
     image_inlining: ['#openai_image_inlining', 'image_inlining', true, false],
     inline_image_quality: ['#openai_inline_image_quality', 'inline_image_quality', false, false],
 +    video_inlining: ['#openai_video_inlining', 'video_inlining', true, false],
     continue_prefill: ['#continue_prefill', 'continue_prefill', true, false],
     continue_postfix: ['#continue_postfix', 'continue_postfix', false, false],
     function_calling: ['#openai_function_calling', 'function_calling', true, false],

     squash_system_messages: false,
     image_inlining: false,
     inline_image_quality: 'low',
 +    video_inlining: false,
     bypass_status_check: false,
     continue_prefill: false,
     function_calling: false,

     squash_system_messages: false,
     image_inlining: false,
     inline_image_quality: 'low',
 +    video_inlining: false,
     bypass_status_check: false,
     continue_prefill: false,
     function_calling: false,

         if (role == 'user' && oai_settings.wrap_in_quotes) content = `"${content}"`;
         const name = chat[j]['name'];
         const image = chat[j]?.extra?.image;
 +        const video = chat[j]?.extra?.video;
         const invocations = chat[j]?.extra?.tool_invocations;
         messages[i] = { 'role': role, 'content': content, name: name, 'image': image, 'video': video, 'invocations': invocations };
         j++;
+    }

+    }
     const imageInlining = isImageInliningSupported();
 +    const videoInlining = isVideoInliningSupported();
     const canUseTools = ToolManager.isToolCallingSupported();
     // Insert chat messages as long as there is budget available

             await chatMessage.addImage(chatPrompt.image);
+        }
 +        if (videoInlining && chatPrompt.video) {
 +            await chatMessage.addVideo(chatPrompt.video);
 +        }
++
         if (canUseTools && Array.isArray(chatPrompt.invocations)) {
             /** @type {import('./tool-calling.js').ToolInvocation[]} */
             const invocations = chatPrompt.invocations;

+        }
+    }
 +    async addVideo(video) {
 +        const textContent = this.content;
 +        const isDataUrl = isDataURL(video);
 +        if (!isDataUrl) {
 +            try {
 +                const response = await fetch(video, { method: 'GET', cache: 'force-cache' });
 +                if (!response.ok) throw new Error('Failed to fetch video');
 +                const blob = await response.blob();
 +                video = await getBase64Async(blob);
 +            } catch (error) {
 +                console.error('Video adding skipped', error);
 +                return;
 +            }
 +        }
++
 +        // Note: No compression for videos (unlike images)
 +        this.content = [
 +            { type: 'text', text: textContent },
 +            { type: 'video_url', video_url: { 'url': video } },
 +        ];
++
 +        try {
 +            // Convservative estimate for video token cost without knowing duration
 +            // Using Gemini calculation (263 tokens per second)
 +            const tokens = 10000; // ~40 second video (60 seconds max)
 +            this.tokens += tokens;
 +        } catch (error) {
 +            this.tokens += 10000;
 +            console.error('Failed to get video token cost', error);
 +        }
 +    }
++
     /**
      * Compress an image if it exceeds the size threshold for the current chat completion source.
      * @param {string} image Data URL of the image.

     oai_settings.assistant_impersonation = settings.assistant_impersonation ?? default_settings.assistant_impersonation;
     oai_settings.image_inlining = settings.image_inlining ?? default_settings.image_inlining;
     oai_settings.inline_image_quality = settings.inline_image_quality ?? default_settings.inline_image_quality;
 +    oai_settings.video_inlining = settings.video_inlining ?? default_settings.video_inlining;
     oai_settings.bypass_status_check = settings.bypass_status_check ?? default_settings.bypass_status_check;
     oai_settings.show_thoughts = settings.show_thoughts ?? default_settings.show_thoughts;
     oai_settings.reasoning_effort = settings.reasoning_effort ?? default_settings.reasoning_effort;

     $('#openai_inline_image_quality').val(oai_settings.inline_image_quality);
     $(`#openai_inline_image_quality option[value="${oai_settings.inline_image_quality}"]`).prop('selected', true);
 +    $('#openai_video_inlining').prop('checked', oai_settings.video_inlining);
++
     $('#model_openai_select').val(oai_settings.openai_model);
     $(`#model_openai_select option[value="${oai_settings.openai_model}"`).prop('selected', true);
     $('#model_claude_select').val(oai_settings.claude_model);

         squash_system_messages: settings.squash_system_messages,
         image_inlining: settings.image_inlining,
         inline_image_quality: settings.inline_image_quality,
 +        video_inlining: settings.video_inlining,
         bypass_status_check: settings.bypass_status_check,
         continue_prefill: settings.continue_prefill,
         continue_postfix: settings.continue_postfix,

+}
 /**
 + * Check if the model supports video inlining
 + * @returns {boolean} True if the model supports video inlining
 + */
 +export function isVideoInliningSupported() {
 +    if (main_api !== 'openai') {
 +        return false;
 +    }
++
 +    if (!oai_settings.video_inlining) {
 +        return false;
 +    }
++
 +    // Only Gemini models support video for now
 +    const videoSupportedModels = [
 +        'gemini-2.0',
 +        'gemini-2.5',
 +        'gemini-exp-1206',
 +    ];
++
 +    switch (oai_settings.chat_completion_source) {
 +        case chat_completion_sources.MAKERSUITE:
 +            return videoSupportedModels.some(model => oai_settings.google_model.includes(model));
 +        case chat_completion_sources.VERTEXAI:
 +            return videoSupportedModels.some(model => oai_settings.vertexai_model.includes(model));
 +        default:
 +            return false;
 +    }
 +}
++
 +/**
  * Proxy stuff
  */
 export function loadProxyPresets(settings) {

         saveSettingsDebounced();
     });
 +    $('#openai_video_inlining').on('input', function () {
 +        oai_settings.video_inlining = !!$(this).prop('checked');
 +        saveSettingsDebounced();
 +    });
++
     $('#continue_prefill').on('input', function () {
         oai_settings.continue_prefill = !!$(this).prop('checked');
         saveSettingsDebounced();

public/style.css+18 -0 Viewed

     max-width: 100% !important;
+}
 +/* Video message styling */
 +.mes_video {
 +    max-width: 100%;
 +    max-height: 400px;
 +    border-radius: 8px;
 +    background: #000;
 +    margin: 0.5rem;
 +}
++
 +/* Ensure video controls are visible */
 +.mes_video::-webkit-media-controls {
 +    display: flex !important;
 +}
++
 +.mes_video::-webkit-media-controls-panel {
 +    background-color: rgba(0, 0, 0, 0.2);
 +}
++
 /* Align the content of this span to the right */
 .delete-button {
     margin-right: 10px;

src/endpoints/images.js+1 -1 Viewed

         const splitParts = request.body.image.split(',');
         const format = splitParts[0].split(';')[0].split('/')[1];
         const base64Data = splitParts[1];
         const validFormat = ['png', 'jpg', 'webp', 'jpeg', 'gif', 'mp4', 'avi', 'mov', 'wmv', 'flv', 'webm', '3gp', 'mkv'].includes(format);
         if (!validFormat) {
             return response.status(400).send({ error: 'Invalid image format' });
+        }

src/prompt-converters.js+13 -0 Viewed

                         data: base64Data,
                     },
                 });
 +            } else if (part.type === 'video_url') {
 +                const videoUrl = part.video_url?.url;
 +                if (videoUrl && videoUrl.startsWith('data:')) {
 +                    const [header, data] = videoUrl.split(',');
 +                    const mimeType = header.match(/data:([^;]+)/)?.[1] || 'video/mp4';
++
 +                    parts.push({
 +                        inlineData: {
 +                            mimeType: mimeType,
 +                            data: data,
 +                        },
 +                    });
 +                }
+            }
         });