Gemini inline video (#4078) * Add inline video attachment support for Gemini 2.5 Pro * file formatting * removed redundant function for saving video to message * removed other redundant function for saving video to message * Seperate inlining check for video * Edit video token cost to be a conservative estimate of 10000 tokens * fixed missing semicolon * Adds seperate ui toggle for video inlining. * Move mes_video out of img_container * Remove title from video element for now * Better visibilty of video with controls --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>

c4d89b2067be362e6ad0085e7d3a1b54d36a257b

Nikolas Brown <nbrown72504@gmail.com>

Signed

Parents: 3ec9b1a09994e33d663e47e425339df39235c973

public/index.html+14 -0 Viewed

                                         </div>
                                     </div>
                                     <div class="range-block" data-source="makersuite,vertexai">
+                                        <label for="openai_video_inlining" class="checkbox_label flexWrap widthFreeExpand">
+                                            <input id="openai_video_inlining" type="checkbox" />
+                                            <span data-i18n="Send inline videos">Send inline videos</span>
+                                        </label>
+                                        <div id="video_inlining_hint" class="flexBasis100p toggle-description justifyLeft">
+                                            <span data-i18n="video_inlining_hint_1">Sends videos in prompts if the model supports it. Use the</span>
+                                            <code><i class="fa-solid fa-paperclip"></i></code>
+                                            <span data-i18n="video_inlining_hint_2">action on any message or the</span>
+                                            <code><i class="fa-solid fa-wand-magic-sparkles"></i></code>
+                                            <span data-i18n="video_inlining_hint_3">menu to attach a video file to the chat.</span>
+                                            <strong data-i18n="video_inlining_hint_4">Videos must be less than 20 MB and under 1 minute long</strong>
+                                        </div>
+                                    </div>
+                                    <div class="range-block" data-source="makersuite,vertexai">
                                         <label for="openai_request_images" class="checkbox_label widthFreeExpand">
                                             <input id="openai_request_images" type="checkbox" />
                                             <span>

public/script.js+25 -0 Viewed

+        }
+    }
+    // Add video to message
+    if (mes.extra?.video) {
+        const container = messageElement.find('.mes_block');
+        const chatHeight = $('#chat').prop('scrollHeight');
+        // Create video element if it doesn't exist
+        let video = messageElement.find('.mes_video');
+        if (video.length === 0) {
+            video = $('<video class="mes_video" controls preload="metadata"></video>');
+            container.append(video);
+        }
+        video.off('loadedmetadata').on('loadedmetadata', function () {
+            if (!adjustScroll) {
+                return;
+            }
+            const scrollPosition = $('#chat').scrollTop();
+            const newChatHeight = $('#chat').prop('scrollHeight');
+            const diff = newChatHeight - chatHeight;
+            $('#chat').scrollTop(scrollPosition + diff);
+        });
+        video.attr('src', mes.extra?.video);
+    }
     // Add file to message
     if (mes.extra?.file) {
         messageElement.find('.mes_file_container').remove();

public/scripts/chats.js+6 -0 Viewed

             const imageUrl = await saveBase64AsFile(base64Data, name2, fileNamePrefix, extension);
             message.extra.image = imageUrl;
             message.extra.inline_image = true;
+        }
+        // If file is video
+        else if (file.type.startsWith('video/')) {
+            const extension = file.type.split('/')[1];
+            const videoUrl = await saveBase64AsFile(base64Data, name2, fileNamePrefix, extension);
+            message.extra.video = videoUrl;
         } else {
             const uniqueFileName = `${fileNamePrefix}.txt`;

public/scripts/openai.js+81 -1 Viewed

     squash_system_messages: ['#squash_system_messages', 'squash_system_messages', true, false],
     image_inlining: ['#openai_image_inlining', 'image_inlining', true, false],
     inline_image_quality: ['#openai_inline_image_quality', 'inline_image_quality', false, false],
+    video_inlining: ['#openai_video_inlining', 'video_inlining', true, false],
     continue_prefill: ['#continue_prefill', 'continue_prefill', true, false],
     continue_postfix: ['#continue_postfix', 'continue_postfix', false, false],
     function_calling: ['#openai_function_calling', 'function_calling', true, false],

     squash_system_messages: false,
     image_inlining: false,
     inline_image_quality: 'low',
+    video_inlining: false,
     bypass_status_check: false,
     continue_prefill: false,
     function_calling: false,

     squash_system_messages: false,
     image_inlining: false,
     inline_image_quality: 'low',
+    video_inlining: false,
     bypass_status_check: false,
     continue_prefill: false,
     function_calling: false,

         if (role == 'user' && oai_settings.wrap_in_quotes) content = `"${content}"`;
         const name = chat[j]['name'];
         const image = chat[j]?.extra?.image;
+        const video = chat[j]?.extra?.video;
         const invocations = chat[j]?.extra?.tool_invocations;
-        messages[i] = { 'role': role, 'content': content, name: name, 'image': image, 'invocations': invocations };
+        messages[i] = { 'role': role, 'content': content, name: name, 'image': image, 'video': video, 'invocations': invocations };
         j++;
+    }

+    }
     const imageInlining = isImageInliningSupported();
+    const videoInlining = isVideoInliningSupported();
     const canUseTools = ToolManager.isToolCallingSupported();
     // Insert chat messages as long as there is budget available

             await chatMessage.addImage(chatPrompt.image);
+        }
+        if (videoInlining && chatPrompt.video) {
+            await chatMessage.addVideo(chatPrompt.video);
+        }
         if (canUseTools && Array.isArray(chatPrompt.invocations)) {
             /** @type {import('./tool-calling.js').ToolInvocation[]} */
             const invocations = chatPrompt.invocations;

+        }
+    }
+    async addVideo(video) {
+        const textContent = this.content;
+        const isDataUrl = isDataURL(video);
+        if (!isDataUrl) {
+            try {
+                const response = await fetch(video, { method: 'GET', cache: 'force-cache' });
+                if (!response.ok) throw new Error('Failed to fetch video');
+                const blob = await response.blob();
+                video = await getBase64Async(blob);
+            } catch (error) {
+                console.error('Video adding skipped', error);
+                return;
+            }
+        }
+        // Note: No compression for videos (unlike images)
+        this.content = [
+            { type: 'text', text: textContent },
+            { type: 'video_url', video_url: { 'url': video } },
+        ];
+        try {
+            // Convservative estimate for video token cost without knowing duration
+            // Using Gemini calculation (263 tokens per second)
+            const tokens = 10000; // ~40 second video (60 seconds max)
+            this.tokens += tokens;
+        } catch (error) {
+            this.tokens += 10000;
+            console.error('Failed to get video token cost', error);
+        }
+    }
     /**
      * Compress an image if it exceeds the size threshold for the current chat completion source.
      * @param {string} image Data URL of the image.

     oai_settings.assistant_impersonation = settings.assistant_impersonation ?? default_settings.assistant_impersonation;
     oai_settings.image_inlining = settings.image_inlining ?? default_settings.image_inlining;
     oai_settings.inline_image_quality = settings.inline_image_quality ?? default_settings.inline_image_quality;
+    oai_settings.video_inlining = settings.video_inlining ?? default_settings.video_inlining;
     oai_settings.bypass_status_check = settings.bypass_status_check ?? default_settings.bypass_status_check;
     oai_settings.show_thoughts = settings.show_thoughts ?? default_settings.show_thoughts;
     oai_settings.reasoning_effort = settings.reasoning_effort ?? default_settings.reasoning_effort;

     $('#openai_inline_image_quality').val(oai_settings.inline_image_quality);
     $(`#openai_inline_image_quality option[value="${oai_settings.inline_image_quality}"]`).prop('selected', true);
+    $('#openai_video_inlining').prop('checked', oai_settings.video_inlining);
     $('#model_openai_select').val(oai_settings.openai_model);
     $(`#model_openai_select option[value="${oai_settings.openai_model}"`).prop('selected', true);
     $('#model_claude_select').val(oai_settings.claude_model);

         squash_system_messages: settings.squash_system_messages,
         image_inlining: settings.image_inlining,
         inline_image_quality: settings.inline_image_quality,
+        video_inlining: settings.video_inlining,
         bypass_status_check: settings.bypass_status_check,
         continue_prefill: settings.continue_prefill,
         continue_postfix: settings.continue_postfix,

+}
 /**
+ * Check if the model supports video inlining
+ * @returns {boolean} True if the model supports video inlining
+ */
+export function isVideoInliningSupported() {
+    if (main_api !== 'openai') {
+        return false;
+    }
+    if (!oai_settings.video_inlining) {
+        return false;
+    }
+    // Only Gemini models support video for now
+    const videoSupportedModels = [
+        'gemini-2.0',
+        'gemini-2.5',
+        'gemini-exp-1206',
+    ];
+    switch (oai_settings.chat_completion_source) {
+        case chat_completion_sources.MAKERSUITE:
+            return videoSupportedModels.some(model => oai_settings.google_model.includes(model));
+        case chat_completion_sources.VERTEXAI:
+            return videoSupportedModels.some(model => oai_settings.vertexai_model.includes(model));
+        default:
+            return false;
+    }
+}
+/**
  * Proxy stuff
  */
 export function loadProxyPresets(settings) {

         saveSettingsDebounced();
     });
+    $('#openai_video_inlining').on('input', function () {
+        oai_settings.video_inlining = !!$(this).prop('checked');
+        saveSettingsDebounced();
+    });
     $('#continue_prefill').on('input', function () {
         oai_settings.continue_prefill = !!$(this).prop('checked');
         saveSettingsDebounced();

public/style.css+18 -0 Viewed

     max-width: 100% !important;
+}
+/* Video message styling */
+.mes_video {
+    max-width: 100%;
+    max-height: 400px;
+    border-radius: 8px;
+    background: #000;
+    margin: 0.5rem;
+}
+/* Ensure video controls are visible */
+.mes_video::-webkit-media-controls {
+    display: flex !important;
+}
+.mes_video::-webkit-media-controls-panel {
+    background-color: rgba(0, 0, 0, 0.2);
+}
 /* Align the content of this span to the right */
 .delete-button {
     margin-right: 10px;

src/endpoints/images.js+1 -1 Viewed

         const splitParts = request.body.image.split(',');
         const format = splitParts[0].split(';')[0].split('/')[1];
         const base64Data = splitParts[1];
-        const validFormat = ['png', 'jpg', 'webp', 'jpeg', 'gif'].includes(format);
+        const validFormat = ['png', 'jpg', 'webp', 'jpeg', 'gif', 'mp4', 'avi', 'mov', 'wmv', 'flv', 'webm', '3gp', 'mkv'].includes(format);
         if (!validFormat) {
             return response.status(400).send({ error: 'Invalid image format' });
+        }

src/prompt-converters.js+13 -0 Viewed

                         data: base64Data,
                     },
                 });
+            } else if (part.type === 'video_url') {
+                const videoUrl = part.video_url?.url;
+                if (videoUrl && videoUrl.startsWith('data:')) {
+                    const [header, data] = videoUrl.split(',');
+                    const mimeType = header.match(/data:([^;]+)/)?.[1] || 'video/mp4';
+                    parts.push({
+                        inlineData: {
+                            mimeType: mimeType,
+                            data: data,
+                        },
+                    });
+                }
+            }
         });