Add captioning for video attachments (#4749) * Add captioning for video attachments * Unify error toast titles * Add MEDIA_SOURCE enum and update media handling to include source information * Unify attachment handling logic * Add error handling for auto-captioning failures * Use string formatting for console error

38679897c6193a7fbf547fb0d2bb5191aee2b3da

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

Parents: 4fcdec47f4a27e196a8ef851e824891d1d83af00

public/global.d.ts+4 -0 Viewed

         url: string;
         title?: string;
         type: string;
+        source?: string;
+    }
     interface ImageGenerationAttachmentProps {

@@ -78,7 +79,10 @@ declare global {
78	}	79	}
79		80
80	interface ImageCaptionAttachmentProps {	81	interface ImageCaptionAttachmentProps {
		82	/** Append title to the message text in case of non-inline captions. */
81	append_title?: boolean;	83	append_title?: boolean;
		84	/** Marker for captioned images to prevent auto-caption from firing again. */
		85	captioned?: boolean;
82	}	86	}
83		87
84	// Global namespace modules	88	// Global namespace modules

public/index.html+1 -0 Viewed

             <div class="mes_media_container mes_video_container">
                 <div class="mes_video_controls">
                     <div title="Expand and zoom" class="right_menu_button fa-lg fa-solid fa-magnifying-glass mes_media_enlarge" data-i18n="[title]Expand and zoom"></div>
+                    <div title="Caption" class="right_menu_button fa-lg fa-solid fa-envelope-open-text mes_img_caption" data-i18n="[title]Caption"></div>
                     <div title="Delete" class="right_menu_button fa-lg fa-solid fa-trash-can mes_media_delete" data-i18n="[title]Delete"></div>
                 </div>
                 <video class="mes_video" controls preload="metadata"></video>

public/script.js+2 -2 Viewed

     trimSpaces,
     clamp,
 } from './scripts/utils.js';
-import { debounce_timeout, GENERATION_TYPE_TRIGGERS, IGNORE_SYMBOL, inject_ids, MEDIA_DISPLAY, MEDIA_TYPE, SCROLL_BEHAVIOR, SWIPE_DIRECTION } from './scripts/constants.js';
+import { debounce_timeout, GENERATION_TYPE_TRIGGERS, IGNORE_SYMBOL, inject_ids, MEDIA_DISPLAY, MEDIA_SOURCE, MEDIA_TYPE, SCROLL_BEHAVIOR, SWIPE_DIRECTION } from './scripts/constants.js';
 import { cancelDebouncedMetadataSave, doDailyExtensionUpdatesCheck, extension_settings, initExtensions, loadExtensionSettings, runGenerationInterceptors, saveMetadataDebounced } from './scripts/extensions.js';
 import { COMMENT_NAME_DEFAULT, CONNECT_API_MAP, executeSlashCommandsOnChatInput, initDefaultSlashCommands, isExecutingCommandsFromChatInput, pauseScriptExecution, stopScriptExecution, UNIQUE_APIS } from './scripts/slash-commands.js';

@@ -6414,7 +6414,7 @@ function saveImageToMessage(img, mes) {
6414	if (!Array.isArray(mes.extra.media)) {	6414	if (!Array.isArray(mes.extra.media)) {
6415	mes.extra.media = [];	6415	mes.extra.media = [];
6416	}	6416	}
6417	mes.extra.media.push({ url: img.image, type: MEDIA_TYPE.IMAGE, title: img.title });	6417	mes.extra.media.push({ url: img.image, type: MEDIA_TYPE.IMAGE, title: img.title, source: MEDIA_SOURCE.API });
6418	mes.extra.inline_image = img.inline;	6418	mes.extra.inline_image = img.inline;
6419	}	6419	}
6420	}	6420	}

public/scripts/chats.js+9 -2 Viewed

 import { t } from './i18n.js';
 import { humanizedDateTime } from './RossAscends-mods.js';
 import { accountStorage } from './util/AccountStorage.js';
-import { MEDIA_DISPLAY, MEDIA_TYPE, SCROLL_BEHAVIOR, SWIPE_DIRECTION } from './constants.js';
+import { MEDIA_DISPLAY, MEDIA_SOURCE, MEDIA_TYPE, SCROLL_BEHAVIOR, SWIPE_DIRECTION } from './constants.js';
 /**
  * @typedef {Object} FileAttachment

                 if (!Array.isArray(message.extra.media)) {
                     message.extra.media = [];
+                }
-                message.extra.media.push({ url: imageUrl, type: mediaType, title: file.name });
+                /** @type {MediaAttachment} */
+                const mediaAttachment = {
+                    url: imageUrl,
+                    type: mediaType,
+                    title: file.name,
+                    source: MEDIA_SOURCE.UPLOAD,
+                };
+                message.extra.media.push(mediaAttachment);
                 message.extra.media_index = message.extra.media.length - 1;
                 message.extra.inline_image = true;
             } else {

public/scripts/constants.js+11 -0 Viewed

  * @enum {string}
  * @readonly
  */
+export const MEDIA_SOURCE = {
+    API: 'api',
+    UPLOAD: 'upload',
+    GENERATED: 'generated',
+    CAPTIONED: 'captioned',
+};
+/**
+ * @enum {string}
+ * @readonly
+ */
 export const MEDIA_DISPLAY = {
     LIST: 'list',
     GALLERY: 'gallery',

public/scripts/extensions/caption/index.js+82 -27 Viewed

 import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from '../../slash-commands/SlashCommandArgument.js';
 import { commonEnumProviders } from '../../slash-commands/SlashCommandCommonEnumsProvider.js';
 import { callGenericPopup, Popup, POPUP_TYPE } from '../../popup.js';
-import { MEDIA_DISPLAY, MEDIA_TYPE, SCROLL_BEHAVIOR } from '../../constants.js';
+import { MEDIA_DISPLAY, MEDIA_SOURCE, MEDIA_TYPE, SCROLL_BEHAVIOR } from '../../constants.js';
 export { MODULE_NAME };
 const MODULE_NAME = 'caption';

     const mediaAttachment = message.extra.media[mediaIndex];
-    if (!mediaAttachment || !mediaAttachment.url || mediaAttachment.type === MEDIA_TYPE.VIDEO) {
+    if (!mediaAttachment || !mediaAttachment.url || mediaAttachment.type === MEDIA_TYPE.AUDIO) {
         return;
+    }
+    if (mediaAttachment.type === MEDIA_TYPE.VIDEO && !isVideoCaptioningAvailable()) {
+        throw new Error('Captioning videos is not supported for the current source.');
+    }
     const imageData = await fetch(mediaAttachment.url);
     const blob = await imageData.blob();
-    const type = imageData.headers.get('Content-Type');
+    const fileName = mediaAttachment.url.split('/').pop().split('?')[0] || 'image.jpg';
-    const file = new File([blob], 'image.png', { type });
+    const file = new File([blob], fileName, { type: blob.type });
     const caption = await getCaptionForFile(file, null, true);
     if (!caption) {

         message.extra.inline_image = false;
         message.mes = wrappedCaption;
         mediaAttachment.title = wrappedCaption;
-    }
+        mediaAttachment.captioned = true;
-    else {
+    } else {
         message.extra.inline_image = true;
         mediaAttachment.append_title = true;
         mediaAttachment.title = wrappedCaption;
+        mediaAttachment.captioned = true;
+    }
+}

  * Sends a captioned message to the chat.
  * @param {string} caption Caption text
  * @param {string} image Image URL
+ * @param {string} mimeType Image MIME type
+ * @returns {Promise<void>}
  */
-async function sendCaptionedMessage(caption, image) {
+async function sendCaptionedMessage(caption, image, mimeType) {
     const messageText = await wrapCaptionTemplate(caption);
     const context = getContext();

     /** @type {MediaAttachment} */
     const mediaAttachment = {
         url: image,
-        type: MEDIA_TYPE.IMAGE,
+        type: MEDIA_TYPE.getFromMime(mimeType) || MEDIA_TYPE.IMAGE,
         title: messageText,
+        captioned: true,
+        source: MEDIA_SOURCE.CAPTIONED,
     };
     /** @type {ChatMessage} */
     const message = {

  */
 async function getCaptionForFile(file, prompt, quiet) {
     try {
+        if (file.type.startsWith('video/') && !isVideoCaptioningAvailable()) {
+            throw new Error('Video captioning is not available for the current source.');
+        }
         setSpinnerIcon();
         const context = getContext();
         const fileData = await getBase64Async(await ensureImageFormatSupported(file));

@@ -359,13 +372,13 @@ async function getCaptionForFile(file, prompt, quiet) {
359	const { caption } = await doCaptionRequest(base64Data, fileData, prompt);	372	const { caption } = await doCaptionRequest(base64Data, fileData, prompt);
360	if (!quiet) {	373	if (!quiet) {
361	const imagePath = await saveBase64AsFile(base64Data, context.name2, '', extension);	374	const imagePath = await saveBase64AsFile(base64Data, context.name2, '', extension);
362	await sendCaptionedMessage(caption, imagePath);	375	await sendCaptionedMessage(caption, imagePath, file.type);
363	}	376	}
364	return caption;	377	return caption;
365	}	378	}
366	catch (error) {	379	catch (error) {
367	const errorMessage = error.message \|\| 'Unknown error';	380	const errorMessage = error.message \|\| 'Unknown error';
368	toastr.error(errorMessage, 'Failed to caption image.');	381	toastr.error(errorMessage, 'Failed to caption');
369	console.error(error);	382	console.error(error);
370	return '';	383	return '';
371	}	384	}

                     toastr.error('The specified message does not contain an image.');
                     return '';
+                }
-                if (mediaAttachment.type === MEDIA_TYPE.VIDEO) {
+                if (mediaAttachment.type === MEDIA_TYPE.AUDIO) {
-                    toastr.error('The specified media is a video. Captioning videos is not supported.');
+                    toastr.error('The specified media is an audio file. Captioning audio files is not supported.');
+                    return '';
+                }
+                if (mediaAttachment.type === MEDIA_TYPE.VIDEO && !isVideoCaptioningAvailable()) {
+                    toastr.error('The specified media is a video. Captioning videos is not supported for the current source.');
                     return '';
+                }
                 const fetchResult = await fetch(mediaAttachment.url);
                 const blob = await fetchResult.blob();
-                const file = new File([blob], 'image.jpg', { type: blob.type });
+                const fileName = mediaAttachment.url.split('/').pop().split('?')[0] || 'image.jpg';
+                const file = new File([blob], fileName, { type: blob.type });
                 return await getCaptionForFile(file, prompt, quiet);
             } catch (error) {
                 toastr.error('Failed to get image from the message. Make sure the image is accessible.');

     return new Promise(resolve => {
         const input = document.createElement('input');
         input.type = 'file';
-        input.accept = 'image/*';
+        input.accept = 'image/*,video/*';
         input.onchange = async (e) => {
             const caption = await onSelectImage(e, prompt, quiet);
             resolve(caption);

     });
+}
+/**
+ * Checks if video captioning is available for the current source.
+ * @returns {boolean} True if video captioning is supported for the current source.
+ */
+function isVideoCaptioningAvailable() {
+    if (extension_settings.caption.source !== 'multimodal') {
+        return false;
+    }
+    return ['google', 'vertexai'].includes(extension_settings.caption.multimodal_api);
+}
 jQuery(async function () {
     function addSendPictureButton() {
         const sendButton = $(`

         });
+    }
     function addPictureSendForm() {
-        const inputHtml = '<input id="img_file" type="file" hidden accept="image/*">';
+        const imgInput = document.createElement('input');
+        imgInput.type = 'file';
+        imgInput.id = 'img_file';
+        imgInput.accept = 'image/*,video/*';
+        imgInput.hidden = true;
+        imgInput.addEventListener('change', (e) => onSelectImage(e, '', false));
         const imgForm = document.createElement('form');
         imgForm.id = 'img_form';
-        $(imgForm).append(inputHtml);
+        imgForm.appendChild(imgInput);
-        $(imgForm).hide();
+        imgForm.hidden = true;
         $('#form_sheld').append(imgForm);
-        $('#img_file').on('change', (e) => onSelectImage(e.originalEvent, '', false));
+    }
     async function switchMultimodalBlocks() {
         await addRemoteEndpointModels();

         saveSettingsDebounced();
     });
-    const onMessageEvent = async (messageId) => {
+    const onMessageEvent = async (/** @type {number} */ messageId) => {
         if (!extension_settings.caption.auto_mode) {
             return;
+        }
         const message = getContext().chat[messageId];
-        await captionExistingMessage(message, 0);
+        if (Array.isArray(message?.extra?.media) && message.extra.media.length > 0) {
+            for (let mediaIndex = 0; mediaIndex < message.extra.media.length; mediaIndex++) {
+                const mediaAttachment = message.extra.media[mediaIndex];
+                if (mediaAttachment.type === MEDIA_TYPE.VIDEO && !isVideoCaptioningAvailable()) {
+                    continue;
+                }
+                if (mediaAttachment.type === MEDIA_TYPE.AUDIO) {
+                    continue;
+                }
+                // Skip already captioned images and non-uploaded (generated, etc.) images
+                if (mediaAttachment.source !== MEDIA_SOURCE.UPLOAD || mediaAttachment.captioned) {
+                    continue;
+                }
+                try {
+                    await captionExistingMessage(message, mediaIndex);
+                } catch (e) {
+                    console.error(`Auto-captioning failed for message ID ${messageId}, media index ${mediaIndex}`, e);
+                    continue;
+                }
+            }
+        }
     };
     eventSource.on(event_types.MESSAGE_SENT, onMessageEvent);

     $(document).on('click', '.mes_img_caption', async function () {
         const animationClass = 'fa-fade';
         const messageBlock = $(this).closest('.mes');
-        const imageBlock = $(this).closest('.mes_img_container');
+        const mediaContainer = $(this).closest('.mes_media_container');
-        const messageImg = imageBlock.find('.mes_img');
+        const messageMedia = mediaContainer.find('.mes_img, .mes_video');
-        if (messageImg.hasClass(animationClass)) return;
+        if (messageMedia.hasClass(animationClass)) return;
-        messageImg.addClass(animationClass);
+        messageMedia.addClass(animationClass);
         try {
             const messageId = Number(messageBlock.attr('mesid'));
-            const imageIndex = Number(imageBlock.attr('data-index'));
+            const mediaIndex = Number(mediaContainer.attr('data-index'));
             const data = getContext().chat[messageId];
-            await captionExistingMessage(data, imageIndex);
+            await captionExistingMessage(data, mediaIndex);
             appendMediaToMessage(data, messageBlock, SCROLL_BEHAVIOR.KEEP);
             await saveChatConditional();
         } catch (e) {
             console.error('Message image recaption failed', e);
+            toastr.error(e.message || 'Unknown error', 'Failed to caption');
         } finally {
-            messageImg.removeClass(animationClass);
+            messageMedia.removeClass(animationClass);
+        }
     });

public/scripts/extensions/shared.js+4 -3 Viewed

     const base64Bytes = base64Img.length * 0.75;
     const compressionLimit = 2 * 1024 * 1024;
     const safeMimeTypes = ['image/jpeg', 'image/png', 'image/webp'];
-    const mimeType = base64Img?.split(';')?.[0]?.split(':')?.[1];
+    const mimeType = base64Img?.split(';')?.[0]?.split(':')?.[1] || 'image/jpeg';
+    const isImage = mimeType.startsWith('image/');
     const thumbnailNeeded = ['google', 'openrouter', 'mistral', 'groq', 'vertexai'].includes(extension_settings.caption.multimodal_api);
-    if ((thumbnailNeeded && base64Bytes > compressionLimit) || isOoba || isKoboldCpp) {
+    if ((isImage && thumbnailNeeded && base64Bytes > compressionLimit) || isOoba || isKoboldCpp) {
         const maxSide = 2048;
         base64Img = await createThumbnail(base64Img, maxSide, maxSide);
-    } else if (!safeMimeTypes.includes(mimeType)) {
+    } else if (isImage && !safeMimeTypes.includes(mimeType)) {
         base64Img = await createThumbnail(base64Img, null, null);
+    }
     if (isOllama && base64Img.startsWith('data:image/')) {

public/scripts/extensions/stable-diffusion/index.js+3 -1 Viewed

     SlashCommandArgument,
     SlashCommandNamedArgument,
 } from '../../slash-commands/SlashCommandArgument.js';
-import { debounce_timeout, MEDIA_DISPLAY, MEDIA_TYPE, SCROLL_BEHAVIOR, VIDEO_EXTENSIONS } from '../../constants.js';
+import { debounce_timeout, MEDIA_DISPLAY, MEDIA_SOURCE, MEDIA_TYPE, SCROLL_BEHAVIOR, VIDEO_EXTENSIONS } from '../../constants.js';
 import { SlashCommandEnumValue } from '../../slash-commands/SlashCommandEnumValue.js';
 import { callGenericPopup, Popup, POPUP_TYPE } from '../../popup.js';
 import { commonEnumProviders } from '../../slash-commands/SlashCommandCommonEnumsProvider.js';

         title: prompt,
         generation_type: generationType,
         negative: additionalNegativePrefix,
+        source: MEDIA_SOURCE.GENERATED,
     };
     /** @type {ChatMessage} */
     const message = {

     const result = {
         url: '',
         type: MEDIA_TYPE.IMAGE,
+        source: MEDIA_SOURCE.GENERATED,
     };
     try {