Add captioning for video attachments (#4749) * Add captioning for video attachments * Unify error toast titles * Add MEDIA_SOURCE enum and update media handling to include source information * Unify attachment handling logic * Add error handling for auto-captioning failures * Use string formatting for console error

38679897c6193a7fbf547fb0d2bb5191aee2b3da

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: 4fcdec47f4a27e196a8ef851e824891d1d83af00
8 files changed, +116 -35Showing whitespace changes
public/global.d.ts+4 -0
@@ -70,6 +70,7 @@ declare global {
7070 url: string;
7171 title?: string;
7272 type: string;
73+ source?: string;
7374 }
7475
7576 interface ImageGenerationAttachmentProps {
@@ -78,7 +79,10 @@ declare global {
7879 }
7980
8081 interface ImageCaptionAttachmentProps {
82+ /** Append title to the message text in case of non-inline captions. */
8183 append_title?: boolean;
84+ /** Marker for captioned images to prevent auto-caption from firing again. */
85+ captioned?: boolean;
8286 }
8387
8488 // Global namespace modules
public/index.html+1 -0
@@ -7295,6 +7295,7 @@
72957295 <div class="mes_media_container mes_video_container">
72967296 <div class="mes_video_controls">
72977297 <div title="Expand and zoom" class="right_menu_button fa-lg fa-solid fa-magnifying-glass mes_media_enlarge" data-i18n="[title]Expand and zoom"></div>
7298+ <div title="Caption" class="right_menu_button fa-lg fa-solid fa-envelope-open-text mes_img_caption" data-i18n="[title]Caption"></div>
72987299 <div title="Delete" class="right_menu_button fa-lg fa-solid fa-trash-can mes_media_delete" data-i18n="[title]Delete"></div>
72997300 </div>
73007301 <video class="mes_video" controls preload="metadata"></video>
public/script.js+2 -2
@@ -183,7 +183,7 @@ import {
183183 trimSpaces,
184184 clamp,
185185} from './scripts/utils.js';
186186import { debounce_timeout, GENERATION_TYPE_TRIGGERS, IGNORE_SYMBOL, inject_ids, MEDIA_DISPLAY, MEDIA_SOURCE, MEDIA_TYPE, SCROLL_BEHAVIOR, SWIPE_DIRECTION } from './scripts/constants.js';
187187
188188import { cancelDebouncedMetadataSave, doDailyExtensionUpdatesCheck, extension_settings, initExtensions, loadExtensionSettings, runGenerationInterceptors, saveMetadataDebounced } from './scripts/extensions.js';
189189import { COMMENT_NAME_DEFAULT, CONNECT_API_MAP, executeSlashCommandsOnChatInput, initDefaultSlashCommands, isExecutingCommandsFromChatInput, pauseScriptExecution, stopScriptExecution, UNIQUE_APIS } from './scripts/slash-commands.js';
@@ -6414,7 +6414,7 @@ function saveImageToMessage(img, mes) {
64146414 if (!Array.isArray(mes.extra.media)) {
64156415 mes.extra.media = [];
64166416 }
64176417 mes.extra.media.push({ url: img.image, type: MEDIA_TYPE.IMAGE, title: img.title, source: MEDIA_SOURCE.API });
64186418 mes.extra.inline_image = img.inline;
64196419 }
64206420}
public/scripts/chats.js+9 -2
@@ -57,7 +57,7 @@ import { renderTemplateAsync } from './templates.js';
5757import { t } from './i18n.js';
5858import { humanizedDateTime } from './RossAscends-mods.js';
5959import { accountStorage } from './util/AccountStorage.js';
6060import { MEDIA_DISPLAY, MEDIA_SOURCE, MEDIA_TYPE, SCROLL_BEHAVIOR, SWIPE_DIRECTION } from './constants.js';
6161
6262/**
6363 * @typedef {Object} FileAttachment
@@ -216,7 +216,14 @@ export async function populateFileAttachment(message, inputId = 'file_form_input
216216 if (!Array.isArray(message.extra.media)) {
217217 message.extra.media = [];
218218 }
219- message.extra.media.push({ url: imageUrl, type: mediaType, title: file.name });
219+ /** @type {MediaAttachment} */
220+ const mediaAttachment = {
221+ url: imageUrl,
222+ type: mediaType,
223+ title: file.name,
224+ source: MEDIA_SOURCE.UPLOAD,
225+ };
226+ message.extra.media.push(mediaAttachment);
220227 message.extra.media_index = message.extra.media.length - 1;
221228 message.extra.inline_image = true;
222229 } else {
public/scripts/constants.js+11 -0
@@ -71,6 +71,17 @@ export const COMETAPI_IGNORE_PATTERNS = [
7171 * @enum {string}
7272 * @readonly
7373 */
74+export const MEDIA_SOURCE = {
75+ API: 'api',
76+ UPLOAD: 'upload',
77+ GENERATED: 'generated',
78+ CAPTIONED: 'captioned',
79+};
80+
81+/**
82+ * @enum {string}
83+ * @readonly
84+ */
7485export const MEDIA_DISPLAY = {
7586 LIST: 'list',
7687 GALLERY: 'gallery',
public/scripts/extensions/caption/index.js+82 -27
@@ -10,7 +10,7 @@ import { SlashCommand } from '../../slash-commands/SlashCommand.js';
1010import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from '../../slash-commands/SlashCommandArgument.js';
1111import { commonEnumProviders } from '../../slash-commands/SlashCommandCommonEnumsProvider.js';
1212import { callGenericPopup, Popup, POPUP_TYPE } from '../../popup.js';
1313import { MEDIA_DISPLAY, MEDIA_SOURCE, MEDIA_TYPE, SCROLL_BEHAVIOR } from '../../constants.js';
1414export { MODULE_NAME };
1515
1616const MODULE_NAME = 'caption';
@@ -135,14 +135,18 @@ async function captionExistingMessage(message, mediaIndex) {
135135
136136 const mediaAttachment = message.extra.media[mediaIndex];
137137
138138 if (!mediaAttachment || !mediaAttachment.url || mediaAttachment.type === MEDIA_TYPE.VIDEOAUDIO) {
139139 return;
140140 }
141141
142+ if (mediaAttachment.type === MEDIA_TYPE.VIDEO && !isVideoCaptioningAvailable()) {
143+ throw new Error('Captioning videos is not supported for the current source.');
144+ }
145+
142146 const imageData = await fetch(mediaAttachment.url);
143147 const blob = await imageData.blob();
144- const type = imageData.headers.get('Content-Type');
148+ const fileName = mediaAttachment.url.split('/').pop().split('?')[0] || 'image.jpg';
145149 const file = new File([blob], 'image.png'fileName, { type: blob.type });
146150 const caption = await getCaptionForFile(file, null, true);
147151
148152 if (!caption) {
@@ -158,11 +162,12 @@ async function captionExistingMessage(message, mediaIndex) {
158162 message.extra.inline_image = false;
159163 message.mes = wrappedCaption;
160164 mediaAttachment.title = wrappedCaption;
161- }
165+ mediaAttachment.captioned = true;
162166 } else {
163167 message.extra.inline_image = true;
164168 mediaAttachment.append_title = true;
165169 mediaAttachment.title = wrappedCaption;
170+ mediaAttachment.captioned = true;
166171 }
167172}
168173
@@ -170,8 +175,10 @@ async function captionExistingMessage(message, mediaIndex) {
170175 * Sends a captioned message to the chat.
171176 * @param {string} caption Caption text
172177 * @param {string} image Image URL
178+ * @param {string} mimeType Image MIME type
179+ * @returns {Promise<void>}
173180 */
174181async function sendCaptionedMessage(caption, image, mimeType) {
175182 const messageText = await wrapCaptionTemplate(caption);
176183
177184 const context = getContext();
@@ -179,8 +186,10 @@ async function sendCaptionedMessage(caption, image) {
179186 /** @type {MediaAttachment} */
180187 const mediaAttachment = {
181188 url: image,
182189 type: MEDIA_TYPE.getFromMime(mimeType) || MEDIA_TYPE.IMAGE,
183190 title: messageText,
191+ captioned: true,
192+ source: MEDIA_SOURCE.CAPTIONED,
184193 };
185194 /** @type {ChatMessage} */
186195 const message = {
@@ -351,6 +360,10 @@ async function onSelectImage(e, prompt, quiet) {
351360 */
352361async function getCaptionForFile(file, prompt, quiet) {
353362 try {
363+ if (file.type.startsWith('video/') && !isVideoCaptioningAvailable()) {
364+ throw new Error('Video captioning is not available for the current source.');
365+ }
366+
354367 setSpinnerIcon();
355368 const context = getContext();
356369 const fileData = await getBase64Async(await ensureImageFormatSupported(file));
@@ -359,13 +372,13 @@ async function getCaptionForFile(file, prompt, quiet) {
359372 const { caption } = await doCaptionRequest(base64Data, fileData, prompt);
360373 if (!quiet) {
361374 const imagePath = await saveBase64AsFile(base64Data, context.name2, '', extension);
362375 await sendCaptionedMessage(caption, imagePath, file.type);
363376 }
364377 return caption;
365378 }
366379 catch (error) {
367380 const errorMessage = error.message || 'Unknown error';
368381 toastr.error(errorMessage, 'Failed to caption image.');
369382 console.error(error);
370383 return '';
371384 }
@@ -399,13 +412,18 @@ async function captionCommandCallback(args, prompt) {
399412 toastr.error('The specified message does not contain an image.');
400413 return '';
401414 }
402415 if (mediaAttachment.type === MEDIA_TYPE.VIDEOAUDIO) {
403416 toastr.error('The specified media is aan videoaudio file. Captioning videosaudio files is not supported.');
417+ return '';
418+ }
419+ if (mediaAttachment.type === MEDIA_TYPE.VIDEO && !isVideoCaptioningAvailable()) {
420+ toastr.error('The specified media is a video. Captioning videos is not supported for the current source.');
404421 return '';
405422 }
406423 const fetchResult = await fetch(mediaAttachment.url);
407424 const blob = await fetchResult.blob();
408- const file = new File([blob], 'image.jpg', { type: blob.type });
425+ const fileName = mediaAttachment.url.split('/').pop().split('?')[0] || 'image.jpg';
426+ const file = new File([blob], fileName, { type: blob.type });
409427 return await getCaptionForFile(file, prompt, quiet);
410428 } catch (error) {
411429 toastr.error('Failed to get image from the message. Make sure the image is accessible.');
@@ -417,7 +435,7 @@ async function captionCommandCallback(args, prompt) {
417435 return new Promise(resolve => {
418436 const input = document.createElement('input');
419437 input.type = 'file';
420438 input.accept = 'image/*,video/*';
421439 input.onchange = async (e) => {
422440 const caption = await onSelectImage(e, prompt, quiet);
423441 resolve(caption);
@@ -427,6 +445,18 @@ async function captionCommandCallback(args, prompt) {
427445 });
428446}
429447
448+/**
449+ * Checks if video captioning is available for the current source.
450+ * @returns {boolean} True if video captioning is supported for the current source.
451+ */
452+function isVideoCaptioningAvailable() {
453+ if (extension_settings.caption.source !== 'multimodal') {
454+ return false;
455+ }
456+
457+ return ['google', 'vertexai'].includes(extension_settings.caption.multimodal_api);
458+}
459+
430460jQuery(async function () {
431461 function addSendPictureButton() {
432462 const sendButton = $(`
@@ -515,13 +545,17 @@ jQuery(async function () {
515545 });
516546 }
517547 function addPictureSendForm() {
518- const inputHtml = '<input id="img_file" type="file" hidden accept="image/*">';
548+ const imgInput = document.createElement('input');
549+ imgInput.type = 'file';
550+ imgInput.id = 'img_file';
551+ imgInput.accept = 'image/*,video/*';
552+ imgInput.hidden = true;
553+ imgInput.addEventListener('change', (e) => onSelectImage(e, '', false));
519554 const imgForm = document.createElement('form');
520555 imgForm.id = 'img_form';
521556 $(imgForm).appendappendChild(inputHtmlimgInput);
522- $(imgForm).hide();
557+ imgForm.hidden = true;
523558 $('#form_sheld').append(imgForm);
524- $('#img_file').on('change', (e) => onSelectImage(e.originalEvent, '', false));
525559 }
526560 async function switchMultimodalBlocks() {
527561 await addRemoteEndpointModels();
@@ -668,13 +702,33 @@ jQuery(async function () {
668702 saveSettingsDebounced();
669703 });
670704
671705 const onMessageEvent = async (/** @type {number} */ messageId) => {
672706 if (!extension_settings.caption.auto_mode) {
673707 return;
674708 }
675709
676710 const message = getContext().chat[messageId];
677- await captionExistingMessage(message, 0);
711+ if (Array.isArray(message?.extra?.media) && message.extra.media.length > 0) {
712+ for (let mediaIndex = 0; mediaIndex < message.extra.media.length; mediaIndex++) {
713+ const mediaAttachment = message.extra.media[mediaIndex];
714+ if (mediaAttachment.type === MEDIA_TYPE.VIDEO && !isVideoCaptioningAvailable()) {
715+ continue;
716+ }
717+ if (mediaAttachment.type === MEDIA_TYPE.AUDIO) {
718+ continue;
719+ }
720+ // Skip already captioned images and non-uploaded (generated, etc.) images
721+ if (mediaAttachment.source !== MEDIA_SOURCE.UPLOAD || mediaAttachment.captioned) {
722+ continue;
723+ }
724+ try {
725+ await captionExistingMessage(message, mediaIndex);
726+ } catch (e) {
727+ console.error(`Auto-captioning failed for message ID ${messageId}, media index ${mediaIndex}`, e);
728+ continue;
729+ }
730+ }
731+ }
678732 };
679733
680734 eventSource.on(event_types.MESSAGE_SENT, onMessageEvent);
@@ -683,21 +737,22 @@ jQuery(async function () {
683737 $(document).on('click', '.mes_img_caption', async function () {
684738 const animationClass = 'fa-fade';
685739 const messageBlock = $(this).closest('.mes');
686740 const imageBlockmediaContainer = $(this).closest('.mes_img_containermes_media_container');
687741 const messageImgmessageMedia = imageBlockmediaContainer.find('.mes_img, .mes_video');
688742 if (messageImgmessageMedia.hasClass(animationClass)) return;
689743 messageImgmessageMedia.addClass(animationClass);
690744 try {
691745 const messageId = Number(messageBlock.attr('mesid'));
692746 const imageIndexmediaIndex = Number(imageBlockmediaContainer.attr('data-index'));
693747 const data = getContext().chat[messageId];
694748 await captionExistingMessage(data, imageIndexmediaIndex);
695749 appendMediaToMessage(data, messageBlock, SCROLL_BEHAVIOR.KEEP);
696750 await saveChatConditional();
697751 } catch (e) {
698752 console.error('Message image recaption failed', e);
753+ toastr.error(e.message || 'Unknown error', 'Failed to caption');
699754 } finally {
700755 messageImgmessageMedia.removeClass(animationClass);
701756 }
702757 });
703758
public/scripts/extensions/shared.js+4 -3
@@ -33,12 +33,13 @@ export async function getMultimodalCaption(base64Img, prompt) {
3333 const base64Bytes = base64Img.length * 0.75;
3434 const compressionLimit = 2 * 1024 * 1024;
3535 const safeMimeTypes = ['image/jpeg', 'image/png', 'image/webp'];
3636 const mimeType = base64Img?.split(';')?.[0]?.split(':')?.[1] || 'image/jpeg';
37+ const isImage = mimeType.startsWith('image/');
3738 const thumbnailNeeded = ['google', 'openrouter', 'mistral', 'groq', 'vertexai'].includes(extension_settings.caption.multimodal_api);
3839 if ((isImage && thumbnailNeeded && base64Bytes > compressionLimit) || isOoba || isKoboldCpp) {
3940 const maxSide = 2048;
4041 base64Img = await createThumbnail(base64Img, maxSide, maxSide);
4142 } else if (isImage && !safeMimeTypes.includes(mimeType)) {
4243 base64Img = await createThumbnail(base64Img, null, null);
4344 }
4445 if (isOllama && base64Img.startsWith('data:image/')) {
public/scripts/extensions/stable-diffusion/index.js+3 -1
@@ -52,7 +52,7 @@ import {
5252 SlashCommandArgument,
5353 SlashCommandNamedArgument,
5454} from '../../slash-commands/SlashCommandArgument.js';
5555import { debounce_timeout, MEDIA_DISPLAY, MEDIA_SOURCE, MEDIA_TYPE, SCROLL_BEHAVIOR, VIDEO_EXTENSIONS } from '../../constants.js';
5656import { SlashCommandEnumValue } from '../../slash-commands/SlashCommandEnumValue.js';
5757import { callGenericPopup, Popup, POPUP_TYPE } from '../../popup.js';
5858import { commonEnumProviders } from '../../slash-commands/SlashCommandCommonEnumsProvider.js';
@@ -4123,6 +4123,7 @@ async function sendMessage(prompt, image, generationType, additionalNegativePref
41234123 title: prompt,
41244124 generation_type: generationType,
41254125 negative: additionalNegativePrefix,
4126+ source: MEDIA_SOURCE.GENERATED,
41264127 };
41274128 /** @type {ChatMessage} */
41284129 const message = {
@@ -4386,6 +4387,7 @@ async function generateMediaSwipe(mediaAttachment, message, onStart, onComplete,
43864387 const result = {
43874388 url: '',
43884389 type: MEDIA_TYPE.IMAGE,
4390+ source: MEDIA_SOURCE.GENERATED,
43894391 };
43904392
43914393 try {