Add automatic image captioning mode

7149f46c9aaadd31b8add999c06be4a5a7ac5888

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: 2670709237c18754b33ab9dcb14c709704cb0c6b
4 files changed, +85 -7Ignore whitespace
public/script.js+5 -0
@@ -408,6 +408,7 @@ export const event_types = {
408408 MESSAGE_EDITED: 'message_edited',
409409 MESSAGE_DELETED: 'message_deleted',
410410 MESSAGE_UPDATED: 'message_updated',
411+ MESSAGE_FILE_EMBEDDED: 'message_file_embedded',
411412 IMPERSONATE_READY: 'impersonate_ready',
412413 CHAT_CHANGED: 'chat_id_changed',
413414 GENERATION_STARTED: 'generation_started',
@@ -3404,6 +3405,10 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
34043405 let regexedMessage = getRegexedString(message, regexType, options);
34053406 regexedMessage = await appendFileContent(chatItem, regexedMessage);
34063407
3408+ if (chatItem?.extra?.append_title && chatItem?.extra?.title) {
3409+ regexedMessage = `${regexedMessage}\n\n${chatItem.extra.title}`;
3410+ }
3411+
34073412 return {
34083413 ...chatItem,
34093414 mes: regexedMessage,
public/scripts/chats.js+3 -0
@@ -417,6 +417,7 @@ function embedMessageFile(messageId, messageBlock) {
417417 }
418418
419419 await populateFileAttachment(message, 'embed_file_input');
420+ await eventSource.emit(event_types.MESSAGE_FILE_EMBEDDED, messageId);
420421 appendMediaToMessage(message, messageBlock);
421422 await saveChatConditional();
422423 }
@@ -614,6 +615,8 @@ async function deleteMessageImage() {
614615 const message = chat[mesId];
615616 delete message.extra.image;
616617 delete message.extra.inline_image;
618+ delete message.extra.title;
619+ delete message.extra.append_title;
617620 mesBlock.find('.mes_img_container').removeClass('img_extra');
618621 mesBlock.find('.mes_img').attr('src', '');
619622 await saveChatConditional();
public/scripts/extensions/caption/index.js+72 -7
@@ -1,6 +1,6 @@
11import { ensureImageFormatSupported, getBase64Async, isTrueBoolean, saveBase64AsFile } from '../../utils.js';
22import { getContext, getApiUrl, doExtrasFetch, extension_settings, modules, renderExtensionTemplateAsync } from '../../extensions.js';
33import { callPopup, eventSource, event_types, getRequestHeaders, saveSettingsDebounced, substituteParamsExtended } from '../../../script.js';
44import { getMessageTimeStamp } from '../../RossAscends-mods.js';
55import { SECRET_KEYS, secret_state } from '../../secrets.js';
66import { getMultimodalCaption } from '../shared.js';
@@ -84,12 +84,11 @@ async function setSpinnerIcon() {
8484}
8585
8686/**
8787 * SendsWraps a captionedcaption messagewith toa themessage chattemplate.
8888 * @param {string} caption CaptionRaw textcaption
8989 * @paramreturns {Promise<string>} image ImageWrapped URLcaption
9090 */
9191async function sendCaptionedMessagewrapCaptionTemplate(caption, image) {
92- const context = getContext();
9392 let template = extension_settings.caption.template || TEMPLATE_DEFAULT;
9493
9594 if (!/{{caption}}/i.test(template)) {
@@ -101,7 +100,7 @@ async function sendCaptionedMessage(caption, image) {
101100
102101 if (extension_settings.caption.refine_mode) {
103102 messageText = await callPopup(
104103 '<h3>Review and edit the generated messagecaption:</h3>Press "Cancel" to abort the caption sending.',
105104 'input',
106105 messageText,
107106 { rows: 5, okButton: 'Send' });
@@ -111,6 +110,55 @@ async function sendCaptionedMessage(caption, image) {
111110 }
112111 }
113112
113+ return messageText;
114+}
115+
116+/**
117+ * Appends caption to an existing message.
118+ * @param {Object} data Message data
119+ * @returns {Promise<void>}
120+ */
121+async function captionExistingMessage(data) {
122+ if (!(data?.extra?.image)) {
123+ return;
124+ }
125+
126+ const imageData = await fetch(data.extra.image);
127+ const blob = await imageData.blob();
128+ const type = imageData.headers.get('Content-Type');
129+ const file = new File([blob], 'image.png', { type });
130+ const caption = await getCaptionForFile(file, null, true);
131+
132+ if (!caption) {
133+ console.warn('Failed to generate a caption for the image.');
134+ return;
135+ }
136+
137+ const wrappedCaption = await wrapCaptionTemplate(caption);
138+
139+ const messageText = String(data.mes).trim();
140+
141+ if (!messageText) {
142+ data.extra.inline_image = false;
143+ data.mes = wrappedCaption;
144+ data.extra.title = wrappedCaption;
145+ }
146+ else {
147+ data.extra.inline_image = true;
148+ data.extra.append_title = true;
149+ data.extra.title = wrappedCaption;
150+ }
151+}
152+
153+/**
154+ * Sends a captioned message to the chat.
155+ * @param {string} caption Caption text
156+ * @param {string} image Image URL
157+ */
158+async function sendCaptionedMessage(caption, image) {
159+ const messageText = await wrapCaptionTemplate(caption);
160+
161+ const context = getContext();
114162 const message = {
115163 name: context.name1,
116164 is_user: true,
@@ -423,6 +471,7 @@ jQuery(async function () {
423471 $('#caption_refine_mode').prop('checked', !!(extension_settings.caption.refine_mode));
424472 $('#caption_allow_reverse_proxy').prop('checked', !!(extension_settings.caption.allow_reverse_proxy));
425473 $('#caption_prompt_ask').prop('checked', !!(extension_settings.caption.prompt_ask));
474+ $('#caption_auto_mode').prop('checked', !!(extension_settings.caption.auto_mode));
426475 $('#caption_source').val(extension_settings.caption.source);
427476 $('#caption_prompt').val(extension_settings.caption.prompt);
428477 $('#caption_template').val(extension_settings.caption.template);
@@ -448,6 +497,22 @@ jQuery(async function () {
448497 extension_settings.caption.prompt_ask = $('#caption_prompt_ask').prop('checked');
449498 saveSettingsDebounced();
450499 });
500+ $('#caption_auto_mode').on('input', () => {
501+ extension_settings.caption.auto_mode = !!$('#caption_auto_mode').prop('checked');
502+ saveSettingsDebounced();
503+ });
504+
505+ const onMessageEvent = async (index) => {
506+ if (!extension_settings.caption.auto_mode) {
507+ return;
508+ }
509+
510+ const data = getContext().chat[index];
511+ await captionExistingMessage(data);
512+ };
513+
514+ eventSource.on(event_types.MESSAGE_SENT, onMessageEvent);
515+ eventSource.on(event_types.MESSAGE_FILE_EMBEDDED, onMessageEvent);
451516
452517 SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'caption',
453518 callback: captionCommandCallback,
public/scripts/extensions/caption/settings.html+5 -0
@@ -92,6 +92,11 @@
9292 </div>
9393 <label for="caption_template"><span data-i18n="Message Template">Message Template</span> <small><span data-i18n="(use _space">(use </span> <code>&lcub;&lcub;caption&rcub;&rcub;</code> <span data-i18n="macro)">macro)</span></small></label>
9494 <textarea id="caption_template" class="text_pole" rows="2" placeholder="&lt; Use default &gt;">{{TEMPLATE_DEFAULT}}</textarea>
95+ <label class="checkbox_label" for="caption_auto_mode">
96+ <input id="caption_auto_mode" type="checkbox" class="checkbox">
97+ <span data-i18n="Automatically caption images">Automatically caption images</span>
98+ <i class="fa-solid fa-info-circle" title="Automatically caption images when they are pasted into the chat or attached to messages."></i>
99+ </label>
95100 <label class="checkbox_label margin-bot-10px" for="caption_refine_mode">
96101 <input id="caption_refine_mode" type="checkbox" class="checkbox">
97102 <span data-i18n="Edit captions before saving">Edit captions before saving</span>