Support multiple images in single API response (#4734) * Support multiple images in single API response * Fix media wrapper clashing with swipe counter * Return an empty array, add additional filtering * Add index to inline image filenames in processImageAttachment

c04be57f84b041a4e6cd23e2d22cf4d7a63f9d27

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

Parents: c14fba88298867ddec8b33728cecc2bb8ebabec6

public/script.js+34 -28 Viewed

         this.reasoningHandler = new ReasoningHandler(timeStarted);
         /** @type {PromptReasoning} */
         this.promptReasoning = promptReasoning;
-        /** @type {string} */
+        /** @type {string[]} */
-        this.image = '';
+        this.images = [];
+    }
     /**

@@ -3309,8 +3309,8 @@ class StreamingProcessor {
3309	chat[messageId].swipe_info.push(...swipeInfoArray);	3309	chat[messageId].swipe_info.push(...swipeInfoArray);
3310	}	3310	}
3311		3311
3312	if (this.image) {	3312	if (Array.isArray(this.images) && this.images.length > 0) {
3313	await processImageAttachment(chat[messageId], { imageUrl: this.image });	3313	await processImageAttachment(chat[messageId], { imageUrls: this.images });
3314	appendMediaToMessage(chat[messageId], $(this.messageDom));	3314	appendMediaToMessage(chat[messageId], $(this.messageDom));
3315	}	3315	}
3316		3316

@@ -3408,7 +3408,7 @@ class StreamingProcessor {
3408	}	3408	}
3409	// Get the updated reasoning string into the handler	3409	// Get the updated reasoning string into the handler
3410	this.reasoningHandler.updateReasoning(this.messageId, state?.reasoning);	3410	this.reasoningHandler.updateReasoning(this.messageId, state?.reasoning);
3411	this.image = state?.image ?? '';	3411	this.images = state?.images ?? [];
3412	await eventSource.emit(event_types.STREAM_TOKEN_RECEIVED, text);	3412	await eventSource.emit(event_types.STREAM_TOKEN_RECEIVED, text);
3413	await sw.tick(async () => await this.onProgressStreaming(this.messageId, this.continueMessage + text));	3413	await sw.tick(async () => await this.onProgressStreaming(this.messageId, this.continueMessage + text));
3414	}	3414	}

         let getMessage = extractMessageFromData(data);
         let title = extractTitleFromData(data);
         let reasoning = extractReasoningFromData(data);
-        let imageUrl = extractImageFromData(data);
+        let imageUrls = extractImagesFromData(data);
         kobold_horde_model = title;
         const swipes = extractMultiSwipes(data, type);

@@ -5014,10 +5014,10 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
5014	else {	5014	else {
5015	// Without streaming we'll be having a full message on continuation. Treat it as a last chunk.	5015	// Without streaming we'll be having a full message on continuation. Treat it as a last chunk.
5016	if (originalType !== 'continue') {	5016	if (originalType !== 'continue') {
5017	({ type, getMessage } = await saveReply({ type, getMessage, title, swipes, reasoning, imageUrl }));	5017	({ type, getMessage } = await saveReply({ type, getMessage, title, swipes, reasoning, imageUrls }));
5018	}	5018	}
5019	else {	5019	else {
5020	({ type, getMessage } = await saveReply({ type: 'appendFinal', getMessage, title, swipes, reasoning, imageUrl }));	5020	({ type, getMessage } = await saveReply({ type: 'appendFinal', getMessage, title, swipes, reasoning, imageUrls }));
5021	}	5021	}
5022		5022
5023	// This relies on `saveReply` having been called to add the message to the chat, so it must be last.	5023	// This relies on `saveReply` having been called to add the message to the chat, so it must be last.

  * @param {object} [options] Extraction options
  * @param {string} [options.mainApi] Main API to use
  * @param {string} [options.chatCompletionSource] Chat completion source
- * @returns {string} Extracted image
+ * @returns {string[]} Extracted images or empty array
  */
-function extractImageFromData(data, { mainApi = null, chatCompletionSource = null } = {}) {
+function extractImagesFromData(data, { mainApi = null, chatCompletionSource = null } = {}) {
     switch (mainApi ?? main_api) {
         case 'openai': {
             switch (chatCompletionSource ?? oai_settings.chat_completion_source) {
                 case chat_completion_sources.VERTEXAI:
                 case chat_completion_sources.MAKERSUITE: {
-                    const inlineData = data?.responseContent?.parts?.find(x => x.inlineData)?.inlineData;
+                    const inlineData = data?.responseContent?.parts?.filter(x => x.inlineData)?.map(x => x.inlineData);
-                    if (inlineData) {
+                    if (Array.isArray(inlineData) && inlineData.length > 0) {
-                        return `data:${inlineData.mimeType};base64,${inlineData.data}`;
+                        return inlineData.map(x => `data:${x.mimeType};base64,${x.data}`).filter(isDataURL);
+                    }
                 } break;
                 case chat_completion_sources.OPENROUTER: {
-                    const imageUrl = data?.choices[0]?.message?.images?.find(x => x.type === 'image_url')?.image_url?.url;
+                    const imageUrl = data?.choices[0]?.message?.images?.filter(x => x.type === 'image_url')?.map(x => x?.image_url?.url);
-                    if (isDataURL(imageUrl)) {
+                    if (Array.isArray(imageUrl) && imageUrl.length > 0) {
-                        return imageUrl;
+                        return imageUrl.filter(isDataURL);
+                    }
                     // TODO: Handle remote URLs
+                }

@@ -5665,7 +5665,7 @@ function extractImageFromData(data, { mainApi = null, chatCompletionSource = nul
5665	} break;	5665	} break;
5666	}	5666	}
5667		5667
5668	return undefined;	5668	return [];
5669	}	5669	}
5670		5670
5671	/**	5671	/**

  * Adds an image to the message.
  * @param {object} message Message object
  * @param {object} sources Image sources
- * @param {string} [sources.imageUrl] Image URL
+ * @param {string[]} [sources.imageUrls] Image URLs
+ *
  * @returns {Promise<void>}
  */
-async function processImageAttachment(message, { imageUrl }) {
+async function processImageAttachment(message, { imageUrls }) {
-    if (!imageUrl) {
+    if (!Array.isArray(imageUrls) || imageUrls.length === 0) {
         return;
+    }
+    for (const [index, imageUrl] of imageUrls.entries()) {
+        if (!imageUrl) {
+            continue;
+        }
         let url = imageUrl;
         if (isDataURL(url)) {
-        const fileName = `inline_image_${Date.now().toString()}`;
+            const fileName = `inline_image_${Date.now().toString()}_${index}`;
             const [mime, base64] = /^data:(.*?);base64,(.*)$/.exec(imageUrl).slice(1);
             url = await saveBase64AsFile(base64, message.name, fileName, mime.split('/')[1]);
+        }
         saveImageToMessage({ image: url, inline: true }, message);
+    }
+}
 /**
  * Saves a resulting message to the chat.

  * @property {string} [title] Message tooltip
  * @property {string[]} [swipes] Extra swipes
  * @property {string} [reasoning] Message reasoning
- * @property {string} [imageUrl] Link to an image
+ * @property {string[]} [imageUrls] Links to images
+ *
  * @typedef {object} SaveReplyResult
  * @property {string} type Type of generation
  * @property {string} getMessage Generated message
  */
-export async function saveReply({ type, getMessage, fromStreaming = false, title = '', swipes = [], reasoning = '', imageUrl = '' }) {
+export async function saveReply({ type, getMessage, fromStreaming = false, title = '', swipes = [], reasoning = '', imageUrls = [] }) {
     // Backward compatibility
     if (arguments.length > 1 && typeof arguments[0] !== 'object') {
         console.trace('saveReply called with positional arguments. Please use an object instead.');
-        [type, getMessage, fromStreaming, title, swipes, reasoning, imageUrl] = arguments;
+        [type, getMessage, fromStreaming, title, swipes, reasoning, imageUrls] = arguments;
+    }
     if (type != 'append' && type != 'continue' && type != 'appendFinal' && chat.length && (chat[chat.length - 1]['swipe_id'] === undefined ||

             chat[chat.length - 1]['extra']['model'] = getGeneratingModel();
             chat[chat.length - 1]['extra']['reasoning'] = reasoning;
             chat[chat.length - 1]['extra']['reasoning_duration'] = null;
-            await processImageAttachment(chat[chat.length - 1], { imageUrl });
+            await processImageAttachment(chat[chat.length - 1], { imageUrls });
             if (power_user.message_token_count_enabled) {
                 const tokenCountText = (reasoning || '') + chat[chat.length - 1]['mes'];
                 chat[chat.length - 1]['extra']['token_count'] = await getTokenCountAsync(tokenCountText, 0);

         chat[chat.length - 1]['extra']['model'] = getGeneratingModel();
         chat[chat.length - 1]['extra']['reasoning'] = reasoning;
         chat[chat.length - 1]['extra']['reasoning_duration'] = null;
-        await processImageAttachment(chat[chat.length - 1], { imageUrl });
+        await processImageAttachment(chat[chat.length - 1], { imageUrls });
         if (power_user.message_token_count_enabled) {
             const tokenCountText = (reasoning || '') + chat[chat.length - 1]['mes'];
             chat[chat.length - 1]['extra']['token_count'] = await getTokenCountAsync(tokenCountText, 0);

         chat[chat.length - 1]['extra']['api'] = getGeneratingApi();
         chat[chat.length - 1]['extra']['model'] = getGeneratingModel();
         chat[chat.length - 1]['extra']['reasoning'] += reasoning;
-        await processImageAttachment(chat[chat.length - 1], { imageUrl });
+        await processImageAttachment(chat[chat.length - 1], { imageUrls });
         // We don't know if the reasoning duration extended, so we don't update it here on purpose.
         if (power_user.message_token_count_enabled) {
             const tokenCountText = (reasoning || '') + chat[chat.length - 1]['mes'];

             chat[chat.length - 1]['extra']['gen_id'] = group_generation_id;
+        }
-        await processImageAttachment(chat[chat.length - 1], { imageUrl });
+        await processImageAttachment(chat[chat.length - 1], { imageUrls });
         const chat_id = (chat.length - 1);
         !fromStreaming && await eventSource.emit(event_types.MESSAGE_RECEIVED, chat_id, type);

public/scripts/openai.js+7 -7 Viewed

             let text = '';
             const swipes = [];
             const toolCalls = [];
-            const state = { reasoning: '', image: '' };
+            const state = { reasoning: '', images: [] };
             while (true) {
                 const { done, value } = await reader.read();
                 if (done) return;

@@ -2598,9 +2598,9 @@ export function getStreamingReply(data, state, { chatCompletionSource = null, ov
2598	}	2598	}
2599	return data?.delta?.text \|\| '';	2599	return data?.delta?.text \|\| '';
2600	} else if ([chat_completion_sources.MAKERSUITE, chat_completion_sources.VERTEXAI].includes(chat_completion_source)) {	2600	} else if ([chat_completion_sources.MAKERSUITE, chat_completion_sources.VERTEXAI].includes(chat_completion_source)) {
2601	const inlineData = data?.candidates?.[0]?.content?.parts?.find(x => x.inlineData)?.inlineData;	2601	const inlineData = data?.candidates?.[0]?.content?.parts?.filter(x => x.inlineData)?.map(x => x.inlineData) \|\| [];
2602	if (inlineData) {	2602	if (Array.isArray(inlineData) && inlineData.length > 0) {
2603	state.image = `data:${inlineData.mimeType};base64,${inlineData.data}`;	2603	state.images.push(...inlineData.map(x => `data:${x.mimeType};base64,${x.data}`).filter(isDataURL));
2604	}	2604	}
2605	if (show_thoughts) {	2605	if (show_thoughts) {
2606	state.reasoning += (data?.candidates?.[0]?.content?.parts?.filter(x => x.thought)?.map(x => x.text)?.[0] \|\| '');	2606	state.reasoning += (data?.candidates?.[0]?.content?.parts?.filter(x => x.thought)?.map(x => x.text)?.[0] \|\| '');

@@ -2619,9 +2619,9 @@ export function getStreamingReply(data, state, { chatCompletionSource = null, ov
2619	}	2619	}
2620	return data.choices?.[0]?.delta?.content \|\| '';	2620	return data.choices?.[0]?.delta?.content \|\| '';
2621	} else if (chat_completion_source === chat_completion_sources.OPENROUTER) {	2621	} else if (chat_completion_source === chat_completion_sources.OPENROUTER) {
2622	const imageUrl = data?.choices?.[0]?.delta?.images?.find(x => x.type === 'image_url')?.image_url?.url;	2622	const imageUrls = data?.choices?.[0]?.delta?.images?.filter(x => x.type === 'image_url')?.map(x => x?.image_url?.url) \|\| [];
2623	if (imageUrl) {	2623	if (Array.isArray(imageUrls) && imageUrls.length > 0) {
2624	state.image = imageUrl;	2624	state.images.push(...imageUrls.filter(isDataURL));
2625	}	2625	}
2626	if (show_thoughts) {	2626	if (show_thoughts) {
2627	state.reasoning += (data.choices?.filter(x => x?.delta?.reasoning)?.[0]?.delta?.reasoning \|\| '');	2627	state.reasoning += (data.choices?.filter(x => x?.delta?.reasoning)?.[0]?.delta?.reasoning \|\| '');

public/scripts/utils.js+1 -1 Viewed

  */
 export function isDataURL(str) {
     const regex = /^data:([a-z]+\/[a-z0-9-+.]+(;[a-z-]+=[a-z0-9-]+)*;?)?(base64)?,([a-z0-9!$&',()*+;=\-_%.~:@/?#]+)?$/i;
-    return regex.test(str);
+    return typeof str === 'string' && regex.test(str);
+}
 /**

public/style.css+1 -0 Viewed

     align-items: center;
     flex-wrap: wrap;
     gap: 0.5em;
+    padding-right: var(--mes-right-spacing);
+}
 .mes_media_wrapper:not(:empty)~.mes_file_wrapper:not(:empty) {