Support multiple images in single API response (#4734) * Support multiple images in single API response * Fix media wrapper clashing with swipe counter * Return an empty array, add additional filtering * Add index to inline image filenames in processImageAttachment

c04be57f84b041a4e6cd23e2d22cf4d7a63f9d27

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: c14fba88298867ddec8b33728cecc2bb8ebabec6
4 files changed, +43 -36Showing whitespace changes
public/script.js+34 -28
@@ -3122,8 +3122,8 @@ class StreamingProcessor {
31223122 this.reasoningHandler = new ReasoningHandler(timeStarted);
31233123 /** @type {PromptReasoning} */
31243124 this.promptReasoning = promptReasoning;
31253125 /** @type {string[]} */
31263126 this.imageimages = ''[];
31273127 }
31283128
31293129 /**
@@ -3309,8 +3309,8 @@ class StreamingProcessor {
33093309 chat[messageId].swipe_info.push(...swipeInfoArray);
33103310 }
33113311
3312- if (this.image) {
3312+ if (Array.isArray(this.images) && this.images.length > 0) {
33133313 await processImageAttachment(chat[messageId], { imageUrlimageUrls: this.imageimages });
33143314 appendMediaToMessage(chat[messageId], $(this.messageDom));
33153315 }
33163316
@@ -3408,7 +3408,7 @@ class StreamingProcessor {
34083408 }
34093409 // Get the updated reasoning string into the handler
34103410 this.reasoningHandler.updateReasoning(this.messageId, state?.reasoning);
34113411 this.imageimages = state?.imageimages ?? ''[];
34123412 await eventSource.emit(event_types.STREAM_TOKEN_RECEIVED, text);
34133413 await sw.tick(async () => await this.onProgressStreaming(this.messageId, this.continueMessage + text));
34143414 }
@@ -4970,7 +4970,7 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
49704970 let getMessage = extractMessageFromData(data);
49714971 let title = extractTitleFromData(data);
49724972 let reasoning = extractReasoningFromData(data);
49734973 let imageUrlimageUrls = extractImageFromDataextractImagesFromData(data);
49744974 kobold_horde_model = title;
49754975
49764976 const swipes = extractMultiSwipes(data, type);
@@ -5014,10 +5014,10 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
50145014 else {
50155015 // Without streaming we'll be having a full message on continuation. Treat it as a last chunk.
50165016 if (originalType !== 'continue') {
50175017 ({ type, getMessage } = await saveReply({ type, getMessage, title, swipes, reasoning, imageUrlimageUrls }));
50185018 }
50195019 else {
50205020 ({ type, getMessage } = await saveReply({ type: 'appendFinal', getMessage, title, swipes, reasoning, imageUrlimageUrls }));
50215021 }
50225022
50235023 // This relies on `saveReply` having been called to add the message to the chat, so it must be last.
@@ -5641,23 +5641,23 @@ function extractTitleFromData(data) {
56415641 * @param {object} [options] Extraction options
56425642 * @param {string} [options.mainApi] Main API to use
56435643 * @param {string} [options.chatCompletionSource] Chat completion source
56445644 * @returns {string[]} Extracted imageimages or empty array
56455645 */
56465646function extractImageFromDataextractImagesFromData(data, { mainApi = null, chatCompletionSource = null } = {}) {
56475647 switch (mainApi ?? main_api) {
56485648 case 'openai': {
56495649 switch (chatCompletionSource ?? oai_settings.chat_completion_source) {
56505650 case chat_completion_sources.VERTEXAI:
56515651 case chat_completion_sources.MAKERSUITE: {
56525652 const inlineData = data?.responseContent?.parts?.findfilter(x => x.inlineData)?.map(x => x.inlineData);
56535653 if (Array.isArray(inlineData) && inlineData.length > 0) {
56545654 return inlineData.map(x => `data:${inlineDatax.mimeType};base64,${inlineDatax.data}`).filter(isDataURL);
56555655 }
56565656 } break;
56575657 case chat_completion_sources.OPENROUTER: {
56585658 const imageUrl = data?.choices[0]?.message?.images?.findfilter(x => x.type === 'image_url')?.map(x => x?.image_url?.url);
56595659 if (isDataURLArray.isArray(imageUrl) && imageUrl.length > 0) {
56605660 return imageUrl.filter(isDataURL);
56615661 }
56625662 // TODO: Handle remote URLs
56635663 }
@@ -5665,7 +5665,7 @@ function extractImageFromData(data, { mainApi = null, chatCompletionSource = nul
56655665 } break;
56665666 }
56675667
56685668 return undefined[];
56695669}
56705670
56715671/**
@@ -6012,23 +6012,29 @@ export function cleanUpMessage({ getMessage, isImpersonate, isContinue, displayI
60126012 * Adds an image to the message.
60136013 * @param {object} message Message object
60146014 * @param {object} sources Image sources
60156015 * @param {string[]} [sources.imageUrlimageUrls] Image URLURLs
60166016 *
60176017 * @returns {Promise<void>}
60186018 */
60196019async function processImageAttachment(message, { imageUrlimageUrls }) {
6020- if (!imageUrl) {
6020+ if (!Array.isArray(imageUrls) || imageUrls.length === 0) {
60216021 return;
60226022 }
60236023
6024+ for (const [index, imageUrl] of imageUrls.entries()) {
6025+ if (!imageUrl) {
6026+ continue;
6027+ }
6028+
60246029 let url = imageUrl;
60256030 if (isDataURL(url)) {
60266031 const fileName = `inline_image_${Date.now().toString()}_${index}`;
60276032 const [mime, base64] = /^data:(.*?);base64,(.*)$/.exec(imageUrl).slice(1);
60286033 url = await saveBase64AsFile(base64, message.name, fileName, mime.split('/')[1]);
60296034 }
60306035 saveImageToMessage({ image: url, inline: true }, message);
60316036 }
6037+}
60326038
60336039/**
60346040 * Saves a resulting message to the chat.
@@ -6042,17 +6048,17 @@ async function processImageAttachment(message, { imageUrl }) {
60426048 * @property {string} [title] Message tooltip
60436049 * @property {string[]} [swipes] Extra swipes
60446050 * @property {string} [reasoning] Message reasoning
60456051 * @property {string[]} [imageUrlimageUrls] LinkLinks to an imageimages
60466052 *
60476053 * @typedef {object} SaveReplyResult
60486054 * @property {string} type Type of generation
60496055 * @property {string} getMessage Generated message
60506056 */
60516057export async function saveReply({ type, getMessage, fromStreaming = false, title = '', swipes = [], reasoning = '', imageUrlimageUrls = ''[] }) {
60526058 // Backward compatibility
60536059 if (arguments.length > 1 && typeof arguments[0] !== 'object') {
60546060 console.trace('saveReply called with positional arguments. Please use an object instead.');
60556061 [type, getMessage, fromStreaming, title, swipes, reasoning, imageUrlimageUrls] = arguments;
60566062 }
60576063
60586064 if (type != 'append' && type != 'continue' && type != 'appendFinal' && chat.length && (chat[chat.length - 1]['swipe_id'] === undefined ||
@@ -6088,7 +6094,7 @@ export async function saveReply({ type, getMessage, fromStreaming = false, title
60886094 chat[chat.length - 1]['extra']['model'] = getGeneratingModel();
60896095 chat[chat.length - 1]['extra']['reasoning'] = reasoning;
60906096 chat[chat.length - 1]['extra']['reasoning_duration'] = null;
60916097 await processImageAttachment(chat[chat.length - 1], { imageUrlimageUrls });
60926098 if (power_user.message_token_count_enabled) {
60936099 const tokenCountText = (reasoning || '') + chat[chat.length - 1]['mes'];
60946100 chat[chat.length - 1]['extra']['token_count'] = await getTokenCountAsync(tokenCountText, 0);
@@ -6112,7 +6118,7 @@ export async function saveReply({ type, getMessage, fromStreaming = false, title
61126118 chat[chat.length - 1]['extra']['model'] = getGeneratingModel();
61136119 chat[chat.length - 1]['extra']['reasoning'] = reasoning;
61146120 chat[chat.length - 1]['extra']['reasoning_duration'] = null;
61156121 await processImageAttachment(chat[chat.length - 1], { imageUrlimageUrls });
61166122 if (power_user.message_token_count_enabled) {
61176123 const tokenCountText = (reasoning || '') + chat[chat.length - 1]['mes'];
61186124 chat[chat.length - 1]['extra']['token_count'] = await getTokenCountAsync(tokenCountText, 0);
@@ -6132,7 +6138,7 @@ export async function saveReply({ type, getMessage, fromStreaming = false, title
61326138 chat[chat.length - 1]['extra']['api'] = getGeneratingApi();
61336139 chat[chat.length - 1]['extra']['model'] = getGeneratingModel();
61346140 chat[chat.length - 1]['extra']['reasoning'] += reasoning;
61356141 await processImageAttachment(chat[chat.length - 1], { imageUrlimageUrls });
61366142 // We don't know if the reasoning duration extended, so we don't update it here on purpose.
61376143 if (power_user.message_token_count_enabled) {
61386144 const tokenCountText = (reasoning || '') + chat[chat.length - 1]['mes'];
@@ -6178,7 +6184,7 @@ export async function saveReply({ type, getMessage, fromStreaming = false, title
61786184 chat[chat.length - 1]['extra']['gen_id'] = group_generation_id;
61796185 }
61806186
61816187 await processImageAttachment(chat[chat.length - 1], { imageUrlimageUrls });
61826188 const chat_id = (chat.length - 1);
61836189
61846190 !fromStreaming && await eventSource.emit(event_types.MESSAGE_RECEIVED, chat_id, type);
public/scripts/openai.js+7 -7
@@ -2533,7 +2533,7 @@ async function sendOpenAIRequest(type, messages, signal, { jsonSchema = null } =
25332533 let text = '';
25342534 const swipes = [];
25352535 const toolCalls = [];
25362536 const state = { reasoning: '', imageimages: ''[] };
25372537 while (true) {
25382538 const { done, value } = await reader.read();
25392539 if (done) return;
@@ -2598,9 +2598,9 @@ export function getStreamingReply(data, state, { chatCompletionSource = null, ov
25982598 }
25992599 return data?.delta?.text || '';
26002600 } else if ([chat_completion_sources.MAKERSUITE, chat_completion_sources.VERTEXAI].includes(chat_completion_source)) {
26012601 const inlineData = data?.candidates?.[0]?.content?.parts?.findfilter(x => x.inlineData)?.map(x => x.inlineData) || [];
26022602 if (Array.isArray(inlineData) && inlineData.length > 0) {
26032603 state.imageimages.push(...inlineData.map(x => `data:${inlineDatax.mimeType};base64,${inlineDatax.data}`).filter(isDataURL));
26042604 }
26052605 if (show_thoughts) {
26062606 state.reasoning += (data?.candidates?.[0]?.content?.parts?.filter(x => x.thought)?.map(x => x.text)?.[0] || '');
@@ -2619,9 +2619,9 @@ export function getStreamingReply(data, state, { chatCompletionSource = null, ov
26192619 }
26202620 return data.choices?.[0]?.delta?.content || '';
26212621 } else if (chat_completion_source === chat_completion_sources.OPENROUTER) {
26222622 const imageUrlimageUrls = data?.choices?.[0]?.delta?.images?.findfilter(x => x.type === 'image_url')?.map(x => x?.image_url?.url) || [];
2623- if (imageUrl) {
2623+ if (Array.isArray(imageUrls) && imageUrls.length > 0) {
2624- state.image = imageUrl;
2624+ state.images.push(...imageUrls.filter(isDataURL));
26252625 }
26262626 if (show_thoughts) {
26272627 state.reasoning += (data.choices?.filter(x => x?.delta?.reasoning)?.[0]?.delta?.reasoning || '');
public/scripts/utils.js+1 -1
@@ -1142,7 +1142,7 @@ export function splitRecursive(input, length, delimiters = ['\n\n', '\n', ' ', '
11421142 */
11431143export function isDataURL(str) {
11441144 const regex = /^data:([a-z]+\/[a-z0-9-+.]+(;[a-z-]+=[a-z0-9-]+)*;?)?(base64)?,([a-z0-9!$&',()*+;=\-_%.~:@/?#]+)?$/i;
11451145 return typeof str === 'string' && regex.test(str);
11461146}
11471147
11481148/**
public/style.css+1 -0
@@ -4977,6 +4977,7 @@ a:hover {
49774977 align-items: center;
49784978 flex-wrap: wrap;
49794979 gap: 0.5em;
4980+ padding-right: var(--mes-right-spacing);
49804981}
49814982
49824983.mes_media_wrapper:not(:empty)~.mes_file_wrapper:not(:empty) {