Gemini inline images (#3681) * Gemini images for non-streaming * Parse images on stream * Add toggle for image request * Add extraction params to extractImageFromData * Add explicit break and return * Add more JSdoc to processImageAttachment * Add file name prefix * Add object argument for saveReply * Add defaults to saveReply params * Use type for saveReply result * Change type check in saveReply backward compat

0017358f8b6de330b88e0b23e4731bcb3249285a

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: 0d2bf0081061deaa21f161a590b8043641a8a1b5
5 files changed, +155 -16Showing whitespace changes
public/index.html+17 -0
@@ -1998,6 +1998,23 @@
19981998 </div>
19991999 </div>
20002000 <div class="range-block" data-source="makersuite">
2001+ <label for="openai_request_images" class="checkbox_label widthFreeExpand">
2002+ <input id="openai_request_images" type="checkbox" />
2003+ <span>
2004+ <span data-i18n="Request inline images">Request inline images</span>
2005+ <i class="opacity50p fa-solid fa-circle-info" title="Gemini 2.0 Flash Experimental"></i>
2006+ </span>
2007+ </label>
2008+ <div class="toggle-description justifyLeft marginBot5">
2009+ <span data-i18n="Allows the model to return image attachments.">
2010+ Allows the model to return image attachments.
2011+ </span>
2012+ <em data-source="makersuite" data-i18n="Request inline images_desc_2">
2013+ Incompatible with the following features: function calling, web search, system prompt.
2014+ </em>
2015+ </div>
2016+ </div>
2017+ <div class="range-block" data-source="makersuite">
20012018 <label for="use_makersuite_sysprompt" class="checkbox_label widthFreeExpand">
20022019 <input id="use_makersuite_sysprompt" type="checkbox" />
20032020 <span>
public/script.js+106 -8
@@ -171,6 +171,7 @@ import {
171171 isElementInViewport,
172172 copyText,
173173 escapeHtml,
174+ saveBase64AsFile,
174175} from './scripts/utils.js';
175176import { debounce_timeout } from './scripts/constants.js';
176177
@@ -3203,6 +3204,8 @@ class StreamingProcessor {
32033204 this.reasoningHandler = new ReasoningHandler(timeStarted);
32043205 /** @type {PromptReasoning} */
32053206 this.promptReasoning = promptReasoning;
3207+ /** @type {string} */
3208+ this.image = '';
32063209 }
32073210
32083211 /**
@@ -3250,7 +3253,7 @@ class StreamingProcessor {
32503253 this.sendTextarea.value = '';
32513254 this.sendTextarea.dispatchEvent(new Event('input', { bubbles: true }));
32523255 } else {
32533256 await saveReply({ type: this.type, text,getMessage: truetext, '',fromStreaming: [],true ''});
32543257 messageId = chat.length - 1;
32553258 await this.#checkDomElements(messageId, continueOnReasoning);
32563259 this.markUIGenStarted();
@@ -3372,6 +3375,11 @@ class StreamingProcessor {
33723375 chat[messageId].swipe_info.push(...swipeInfoArray);
33733376 }
33743377
3378+ if (this.image) {
3379+ await processImageAttachment(chat[messageId], { imageUrl: this.image, parsedImage: null });
3380+ appendMediaToMessage(chat[messageId], $(this.messageDom));
3381+ }
3382+
33753383 if (this.type !== 'impersonate') {
33763384 await eventSource.emit(event_types.MESSAGE_RECEIVED, this.messageId, this.type);
33773385 await eventSource.emit(event_types.CHARACTER_MESSAGE_RENDERED, this.messageId, this.type);
@@ -3468,6 +3476,7 @@ class StreamingProcessor {
34683476 }
34693477 // Get the updated reasoning string into the handler
34703478 this.reasoningHandler.updateReasoning(this.messageId, state?.reasoning);
3479+ this.image = state?.image ?? '';
34713480 await eventSource.emit(event_types.STREAM_TOKEN_RECEIVED, text);
34723481 await sw.tick(async () => await this.onProgressStreaming(this.messageId, this.continueMessage + text));
34733482 }
@@ -4866,6 +4875,7 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
48664875 let getMessage = extractMessageFromData(data);
48674876 let title = extractTitleFromData(data);
48684877 let reasoning = extractReasoningFromData(data);
4878+ let imageUrl = extractImageFromData(data);
48694879 kobold_horde_model = title;
48704880
48714881 const swipes = extractMultiSwipes(data, type);
@@ -4898,10 +4908,10 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
48984908 else {
48994909 // Without streaming we'll be having a full message on continuation. Treat it as a last chunk.
49004910 if (originalType !== 'continue') {
49014911 ({ type, getMessage } = await saveReply({ type, getMessage, false, title, swipes, reasoning, imageUrl }));
49024912 }
49034913 else {
49044914 ({ type, getMessage } = await saveReply({ type: 'appendFinal', getMessage, false, title, swipes, reasoning, imageUrl }));
49054915 }
49064916
49074917 // This relies on `saveReply` having been called to add the message to the chat, so it must be last.
@@ -5726,6 +5736,32 @@ function extractTitleFromData(data) {
57265736}
57275737
57285738/**
5739+ * Extracts the image from the response data.
5740+ * @param {object} data Response data
5741+ * @param {object} [options] Extraction options
5742+ * @param {string} [options.mainApi] Main API to use
5743+ * @param {string} [options.chatCompletionSource] Chat completion source
5744+ * @returns {string} Extracted image
5745+ */
5746+function extractImageFromData(data, { mainApi = null, chatCompletionSource = null } = {}) {
5747+ switch (mainApi ?? main_api) {
5748+ case 'openai': {
5749+ switch (chatCompletionSource ?? oai_settings.chat_completion_source) {
5750+ case chat_completion_sources.MAKERSUITE: {
5751+ const inlineData = data?.responseContent?.parts?.find(x => x.inlineData)?.inlineData;
5752+ if (inlineData) {
5753+ return `data:${inlineData.mimeType};base64,${inlineData.data}`;
5754+ }
5755+ } break;
5756+
5757+ }
5758+ } break;
5759+ }
5760+
5761+ return undefined;
5762+}
5763+
5764+/**
57295765 * parseAndSaveLogprobs receives the full data response for a non-streaming
57305766 * generation, parses logprobs for all tokens in the message, and saves them
57315767 * to the currently active message.
@@ -5974,7 +6010,59 @@ export function cleanUpMessage(getMessage, isImpersonate, isContinue, displayInc
59746010 return getMessage;
59756011}
59766012
5977-export async function saveReply(type, getMessage, fromStreaming, title, swipes, reasoning) {
6013+/**
6014+ * Adds an image to the message.
6015+ * @param {object} message Message object
6016+ * @param {object} sources Image sources
6017+ * @param {ParsedImage} [sources.parsedImage] Parsed image
6018+ * @param {string} [sources.imageUrl] Image URL
6019+ *
6020+ * @returns {Promise<void>}
6021+ */
6022+async function processImageAttachment(message, { parsedImage, imageUrl }) {
6023+ if (parsedImage?.image) {
6024+ saveImageToMessage(parsedImage, message);
6025+ return;
6026+ }
6027+
6028+ if (!imageUrl) {
6029+ return;
6030+ }
6031+
6032+ let url = imageUrl;
6033+ if (isDataURL(url)) {
6034+ const fileName = `inline_image_${Date.now().toString()}`;
6035+ const [mime, base64] = /^data:(.*?);base64,(.*)$/.exec(imageUrl).slice(1);
6036+ url = await saveBase64AsFile(base64, message.name, fileName, mime.split('/')[1]);
6037+ }
6038+ saveImageToMessage({ image: url, inline: true }, message);
6039+}
6040+
6041+/**
6042+ * Saves a resulting message to the chat.
6043+ * @param {SaveReplyParams} params
6044+ * @returns {Promise<SaveReplyResult>} Promise when the message is saved
6045+ *
6046+ * @typedef {object} SaveReplyParams
6047+ * @property {string} type Type of generation
6048+ * @property {string} getMessage Generated message
6049+ * @property {boolean} [fromStreaming] If the message is from streaming
6050+ * @property {string} [title] Message tooltip
6051+ * @property {string[]} [swipes] Extra swipes
6052+ * @property {string} [reasoning] Message reasoning
6053+ * @property {string} [imageUrl] Link to an image
6054+ *
6055+ * @typedef {object} SaveReplyResult
6056+ * @property {string} type Type of generation
6057+ * @property {string} getMessage Generated message
6058+ */
6059+export async function saveReply({ type, getMessage, fromStreaming = false, title = '', swipes = [], reasoning = '', imageUrl = '' }) {
6060+ // Backward compatibility
6061+ if (arguments.length > 1 && typeof arguments[0] !== 'object') {
6062+ console.trace('saveReply called with positional arguments. Please use an object instead.');
6063+ [type, getMessage, fromStreaming, title, swipes, reasoning, imageUrl] = arguments;
6064+ }
6065+
59786066 if (type != 'append' && type != 'continue' && type != 'appendFinal' && chat.length && (chat[chat.length - 1]['swipe_id'] === undefined ||
59796067 chat[chat.length - 1]['is_user'])) {
59806068 type = 'normal';
@@ -5995,8 +6083,8 @@ export async function saveReply(type, getMessage, fromStreaming, title, swipes,
59956083
59966084 let oldMessage = '';
59976085 const generationFinished = new Date();
59986086 const imgparsedImage = extractImageFromMessage(getMessage);
59996087 getMessage = imgparsedImage.getMessage;
60006088 if (type === 'swipe') {
60016089 oldMessage = chat[chat.length - 1]['mes'];
60026090 chat[chat.length - 1]['swipes'].length++;
@@ -6010,6 +6098,7 @@ export async function saveReply(type, getMessage, fromStreaming, title, swipes,
60106098 chat[chat.length - 1]['extra']['model'] = getGeneratingModel();
60116099 chat[chat.length - 1]['extra']['reasoning'] = reasoning;
60126100 chat[chat.length - 1]['extra']['reasoning_duration'] = null;
6101+ await processImageAttachment(chat[chat.length - 1], { parsedImage, imageUrl });
60136102 if (power_user.message_token_count_enabled) {
60146103 const tokenCountText = (reasoning || '') + chat[chat.length - 1]['mes'];
60156104 chat[chat.length - 1]['extra']['token_count'] = await getTokenCountAsync(tokenCountText, 0);
@@ -6033,6 +6122,7 @@ export async function saveReply(type, getMessage, fromStreaming, title, swipes,
60336122 chat[chat.length - 1]['extra']['model'] = getGeneratingModel();
60346123 chat[chat.length - 1]['extra']['reasoning'] = reasoning;
60356124 chat[chat.length - 1]['extra']['reasoning_duration'] = null;
6125+ await processImageAttachment(chat[chat.length - 1], { parsedImage, imageUrl });
60366126 if (power_user.message_token_count_enabled) {
60376127 const tokenCountText = (reasoning || '') + chat[chat.length - 1]['mes'];
60386128 chat[chat.length - 1]['extra']['token_count'] = await getTokenCountAsync(tokenCountText, 0);
@@ -6052,6 +6142,7 @@ export async function saveReply(type, getMessage, fromStreaming, title, swipes,
60526142 chat[chat.length - 1]['extra']['api'] = getGeneratingApi();
60536143 chat[chat.length - 1]['extra']['model'] = getGeneratingModel();
60546144 chat[chat.length - 1]['extra']['reasoning'] += reasoning;
6145+ await processImageAttachment(chat[chat.length - 1], { parsedImage, imageUrl });
60556146 // We don't know if the reasoning duration extended, so we don't update it here on purpose.
60566147 if (power_user.message_token_count_enabled) {
60576148 const tokenCountText = (reasoning || '') + chat[chat.length - 1]['mes'];
@@ -6097,7 +6188,7 @@ export async function saveReply(type, getMessage, fromStreaming, title, swipes,
60976188 chat[chat.length - 1]['extra']['gen_id'] = group_generation_id;
60986189 }
60996190
61006191 saveImageToMessage(img,await processImageAttachment(chat[chat.length - 1], { parsedImage, imageUrl: imageUrl });
61016192 const chat_id = (chat.length - 1);
61026193
61036194 !fromStreaming && await eventSource.emit(event_types.MESSAGE_RECEIVED, chat_id, type);
@@ -6203,6 +6294,12 @@ export function syncMesToSwipe(messageId = null) {
62036294 return true;
62046295}
62056296
6297+/**
6298+ * Saves the image to the message object.
6299+ * @param {ParsedImage} img Image object
6300+ * @param {object} mes Chat message object
6301+ * @typedef {{ image?: string, title?: string, inline?: boolean }} ParsedImage
6302+ */
62066303function saveImageToMessage(img, mes) {
62076304 if (mes && img.image) {
62086305 if (!mes.extra || typeof mes.extra !== 'object') {
@@ -6210,6 +6307,7 @@ function saveImageToMessage(img, mes) {
62106307 }
62116308 mes.extra.image = img.image;
62126309 mes.extra.title = img.title;
6310+ mes.extra.inline_image = img.inline;
62136311 }
62146312}
62156313
@@ -6252,7 +6350,7 @@ function extractImageFromMessage(getMessage) {
62526350 const image = results ? results[1] : '';
62536351 const title = results ? results[2] : '';
62546352 getMessage = getMessage.replace(regex, '');
62556353 return { getMessage, image, title, inline: true };
62566354}
62576355
62586356/**
public/scripts/openai.js+17 -1
@@ -305,6 +305,7 @@ export const settingsToUpdate = {
305305 seed: ['#seed_openai', 'seed', false],
306306 n: ['#n_openai', 'n', false],
307307 bypass_status_check: ['#openai_bypass_status_check', 'bypass_status_check', true],
308+ request_images: ['#openai_request_images', 'request_images', true],
308309};
309310
310311const default_settings = {
@@ -383,6 +384,7 @@ const default_settings = {
383384 show_thoughts: true,
384385 reasoning_effort: 'medium',
385386 enable_web_search: false,
387+ request_images: false,
386388 seed: -1,
387389 n: 1,
388390};
@@ -463,6 +465,7 @@ const oai_settings = {
463465 show_thoughts: true,
464466 reasoning_effort: 'medium',
465467 enable_web_search: false,
468+ request_images: false,
466469 seed: -1,
467470 n: 1,
468471};
@@ -2014,6 +2017,7 @@ async function sendOpenAIRequest(type, messages, signal) {
20142017 'include_reasoning': Boolean(oai_settings.show_thoughts),
20152018 'reasoning_effort': String(oai_settings.reasoning_effort),
20162019 'enable_web_search': Boolean(oai_settings.enable_web_search),
2020+ 'request_images': Boolean(oai_settings.request_images),
20172021 };
20182022
20192023 if (!canMultiSwipe && ToolManager.canPerformToolCalls(type)) {
@@ -2200,7 +2204,7 @@ async function sendOpenAIRequest(type, messages, signal) {
22002204 let text = '';
22012205 const swipes = [];
22022206 const toolCalls = [];
22032207 const state = { reasoning: '', image: '' };
22042208 while (true) {
22052209 const { done, value } = await reader.read();
22062210 if (done) return;
@@ -2258,6 +2262,10 @@ function getStreamingReply(data, state) {
22582262 }
22592263 return data?.delta?.text || '';
22602264 } else if (oai_settings.chat_completion_source === chat_completion_sources.MAKERSUITE) {
2265+ const inlineData = data?.candidates?.[0]?.content?.parts?.find(x => x.inlineData)?.inlineData;
2266+ if (inlineData) {
2267+ state.image = `data:${inlineData.mimeType};base64,${inlineData.data}`;
2268+ }
22612269 if (oai_settings.show_thoughts) {
22622270 state.reasoning += (data?.candidates?.[0]?.content?.parts?.filter(x => x.thought)?.map(x => x.text)?.[0] || '');
22632271 }
@@ -3242,6 +3250,7 @@ function loadOpenAISettings(data, settings) {
32423250 oai_settings.show_thoughts = settings.show_thoughts ?? default_settings.show_thoughts;
32433251 oai_settings.reasoning_effort = settings.reasoning_effort ?? default_settings.reasoning_effort;
32443252 oai_settings.enable_web_search = settings.enable_web_search ?? default_settings.enable_web_search;
3253+ oai_settings.request_images = settings.request_images ?? default_settings.request_images;
32453254 oai_settings.seed = settings.seed ?? default_settings.seed;
32463255 oai_settings.n = settings.n ?? default_settings.n;
32473256
@@ -3370,6 +3379,7 @@ function loadOpenAISettings(data, settings) {
33703379 $('#n_openai').val(oai_settings.n);
33713380 $('#openai_show_thoughts').prop('checked', oai_settings.show_thoughts);
33723381 $('#openai_enable_web_search').prop('checked', oai_settings.enable_web_search);
3382+ $('#openai_request_images').prop('checked', oai_settings.request_images);
33733383
33743384 $('#openai_reasoning_effort').val(oai_settings.reasoning_effort);
33753385 $(`#openai_reasoning_effort option[value="${oai_settings.reasoning_effort}"]`).prop('selected', true);
@@ -3641,6 +3651,7 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) {
36413651 show_thoughts: settings.show_thoughts,
36423652 reasoning_effort: settings.reasoning_effort,
36433653 enable_web_search: settings.enable_web_search,
3654+ request_images: settings.request_images,
36443655 seed: settings.seed,
36453656 n: settings.n,
36463657 };
@@ -5603,6 +5614,11 @@ export function initOpenAI() {
56035614 saveSettingsDebounced();
56045615 });
56055616
5617+ $('#openai_request_images').on('input', function () {
5618+ oai_settings.request_images = !!$(this).prop('checked');
5619+ saveSettingsDebounced();
5620+ });
5621+
56065622 if (!CSS.supports('field-sizing', 'content')) {
56075623 $(document).on('input', '#openai_settings .autoSetHeight', function () {
56085624 resetScrollHeight($(this));
public/scripts/sse-stream.js+2 -1
@@ -138,10 +138,11 @@ async function* parseStreamData(json) {
138138 for (let i = 0; i < json.candidates.length; i++) {
139139 const isNotPrimary = json.candidates?.[0]?.index > 0;
140140 const hasToolCalls = json?.candidates?.[0]?.content?.parts?.some(p => p?.functionCall);
141+ const hasInlineData = json?.candidates?.[0]?.content?.parts?.some(p => p?.inlineData);
141142 if (isNotPrimary || json.candidates.length === 0) {
142143 return null;
143144 }
144145 if (hasToolCalls || hasInlineData) {
145146 yield { data: json, chunk: '' };
146147 return;
147148 }
src/endpoints/backends/chat-completions.js+13 -6
@@ -338,6 +338,7 @@ async function sendMakerSuiteRequest(request, response) {
338338 const model = String(request.body.model);
339339 const stream = Boolean(request.body.stream);
340340 const enableWebSearch = Boolean(request.body.enable_web_search);
341+ const requestImages = Boolean(request.body.request_images);
341342 const isThinking = model.includes('thinking');
342343
343344 const generationConfig = {
@@ -356,7 +357,12 @@ async function sendMakerSuiteRequest(request, response) {
356357 delete generationConfig.stopSequences;
357358 }
358359
359- const should_use_system_prompt = (
360+ const useMultiModal = requestImages && (model.includes('gemini-2.0-flash-exp'));
361+ if (useMultiModal) {
362+ generationConfig.responseModalities = ['text', 'image'];
363+ }
364+
365+ const useSystemPrompt = !useMultiModal && (
360366 model.includes('gemini-2.0-pro') ||
361367 model.includes('gemini-2.0-flash') ||
362368 model.includes('gemini-2.0-flash-thinking-exp') ||
@@ -366,7 +372,7 @@ async function sendMakerSuiteRequest(request, response) {
366372 ) && request.body.use_makersuite_sysprompt;
367373
368374 const tools = [];
369375 const prompt = convertGooglePrompt(request.body.messages, model, should_use_system_promptuseSystemPrompt, getPromptNames(request));
370376 let safetySettings = GEMINI_SAFETY;
371377
372378 // These old models do not support setting the threshold to OFF at all.
@@ -379,14 +385,14 @@ async function sendMakerSuiteRequest(request, response) {
379385 }
380386 // Most of the other models allow for setting the threshold of filters, except for HARM_CATEGORY_CIVIC_INTEGRITY, to OFF.
381387
382388 if (enableWebSearch && !useMultiModal) {
383389 const searchTool = model.includes('1.5') || model.includes('1.0')
384390 ? ({ google_search_retrieval: {} })
385391 : ({ google_search: {} });
386392 tools.push(searchTool);
387393 }
388394
389395 if (Array.isArray(request.body.tools) && request.body.tools.length > 0 && !useMultiModal) {
390396 const functionDeclarations = [];
391397 for (const tool of request.body.tools) {
392398 if (tool.type === 'function') {
@@ -405,7 +411,7 @@ async function sendMakerSuiteRequest(request, response) {
405411 generationConfig: generationConfig,
406412 };
407413
408414 if (should_use_system_promptuseSystemPrompt) {
409415 body.systemInstruction = prompt.system_instruction;
410416 }
411417
@@ -469,10 +475,11 @@ async function sendMakerSuiteRequest(request, response) {
469475
470476 const responseContent = candidates[0].content ?? candidates[0].output;
471477 const functionCall = (candidates?.[0]?.content?.parts ?? []).some(part => part.functionCall);
478+ const inlineData = (candidates?.[0]?.content?.parts ?? []).some(part => part.inlineData);
472479 console.warn('Google AI Studio response:', responseContent);
473480
474481 const responseText = typeof responseContent === 'string' ? responseContent : responseContent?.parts?.filter(part => !part.thought)?.map(part => part.text)?.join('\n\n');
475482 if (!responseText && !functionCall && !inlineData) {
476483 let message = 'Google AI Studio Candidate text empty';
477484 console.warn(message, generateResponseJson);
478485 return response.send({ error: { message } });