Audio inlining for OpenAI and Custom-compatible (#4964) * Audio inlining for OpenAI and Custom-compatible * Add context sizes * chatgpt-image-latest * Add quality control for gpt-image

7320aa948de6bf76ba0eb0e7cd7abbf2e107ceea

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: 7331dba056583f4fd1118686ed2fe6fbea5785e6
6 files changed, +49 -11Showing whitespace changes
public/index.html+1 -1
@@ -2024,7 +2024,7 @@
20242024 <i class="icon-supported fa-solid fa-film" title="Supported by the current model" data-i18n="[title]Supported by the current model"></i>
20252025 <i class="icon-unsupported fa-solid fa-film" title="Unsupported by the current model" data-i18n="[title]Unsupported by the current model"></i>
20262026 </div>
20272027 <div id="openai_audio_inlining_supported" data-source="makersuite,vertexai,openrouter,openai,custom">
20282028 <i class="icon-supported fa-solid fa-music" title="Supported by the current model" data-i18n="[title]Supported by the current model"></i>
20292029 <i class="icon-unsupported fa-solid fa-music" title="Unsupported by the current model" data-i18n="[title]Unsupported by the current model"></i>
20302030 </div>
public/scripts/extensions/stable-diffusion/index.js+10 -3
@@ -320,6 +320,7 @@ const defaultSettings = {
320320 // OpenAI settings
321321 openai_style: 'vivid',
322322 openai_quality: 'standard',
323+ openai_quality_gpt: 'auto',
323324 openai_duration: '8',
324325
325326 style: 'Default',
@@ -528,6 +529,7 @@ async function loadSettings() {
528529 $('#sd_interactive_mode').prop('checked', extension_settings.sd.interactive_mode);
529530 $('#sd_openai_style').val(extension_settings.sd.openai_style);
530531 $('#sd_openai_quality').val(extension_settings.sd.openai_quality);
532+ $('#sd_openai_quality_gpt').val(extension_settings.sd.openai_quality_gpt);
531533 $('#sd_openai_duration').val(extension_settings.sd.openai_duration);
532534 $('#sd_comfy_type').val(extension_settings.sd.comfy_type);
533535 $('#sd_comfy_url').val(extension_settings.sd.comfy_url);
@@ -1850,7 +1852,7 @@ function switchModelSpecificControls(modelId) {
18501852
18511853 modelControls.each(function () {
18521854 const models = String($(this).attr('data-sd-model') || '').split(',').map(m => m.trim());
18531855 $(this).toggle(models.includessome(m => modelId.includes(m)));
18541856 });
18551857}
18561858
@@ -2169,6 +2171,7 @@ async function loadOpenAiModels() {
21692171 { value: 'gpt-image-1.5', text: 'gpt-image-1.5' },
21702172 { value: 'gpt-image-1-mini', text: 'gpt-image-1-mini' },
21712173 { value: 'gpt-image-1', text: 'gpt-image-1' },
2174+ { value: 'chatgpt-image-latest', text: 'chatgpt-image-latest' },
21722175 { value: 'dall-e-3', text: 'dall-e-3' },
21732176 { value: 'dall-e-2', text: 'dall-e-2' },
21742177 { value: 'sora-2', text: 'sora-2' },
@@ -3697,7 +3700,7 @@ async function generateOpenAiImage(prompt, signal) {
36973700
36983701 const isDalle2 = /dall-e-2/.test(extension_settings.sd.model);
36993702 const isDalle3 = /dall-e-3/.test(extension_settings.sd.model);
37003703 const isGptImg = /gpt-image-(1|latest)/.test(extension_settings.sd.model);
37013704 const isSora2 = /sora-2/.test(extension_settings.sd.model);
37023705
37033706 if (isDalle2 && prompt.length > dalle2PromptLimit) {
@@ -3770,7 +3773,7 @@ async function generateOpenAiImage(prompt, signal) {
37703773 model: extension_settings.sd.model,
37713774 size: `${width}x${height}`,
37723775 n: 1,
37733776 quality: isDalle3 ? extension_settings.sd.openai_quality : (isGptImg ? extension_settings.sd.openai_quality_gpt : undefined),
37743777 style: isDalle3 ? extension_settings.sd.openai_style : undefined,
37753778 response_format: isDalle2 || isDalle3 ? 'b64_json' : undefined,
37763779 moderation: isGptImg ? 'low' : undefined,
@@ -5309,6 +5312,10 @@ jQuery(async () => {
53095312 extension_settings.sd.electronhub_quality = String($(this).val());
53105313 saveSettingsDebounced();
53115314 });
5315+ $('#sd_openai_quality_gpt').on('input', function () {
5316+ extension_settings.sd.openai_quality_gpt = String($(this).val());
5317+ saveSettingsDebounced();
5318+ });
53125319
53135320 if (!CSS.supports('field-sizing', 'content')) {
53145321 $('.sd_settings .inline-drawer-toggle').on('click', function () {
public/scripts/extensions/stable-diffusion/settings.html+10 -1
@@ -178,7 +178,16 @@
178178 <option value="natural">Natural</option>
179179 </select>
180180 </div>
181181 <div data-sd-model="dall-e-3,cogview-4gpt-250304image" class="flex1">
182+ <label for="sd_openai_quality_gpt" data-i18n="Image Quality">Image Quality</label>
183+ <select id="sd_openai_quality_gpt">
184+ <option value="auto" data-i18n="Auto">Auto</option>
185+ <option value="low" data-i18n="Low">Low</option>
186+ <option value="medium" data-i18n="Medium">Medium</option>
187+ <option value="high" data-i18n="High">High</option>
188+ </select>
189+ </div>
190+ <div data-sd-model="dall-e-3,cogview-4" class="flex1">
182191 <label for="sd_openai_quality" data-i18n="Image Quality">Image Quality</label>
183192 <select id="sd_openai_quality">
184193 <option value="standard" data-i18n="Standard">Standard</option>
public/scripts/openai.js+16 -1
@@ -4689,6 +4689,9 @@ function getMaxContextOpenAI(value) {
46894689 else if (value.includes('gpt-4.1')) {
46904690 return max_1mil;
46914691 }
4692+ else if (value.includes('gpt-audio')) {
4693+ return max_128k;
4694+ }
46924695 else if (value.startsWith('o1')) {
46934696 return max_128k;
46944697 }
@@ -4707,6 +4710,9 @@ function getMaxContextOpenAI(value) {
47074710 else if (['gpt-4-32k', 'gpt-4-32k-0314', 'gpt-4-32k-0613'].includes(value)) {
47084711 return max_32k;
47094712 }
4713+ else if (value.includes('gpt-realtime')) {
4714+ return max_32k;
4715+ }
47104716 else if (['gpt-3.5-turbo-16k', 'gpt-3.5-turbo-16k-0613'].includes(value)) {
47114717 return max_16k;
47124718 }
@@ -5946,21 +5952,30 @@ export function isAudioInliningSupported() {
59465952 return false;
59475953 }
59485954
5949- // Only Gemini models support audio for now
59505955 const audioSupportedModels = [
59515956 'gemini-2.0',
59525957 'gemini-2.5',
59535958 'gemini-3',
59545959 'gemini-exp-1206',
5960+ 'gpt-4o-audio',
5961+ 'gpt-4o-realtime',
5962+ 'gpt-4o-mini-audio',
5963+ 'gpt-4o-mini-realtime',
5964+ 'gpt-audio',
5965+ 'gpt-realtime',
59555966 ];
59565967
59575968 switch (oai_settings.chat_completion_source) {
5969+ case chat_completion_sources.OPENAI:
5970+ return audioSupportedModels.some(model => oai_settings.openai_model.includes(model));
59585971 case chat_completion_sources.MAKERSUITE:
59595972 return audioSupportedModels.some(model => oai_settings.google_model.includes(model));
59605973 case chat_completion_sources.VERTEXAI:
59615974 return audioSupportedModels.some(model => oai_settings.vertexai_model.includes(model));
59625975 case chat_completion_sources.OPENROUTER:
59635976 return (Array.isArray(model_list) && model_list.find(m => m.id === oai_settings.openrouter_model)?.architecture?.input_modalities?.includes('audio'));
5977+ case chat_completion_sources.CUSTOM:
5978+ return true;
59645979 default:
59655980 return false;
59665981 }
src/endpoints/backends/chat-completions.js+4 -1
@@ -2063,6 +2063,8 @@ router.post('/generate', async function (request, response) {
20632063 if (getConfigValue('openai.randomizeUserId', false, 'boolean')) {
20642064 bodyParams['user'] = uuidv4();
20652065 }
2066+
2067+ embedOpenRouterMedia(request.body.messages, { audio: true, video: false });
20662068 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENROUTER) {
20672069 apiUrl = 'https://openrouter.ai/api/v1';
20682070 apiKey = readSecret(request.user.directories, SECRET_KEYS.OPENROUTER);
@@ -2122,7 +2124,7 @@ router.post('/generate', async function (request, response) {
21222124 const enableGeminiSystemPromptCache = getConfigValue('gemini.enableSystemPromptCache', false, 'boolean');
21232125
21242126 if (Array.isArray(request.body.messages)) {
21252127 embedOpenRouterMedia(request.body.messages, { audio: true, video: true });
21262128 addOpenRouterSignatures(request.body.messages, request.body.model);
21272129
21282130 if (isClaude) {
@@ -2160,6 +2162,7 @@ router.post('/generate', async function (request, response) {
21602162
21612163 mergeObjectWithYaml(bodyParams, request.body.custom_include_body);
21622164 mergeObjectWithYaml(headers, request.body.custom_include_headers);
2165+ embedOpenRouterMedia(request.body.messages, { audio: true, video: false });
21632166 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.PERPLEXITY) {
21642167 apiUrl = API_PERPLEXITY;
21652168 apiKey = readSecret(request.user.directories, SECRET_KEYS.PERPLEXITY);
src/prompt-converters.js+8 -4
@@ -1296,10 +1296,14 @@ export function calculateGoogleBudgetTokens(maxTokens, reasoningEffort, model) {
12961296}
12971297
12981298/**
12991299 * Embed media content in OpenRouter messages (OpenAI-compatible).
13001300 * @param {object[]} messages Array of messages
1301+ * @param {object} options Options for embedding
1302+ * @param {boolean} [options.audio] Enable audio embedding (default: true)
1303+ * @param {boolean} [options.video] Enable video embedding (default: true)
1304+ * @returns {void}
13011305 */
1302-export function embedOpenRouterMedia(messages) {
1306+export function embedOpenRouterMedia(messages, { audio = true, video = true } = { audio: true, video: true }) {
13031307 if (!Array.isArray(messages)) {
13041308 return;
13051309 }
@@ -1310,11 +1314,11 @@ export function embedOpenRouterMedia(messages) {
13101314 }
13111315
13121316 for (const contentPart of message.content) {
13131317 if (video && contentPart?.type === 'video_url' && contentPart.video_url?.url?.startsWith('data:')) {
13141318 contentPart.type = 'input_video';
13151319 }
13161320
13171321 if (audio && contentPart?.type === 'audio_url' && contentPart.audio_url?.url?.startsWith('data:')) {
13181322 const formatMap = {
13191323 'audio/mpeg': 'mp3',
13201324 'audio/wav': 'wav',