Audio inlining for OpenAI and Custom-compatible (#4964) * Audio inlining for OpenAI and Custom-compatible * Add context sizes * chatgpt-image-latest * Add quality control for gpt-image

7320aa948de6bf76ba0eb0e7cd7abbf2e107ceea

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: 7331dba056583f4fd1118686ed2fe6fbea5785e6
6 files changed, +49 -11Ignore whitespace
public/index.html+1 -1
@@ -2024,7 +2024,7 @@
2024 <i class="icon-supported fa-solid fa-film" title="Supported by the current model" data-i18n="[title]Supported by the current model"></i>2024 <i class="icon-supported fa-solid fa-film" title="Supported by the current model" data-i18n="[title]Supported by the current model"></i>
2025 <i class="icon-unsupported fa-solid fa-film" title="Unsupported by the current model" data-i18n="[title]Unsupported by the current model"></i>2025 <i class="icon-unsupported fa-solid fa-film" title="Unsupported by the current model" data-i18n="[title]Unsupported by the current model"></i>
2026 </div>2026 </div>
2027 <div id="openai_audio_inlining_supported" data-source="makersuite,vertexai,openrouter">2027 <div id="openai_audio_inlining_supported" data-source="makersuite,vertexai,openrouter,openai,custom">
2028 <i class="icon-supported fa-solid fa-music" title="Supported by the current model" data-i18n="[title]Supported by the current model"></i>2028 <i class="icon-supported fa-solid fa-music" title="Supported by the current model" data-i18n="[title]Supported by the current model"></i>
2029 <i class="icon-unsupported fa-solid fa-music" title="Unsupported by the current model" data-i18n="[title]Unsupported by the current model"></i>2029 <i class="icon-unsupported fa-solid fa-music" title="Unsupported by the current model" data-i18n="[title]Unsupported by the current model"></i>
2030 </div>2030 </div>
public/scripts/extensions/stable-diffusion/index.js+10 -3
@@ -320,6 +320,7 @@ const defaultSettings = {
320 // OpenAI settings320 // OpenAI settings
321 openai_style: 'vivid',321 openai_style: 'vivid',
322 openai_quality: 'standard',322 openai_quality: 'standard',
323 openai_quality_gpt: 'auto',
323 openai_duration: '8',324 openai_duration: '8',
324325
325 style: 'Default',326 style: 'Default',
@@ -528,6 +529,7 @@ async function loadSettings() {
528 $('#sd_interactive_mode').prop('checked', extension_settings.sd.interactive_mode);529 $('#sd_interactive_mode').prop('checked', extension_settings.sd.interactive_mode);
529 $('#sd_openai_style').val(extension_settings.sd.openai_style);530 $('#sd_openai_style').val(extension_settings.sd.openai_style);
530 $('#sd_openai_quality').val(extension_settings.sd.openai_quality);531 $('#sd_openai_quality').val(extension_settings.sd.openai_quality);
532 $('#sd_openai_quality_gpt').val(extension_settings.sd.openai_quality_gpt);
531 $('#sd_openai_duration').val(extension_settings.sd.openai_duration);533 $('#sd_openai_duration').val(extension_settings.sd.openai_duration);
532 $('#sd_comfy_type').val(extension_settings.sd.comfy_type);534 $('#sd_comfy_type').val(extension_settings.sd.comfy_type);
533 $('#sd_comfy_url').val(extension_settings.sd.comfy_url);535 $('#sd_comfy_url').val(extension_settings.sd.comfy_url);
@@ -1850,7 +1852,7 @@ function switchModelSpecificControls(modelId) {
18501852
1851 modelControls.each(function () {1853 modelControls.each(function () {
1852 const models = String($(this).attr('data-sd-model') || '').split(',').map(m => m.trim());1854 const models = String($(this).attr('data-sd-model') || '').split(',').map(m => m.trim());
1853 $(this).toggle(models.includes(modelId));1855 $(this).toggle(models.some(m => modelId.includes(m)));
1854 });1856 });
1855}1857}
18561858
@@ -2169,6 +2171,7 @@ async function loadOpenAiModels() {
2169 { value: 'gpt-image-1.5', text: 'gpt-image-1.5' },2171 { value: 'gpt-image-1.5', text: 'gpt-image-1.5' },
2170 { value: 'gpt-image-1-mini', text: 'gpt-image-1-mini' },2172 { value: 'gpt-image-1-mini', text: 'gpt-image-1-mini' },
2171 { value: 'gpt-image-1', text: 'gpt-image-1' },2173 { value: 'gpt-image-1', text: 'gpt-image-1' },
2174 { value: 'chatgpt-image-latest', text: 'chatgpt-image-latest' },
2172 { value: 'dall-e-3', text: 'dall-e-3' },2175 { value: 'dall-e-3', text: 'dall-e-3' },
2173 { value: 'dall-e-2', text: 'dall-e-2' },2176 { value: 'dall-e-2', text: 'dall-e-2' },
2174 { value: 'sora-2', text: 'sora-2' },2177 { value: 'sora-2', text: 'sora-2' },
@@ -3697,7 +3700,7 @@ async function generateOpenAiImage(prompt, signal) {
36973700
3698 const isDalle2 = /dall-e-2/.test(extension_settings.sd.model);3701 const isDalle2 = /dall-e-2/.test(extension_settings.sd.model);
3699 const isDalle3 = /dall-e-3/.test(extension_settings.sd.model);3702 const isDalle3 = /dall-e-3/.test(extension_settings.sd.model);
3700 const isGptImg = /gpt-image-1/.test(extension_settings.sd.model);3703 const isGptImg = /gpt-image-(1|latest)/.test(extension_settings.sd.model);
3701 const isSora2 = /sora-2/.test(extension_settings.sd.model);3704 const isSora2 = /sora-2/.test(extension_settings.sd.model);
37023705
3703 if (isDalle2 && prompt.length > dalle2PromptLimit) {3706 if (isDalle2 && prompt.length > dalle2PromptLimit) {
@@ -3770,7 +3773,7 @@ async function generateOpenAiImage(prompt, signal) {
3770 model: extension_settings.sd.model,3773 model: extension_settings.sd.model,
3771 size: `${width}x${height}`,3774 size: `${width}x${height}`,
3772 n: 1,3775 n: 1,
3773 quality: isDalle3 ? extension_settings.sd.openai_quality : undefined,3776 quality: isDalle3 ? extension_settings.sd.openai_quality : (isGptImg ? extension_settings.sd.openai_quality_gpt : undefined),
3774 style: isDalle3 ? extension_settings.sd.openai_style : undefined,3777 style: isDalle3 ? extension_settings.sd.openai_style : undefined,
3775 response_format: isDalle2 || isDalle3 ? 'b64_json' : undefined,3778 response_format: isDalle2 || isDalle3 ? 'b64_json' : undefined,
3776 moderation: isGptImg ? 'low' : undefined,3779 moderation: isGptImg ? 'low' : undefined,
@@ -5309,6 +5312,10 @@ jQuery(async () => {
5309 extension_settings.sd.electronhub_quality = String($(this).val());5312 extension_settings.sd.electronhub_quality = String($(this).val());
5310 saveSettingsDebounced();5313 saveSettingsDebounced();
5311 });5314 });
5315 $('#sd_openai_quality_gpt').on('input', function () {
5316 extension_settings.sd.openai_quality_gpt = String($(this).val());
5317 saveSettingsDebounced();
5318 });
53125319
5313 if (!CSS.supports('field-sizing', 'content')) {5320 if (!CSS.supports('field-sizing', 'content')) {
5314 $('.sd_settings .inline-drawer-toggle').on('click', function () {5321 $('.sd_settings .inline-drawer-toggle').on('click', function () {
public/scripts/extensions/stable-diffusion/settings.html+10 -1
@@ -178,7 +178,16 @@
178 <option value="natural">Natural</option>178 <option value="natural">Natural</option>
179 </select>179 </select>
180 </div>180 </div>
181 <div data-sd-model="dall-e-3,cogview-4-250304" class="flex1">181 <div data-sd-model="gpt-image" class="flex1">
182 <label for="sd_openai_quality_gpt" data-i18n="Image Quality">Image Quality</label>
183 <select id="sd_openai_quality_gpt">
184 <option value="auto" data-i18n="Auto">Auto</option>
185 <option value="low" data-i18n="Low">Low</option>
186 <option value="medium" data-i18n="Medium">Medium</option>
187 <option value="high" data-i18n="High">High</option>
188 </select>
189 </div>
190 <div data-sd-model="dall-e-3,cogview-4" class="flex1">
182 <label for="sd_openai_quality" data-i18n="Image Quality">Image Quality</label>191 <label for="sd_openai_quality" data-i18n="Image Quality">Image Quality</label>
183 <select id="sd_openai_quality">192 <select id="sd_openai_quality">
184 <option value="standard" data-i18n="Standard">Standard</option>193 <option value="standard" data-i18n="Standard">Standard</option>
public/scripts/openai.js+16 -1
@@ -4689,6 +4689,9 @@ function getMaxContextOpenAI(value) {
4689 else if (value.includes('gpt-4.1')) {4689 else if (value.includes('gpt-4.1')) {
4690 return max_1mil;4690 return max_1mil;
4691 }4691 }
4692 else if (value.includes('gpt-audio')) {
4693 return max_128k;
4694 }
4692 else if (value.startsWith('o1')) {4695 else if (value.startsWith('o1')) {
4693 return max_128k;4696 return max_128k;
4694 }4697 }
@@ -4707,6 +4710,9 @@ function getMaxContextOpenAI(value) {
4707 else if (['gpt-4-32k', 'gpt-4-32k-0314', 'gpt-4-32k-0613'].includes(value)) {4710 else if (['gpt-4-32k', 'gpt-4-32k-0314', 'gpt-4-32k-0613'].includes(value)) {
4708 return max_32k;4711 return max_32k;
4709 }4712 }
4713 else if (value.includes('gpt-realtime')) {
4714 return max_32k;
4715 }
4710 else if (['gpt-3.5-turbo-16k', 'gpt-3.5-turbo-16k-0613'].includes(value)) {4716 else if (['gpt-3.5-turbo-16k', 'gpt-3.5-turbo-16k-0613'].includes(value)) {
4711 return max_16k;4717 return max_16k;
4712 }4718 }
@@ -5946,21 +5952,30 @@ export function isAudioInliningSupported() {
5946 return false;5952 return false;
5947 }5953 }
59485954
5949 // Only Gemini models support audio for now
5950 const audioSupportedModels = [5955 const audioSupportedModels = [
5951 'gemini-2.0',5956 'gemini-2.0',
5952 'gemini-2.5',5957 'gemini-2.5',
5953 'gemini-3',5958 'gemini-3',
5954 'gemini-exp-1206',5959 'gemini-exp-1206',
5960 'gpt-4o-audio',
5961 'gpt-4o-realtime',
5962 'gpt-4o-mini-audio',
5963 'gpt-4o-mini-realtime',
5964 'gpt-audio',
5965 'gpt-realtime',
5955 ];5966 ];
59565967
5957 switch (oai_settings.chat_completion_source) {5968 switch (oai_settings.chat_completion_source) {
5969 case chat_completion_sources.OPENAI:
5970 return audioSupportedModels.some(model => oai_settings.openai_model.includes(model));
5958 case chat_completion_sources.MAKERSUITE:5971 case chat_completion_sources.MAKERSUITE:
5959 return audioSupportedModels.some(model => oai_settings.google_model.includes(model));5972 return audioSupportedModels.some(model => oai_settings.google_model.includes(model));
5960 case chat_completion_sources.VERTEXAI:5973 case chat_completion_sources.VERTEXAI:
5961 return audioSupportedModels.some(model => oai_settings.vertexai_model.includes(model));5974 return audioSupportedModels.some(model => oai_settings.vertexai_model.includes(model));
5962 case chat_completion_sources.OPENROUTER:5975 case chat_completion_sources.OPENROUTER:
5963 return (Array.isArray(model_list) && model_list.find(m => m.id === oai_settings.openrouter_model)?.architecture?.input_modalities?.includes('audio'));5976 return (Array.isArray(model_list) && model_list.find(m => m.id === oai_settings.openrouter_model)?.architecture?.input_modalities?.includes('audio'));
5977 case chat_completion_sources.CUSTOM:
5978 return true;
5964 default:5979 default:
5965 return false;5980 return false;
5966 }5981 }
src/endpoints/backends/chat-completions.js+4 -1
@@ -2063,6 +2063,8 @@ router.post('/generate', async function (request, response) {
2063 if (getConfigValue('openai.randomizeUserId', false, 'boolean')) {2063 if (getConfigValue('openai.randomizeUserId', false, 'boolean')) {
2064 bodyParams['user'] = uuidv4();2064 bodyParams['user'] = uuidv4();
2065 }2065 }
2066
2067 embedOpenRouterMedia(request.body.messages, { audio: true, video: false });
2066 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENROUTER) {2068 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENROUTER) {
2067 apiUrl = 'https://openrouter.ai/api/v1';2069 apiUrl = 'https://openrouter.ai/api/v1';
2068 apiKey = readSecret(request.user.directories, SECRET_KEYS.OPENROUTER);2070 apiKey = readSecret(request.user.directories, SECRET_KEYS.OPENROUTER);
@@ -2122,7 +2124,7 @@ router.post('/generate', async function (request, response) {
2122 const enableGeminiSystemPromptCache = getConfigValue('gemini.enableSystemPromptCache', false, 'boolean');2124 const enableGeminiSystemPromptCache = getConfigValue('gemini.enableSystemPromptCache', false, 'boolean');
21232125
2124 if (Array.isArray(request.body.messages)) {2126 if (Array.isArray(request.body.messages)) {
2125 embedOpenRouterMedia(request.body.messages);2127 embedOpenRouterMedia(request.body.messages, { audio: true, video: true });
2126 addOpenRouterSignatures(request.body.messages, request.body.model);2128 addOpenRouterSignatures(request.body.messages, request.body.model);
21272129
2128 if (isClaude) {2130 if (isClaude) {
@@ -2160,6 +2162,7 @@ router.post('/generate', async function (request, response) {
21602162
2161 mergeObjectWithYaml(bodyParams, request.body.custom_include_body);2163 mergeObjectWithYaml(bodyParams, request.body.custom_include_body);
2162 mergeObjectWithYaml(headers, request.body.custom_include_headers);2164 mergeObjectWithYaml(headers, request.body.custom_include_headers);
2165 embedOpenRouterMedia(request.body.messages, { audio: true, video: false });
2163 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.PERPLEXITY) {2166 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.PERPLEXITY) {
2164 apiUrl = API_PERPLEXITY;2167 apiUrl = API_PERPLEXITY;
2165 apiKey = readSecret(request.user.directories, SECRET_KEYS.PERPLEXITY);2168 apiKey = readSecret(request.user.directories, SECRET_KEYS.PERPLEXITY);
src/prompt-converters.js+8 -4
@@ -1296,10 +1296,14 @@ export function calculateGoogleBudgetTokens(maxTokens, reasoningEffort, model) {
1296}1296}
12971297
1298/**1298/**
1299 * Embed media content in OpenRouter messages.1299 * Embed media content in OpenRouter messages (OpenAI-compatible).
1300 * @param {object[]} messages Array of messages1300 * @param {object[]} messages Array of messages
1301 * @param {object} options Options for embedding
1302 * @param {boolean} [options.audio] Enable audio embedding (default: true)
1303 * @param {boolean} [options.video] Enable video embedding (default: true)
1304 * @returns {void}
1301 */1305 */
1302export function embedOpenRouterMedia(messages) {1306export function embedOpenRouterMedia(messages, { audio = true, video = true } = { audio: true, video: true }) {
1303 if (!Array.isArray(messages)) {1307 if (!Array.isArray(messages)) {
1304 return;1308 return;
1305 }1309 }
@@ -1310,11 +1314,11 @@ export function embedOpenRouterMedia(messages) {
1310 }1314 }
13111315
1312 for (const contentPart of message.content) {1316 for (const contentPart of message.content) {
1313 if (contentPart?.type === 'video_url' && contentPart.video_url?.url?.startsWith('data:')) {1317 if (video && contentPart?.type === 'video_url' && contentPart.video_url?.url?.startsWith('data:')) {
1314 contentPart.type = 'input_video';1318 contentPart.type = 'input_video';
1315 }1319 }
13161320
1317 if (contentPart?.type === 'audio_url' && contentPart.audio_url?.url?.startsWith('data:')) {1321 if (audio && contentPart?.type === 'audio_url' && contentPart.audio_url?.url?.startsWith('data:')) {
1318 const formatMap = {1322 const formatMap = {
1319 'audio/mpeg': 'mp3',1323 'audio/mpeg': 'mp3',
1320 'audio/wav': 'wav',1324 'audio/wav': 'wav',