Add audio/video inlining for OpenRouter (#4751) * Add OpenRouter inlining support for video and audio * Refactor modality checks

94b95013bccc22f90740321694c4fac0037857fb

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: 38679897c6193a7fbf547fb0d2bb5191aee2b3da
5 files changed, +72 -10Ignore whitespace
public/index.html+4 -4
@@ -2005,7 +2005,7 @@
2005 </div>2005 </div>
2006 </div>2006 </div>
2007 </div>2007 </div>
2008 <div class="range-block" data-source="makersuite,vertexai">2008 <div class="range-block" data-source="makersuite,vertexai,openrouter">
2009 <label for="openai_video_inlining" class="checkbox_label flexWrap widthFreeExpand">2009 <label for="openai_video_inlining" class="checkbox_label flexWrap widthFreeExpand">
2010 <input id="openai_video_inlining" type="checkbox" />2010 <input id="openai_video_inlining" type="checkbox" />
2011 <span data-i18n="Send inline videos">Send inline videos</span>2011 <span data-i18n="Send inline videos">Send inline videos</span>
@@ -2016,10 +2016,10 @@
2016 </label>2016 </label>
2017 <div id="video_inlining_hint" class="flexBasis100p toggle-description justifyLeft">2017 <div id="video_inlining_hint" class="flexBasis100p toggle-description justifyLeft">
2018 <span data-i18n="video_inlining_hint_1">Sends videos in prompts if the model supports it.</span>2018 <span data-i18n="video_inlining_hint_1">Sends videos in prompts if the model supports it.</span>
2019 <strong data-i18n="video_inlining_hint_4">Videos must be less than 20 MB and under 1 minute long</strong>2019 <strong data-source="makersuite,vertexai" data-i18n="video_inlining_hint_4">Videos must be less than 20 MB and under 1 minute long</strong>
2020 </div>2020 </div>
2021 </div>2021 </div>
2022 <div class="range-block" data-source="makersuite,vertexai">2022 <div class="range-block" data-source="makersuite,vertexai,openrouter">
2023 <label for="openai_audio_inlining" class="checkbox_label flexWrap widthFreeExpand">2023 <label for="openai_audio_inlining" class="checkbox_label flexWrap widthFreeExpand">
2024 <input id="openai_audio_inlining" type="checkbox" />2024 <input id="openai_audio_inlining" type="checkbox" />
2025 <span data-i18n="Send inline audio">Send inline audio</span>2025 <span data-i18n="Send inline audio">Send inline audio</span>
@@ -2030,7 +2030,7 @@
2030 </label>2030 </label>
2031 <div id="audio_inlining_hint" class="flexBasis100p toggle-description justifyLeft">2031 <div id="audio_inlining_hint" class="flexBasis100p toggle-description justifyLeft">
2032 <span data-i18n="audio_inlining_hint_1">Sends audio in prompts if the model supports it.</span>2032 <span data-i18n="audio_inlining_hint_1">Sends audio in prompts if the model supports it.</span>
2033 <strong data-i18n="audio_inlining_hint_2">Audio must be less than 20 MB</strong>2033 <strong data-source="makersuite,vertexai" data-i18n="audio_inlining_hint_2">Audio must be less than 20 MB</strong>
2034 </div>2034 </div>
2035 </div>2035 </div>
2036 <div class="range-block" data-source="makersuite,vertexai">2036 <div class="range-block" data-source="makersuite,vertexai">
public/scripts/openai.js+5 -1
@@ -5661,7 +5661,7 @@ export function isImageInliningSupported() {
5661 case chat_completion_sources.CLAUDE:5661 case chat_completion_sources.CLAUDE:
5662 return visionSupportedModels.some(model => oai_settings.claude_model.includes(model));5662 return visionSupportedModels.some(model => oai_settings.claude_model.includes(model));
5663 case chat_completion_sources.OPENROUTER:5663 case chat_completion_sources.OPENROUTER:
5664 return (Array.isArray(model_list) && ['text+image->text+image', 'text+image->text'].includes(model_list.find(m => m.id === oai_settings.openrouter_model)?.architecture?.modality));5664 return (Array.isArray(model_list) && model_list.find(m => m.id === oai_settings.openrouter_model)?.architecture?.input_modalities?.includes('image'));
5665 case chat_completion_sources.CUSTOM:5665 case chat_completion_sources.CUSTOM:
5666 return true;5666 return true;
5667 case chat_completion_sources.MISTRALAI:5667 case chat_completion_sources.MISTRALAI:
@@ -5715,6 +5715,8 @@ export function isVideoInliningSupported() {
5715 return videoSupportedModels.some(model => oai_settings.google_model.includes(model));5715 return videoSupportedModels.some(model => oai_settings.google_model.includes(model));
5716 case chat_completion_sources.VERTEXAI:5716 case chat_completion_sources.VERTEXAI:
5717 return videoSupportedModels.some(model => oai_settings.vertexai_model.includes(model));5717 return videoSupportedModels.some(model => oai_settings.vertexai_model.includes(model));
5718 case chat_completion_sources.OPENROUTER:
5719 return (Array.isArray(model_list) && model_list.find(m => m.id === oai_settings.openrouter_model)?.architecture?.input_modalities?.includes('video'));
5718 default:5720 default:
5719 return false;5721 return false;
5720 }5722 }
@@ -5745,6 +5747,8 @@ export function isAudioInliningSupported() {
5745 return audioSupportedModels.some(model => oai_settings.google_model.includes(model));5747 return audioSupportedModels.some(model => oai_settings.google_model.includes(model));
5746 case chat_completion_sources.VERTEXAI:5748 case chat_completion_sources.VERTEXAI:
5747 return audioSupportedModels.some(model => oai_settings.vertexai_model.includes(model));5749 return audioSupportedModels.some(model => oai_settings.vertexai_model.includes(model));
5750 case chat_completion_sources.OPENROUTER:
5751 return (Array.isArray(model_list) && model_list.find(m => m.id === oai_settings.openrouter_model)?.architecture?.input_modalities?.includes('audio'));
5748 default:5752 default:
5749 return false;5753 return false;
5750 }5754 }
src/endpoints/backends/chat-completions.js+6 -2
@@ -40,6 +40,7 @@ import {
40 postProcessPrompt,40 postProcessPrompt,
41 PROMPT_PROCESSING_TYPE,41 PROMPT_PROCESSING_TYPE,
42 addAssistantPrefix,42 addAssistantPrefix,
43 embedOpenRouterMedia,
43} from '../../prompt-converters.js';44} from '../../prompt-converters.js';
4445
45import { readSecret, SECRET_KEYS } from '../secrets.js';46import { readSecret, SECRET_KEYS } from '../secrets.js';
@@ -1837,8 +1838,11 @@ router.post('/generate', function (request, response) {
1837 const cachingAtDepth = getConfigValue('claude.cachingAtDepth', -1, 'number');1838 const cachingAtDepth = getConfigValue('claude.cachingAtDepth', -1, 'number');
1838 const isClaude3or4 = /anthropic\/claude-(3|opus-4|sonnet-4|haiku-4)/.test(request.body.model);1839 const isClaude3or4 = /anthropic\/claude-(3|opus-4|sonnet-4|haiku-4)/.test(request.body.model);
1839 const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';1840 const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';
1840 if (Array.isArray(request.body.messages) && Number.isInteger(cachingAtDepth) && cachingAtDepth >= 0 && isClaude3or4) {1841 if (Array.isArray(request.body.messages)) {
1841 cachingAtDepthForOpenRouterClaude(request.body.messages, cachingAtDepth, cacheTTL);1842 embedOpenRouterMedia(request.body.messages);
1843 if (Number.isInteger(cachingAtDepth) && cachingAtDepth >= 0 && isClaude3or4) {
1844 cachingAtDepthForOpenRouterClaude(request.body.messages, cachingAtDepth, cacheTTL);
1845 }
1842 }1846 }
18431847
1844 const isGemini = /google\/gemini/.test(request.body.model);1848 const isGemini = /google\/gemini/.test(request.body.model);
src/endpoints/openrouter.js+17 -3
@@ -46,8 +46,19 @@ router.post('/models/multimodal', async (_req, res) => {
4646
47 /** @type {any} */47 /** @type {any} */
48 const data = await response.json();48 const data = await response.json();
49 const models = data?.data || [];49
50 const multimodalModels = models.filter(m => ['text+image->text+image', 'text+image->text'].includes(m?.architecture?.modality)).map(m => m.id);50 if (!Array.isArray(data?.data)) {
51 console.warn('OpenRouter API response was not an array');
52 return res.json([]);
53 }
54
55 const multimodalModels = data.data
56 .filter(m => Array.isArray(m?.architecture?.input_modalities))
57 .filter(m => m.architecture.input_modalities.includes('image'))
58 .filter(m => Array.isArray(m?.architecture?.output_modalities))
59 .filter(m => m.architecture.output_modalities.includes('text'))
60 .sort((a, b) => a?.id && b?.id && a.id.localeCompare(b.id))
61 .map(m => m.id);
5162
52 return res.json(multimodalModels);63 return res.json(multimodalModels);
53 } catch (error) {64 } catch (error) {
@@ -80,8 +91,11 @@ router.post('/models/embedding', async (_req, res) => {
80 }91 }
8192
82 const embeddingModels = data.data93 const embeddingModels = data.data
94 .filter(m => Array.isArray(m?.architecture?.input_modalities))
95 .filter(m => m.architecture.input_modalities.includes('text'))
83 .filter(m => Array.isArray(m?.architecture?.output_modalities))96 .filter(m => Array.isArray(m?.architecture?.output_modalities))
84 .filter(m => m.architecture.output_modalities.includes('embeddings'));97 .filter(m => m.architecture.output_modalities.includes('embeddings'))
98 .sort((a, b) => a?.id && b?.id && a.id.localeCompare(b.id));
8599
86 return res.json(embeddingModels);100 return res.json(embeddingModels);
87 } catch (error) {101 } catch (error) {
src/prompt-converters.js+40 -0
@@ -1156,3 +1156,43 @@ export function calculateGoogleBudgetTokens(maxTokens, reasoningEffort, model) {
11561156
1157 return null;1157 return null;
1158}1158}
1159
1160/**
1161 * Embed media content in OpenRouter messages.
1162 * @param {object[]} messages Array of messages
1163 */
1164export function embedOpenRouterMedia(messages) {
1165 if (!Array.isArray(messages)) {
1166 return;
1167 }
1168
1169 for (const message of messages) {
1170 if (!Array.isArray(message.content)) {
1171 continue;
1172 }
1173
1174 for (const contentPart of message.content) {
1175 if (contentPart?.type === 'video_url' && contentPart.video_url?.url?.startsWith('data:')) {
1176 contentPart.type = 'input_video';
1177 }
1178
1179 if (contentPart?.type === 'audio_url' && contentPart.audio_url?.url?.startsWith('data:')) {
1180 const formatMap = {
1181 'audio/mpeg': 'mp3',
1182 'audio/wav': 'wav',
1183 };
1184
1185 const [header, base64Data] = contentPart.audio_url.url.split(',');
1186 const mimeType = header.match(/data:([^;]+)/)?.[1] || 'audio/mpeg';
1187
1188 contentPart.type = 'input_audio';
1189 contentPart.input_audio = {
1190 format: formatMap[mimeType] || 'mp3',
1191 data: base64Data,
1192 };
1193
1194 delete contentPart.audio_url;
1195 }
1196 }
1197 }
1198}