Add audio/video inlining for OpenRouter (#4751) * Add OpenRouter inlining support for video and audio * Refactor modality checks

94b95013bccc22f90740321694c4fac0037857fb

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: 38679897c6193a7fbf547fb0d2bb5191aee2b3da
5 files changed, +72 -10Ignore whitespace
public/index.html+4 -4
@@ -2005,7 +2005,7 @@
20052005 </div>
20062006 </div>
20072007 </div>
20082008 <div class="range-block" data-source="makersuite,vertexai,openrouter">
20092009 <label for="openai_video_inlining" class="checkbox_label flexWrap widthFreeExpand">
20102010 <input id="openai_video_inlining" type="checkbox" />
20112011 <span data-i18n="Send inline videos">Send inline videos</span>
@@ -2016,10 +2016,10 @@
20162016 </label>
20172017 <div id="video_inlining_hint" class="flexBasis100p toggle-description justifyLeft">
20182018 <span data-i18n="video_inlining_hint_1">Sends videos in prompts if the model supports it.</span>
20192019 <strong data-source="makersuite,vertexai" data-i18n="video_inlining_hint_4">Videos must be less than 20 MB and under 1 minute long</strong>
20202020 </div>
20212021 </div>
20222022 <div class="range-block" data-source="makersuite,vertexai,openrouter">
20232023 <label for="openai_audio_inlining" class="checkbox_label flexWrap widthFreeExpand">
20242024 <input id="openai_audio_inlining" type="checkbox" />
20252025 <span data-i18n="Send inline audio">Send inline audio</span>
@@ -2030,7 +2030,7 @@
20302030 </label>
20312031 <div id="audio_inlining_hint" class="flexBasis100p toggle-description justifyLeft">
20322032 <span data-i18n="audio_inlining_hint_1">Sends audio in prompts if the model supports it.</span>
20332033 <strong data-source="makersuite,vertexai" data-i18n="audio_inlining_hint_2">Audio must be less than 20 MB</strong>
20342034 </div>
20352035 </div>
20362036 <div class="range-block" data-source="makersuite,vertexai">
public/scripts/openai.js+5 -1
@@ -5661,7 +5661,7 @@ export function isImageInliningSupported() {
56615661 case chat_completion_sources.CLAUDE:
56625662 return visionSupportedModels.some(model => oai_settings.claude_model.includes(model));
56635663 case chat_completion_sources.OPENROUTER:
56645664 return (Array.isArray(model_list) && ['text+image->text+image', 'text+image->text'].includes(model_list.find(m => m.id === oai_settings.openrouter_model)?.architecture?.modalityinput_modalities?.includes('image'));
56655665 case chat_completion_sources.CUSTOM:
56665666 return true;
56675667 case chat_completion_sources.MISTRALAI:
@@ -5715,6 +5715,8 @@ export function isVideoInliningSupported() {
57155715 return videoSupportedModels.some(model => oai_settings.google_model.includes(model));
57165716 case chat_completion_sources.VERTEXAI:
57175717 return videoSupportedModels.some(model => oai_settings.vertexai_model.includes(model));
5718+ case chat_completion_sources.OPENROUTER:
5719+ return (Array.isArray(model_list) && model_list.find(m => m.id === oai_settings.openrouter_model)?.architecture?.input_modalities?.includes('video'));
57185720 default:
57195721 return false;
57205722 }
@@ -5745,6 +5747,8 @@ export function isAudioInliningSupported() {
57455747 return audioSupportedModels.some(model => oai_settings.google_model.includes(model));
57465748 case chat_completion_sources.VERTEXAI:
57475749 return audioSupportedModels.some(model => oai_settings.vertexai_model.includes(model));
5750+ case chat_completion_sources.OPENROUTER:
5751+ return (Array.isArray(model_list) && model_list.find(m => m.id === oai_settings.openrouter_model)?.architecture?.input_modalities?.includes('audio'));
57485752 default:
57495753 return false;
57505754 }
src/endpoints/backends/chat-completions.js+6 -2
@@ -40,6 +40,7 @@ import {
4040 postProcessPrompt,
4141 PROMPT_PROCESSING_TYPE,
4242 addAssistantPrefix,
43+ embedOpenRouterMedia,
4344} from '../../prompt-converters.js';
4445
4546import { readSecret, SECRET_KEYS } from '../secrets.js';
@@ -1837,8 +1838,11 @@ router.post('/generate', function (request, response) {
18371838 const cachingAtDepth = getConfigValue('claude.cachingAtDepth', -1, 'number');
18381839 const isClaude3or4 = /anthropic\/claude-(3|opus-4|sonnet-4|haiku-4)/.test(request.body.model);
18391840 const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';
18401841 if (Array.isArray(request.body.messages) && Number.isInteger(cachingAtDepth) && cachingAtDepth >= 0 && isClaude3or4) {
18411842 cachingAtDepthForOpenRouterClaudeembedOpenRouterMedia(request.body.messages, cachingAtDepth, cacheTTL);
1843+ if (Number.isInteger(cachingAtDepth) && cachingAtDepth >= 0 && isClaude3or4) {
1844+ cachingAtDepthForOpenRouterClaude(request.body.messages, cachingAtDepth, cacheTTL);
1845+ }
18421846 }
18431847
18441848 const isGemini = /google\/gemini/.test(request.body.model);
src/endpoints/openrouter.js+17 -3
@@ -46,8 +46,19 @@ router.post('/models/multimodal', async (_req, res) => {
4646
4747 /** @type {any} */
4848 const data = await response.json();
49- const models = data?.data || [];
49+
50- const multimodalModels = models.filter(m => ['text+image->text+image', 'text+image->text'].includes(m?.architecture?.modality)).map(m => m.id);
50+ if (!Array.isArray(data?.data)) {
51+ console.warn('OpenRouter API response was not an array');
52+ return res.json([]);
53+ }
54+
55+ const multimodalModels = data.data
56+ .filter(m => Array.isArray(m?.architecture?.input_modalities))
57+ .filter(m => m.architecture.input_modalities.includes('image'))
58+ .filter(m => Array.isArray(m?.architecture?.output_modalities))
59+ .filter(m => m.architecture.output_modalities.includes('text'))
60+ .sort((a, b) => a?.id && b?.id && a.id.localeCompare(b.id))
61+ .map(m => m.id);
5162
5263 return res.json(multimodalModels);
5364 } catch (error) {
@@ -80,8 +91,11 @@ router.post('/models/embedding', async (_req, res) => {
8091 }
8192
8293 const embeddingModels = data.data
94+ .filter(m => Array.isArray(m?.architecture?.input_modalities))
95+ .filter(m => m.architecture.input_modalities.includes('text'))
8396 .filter(m => Array.isArray(m?.architecture?.output_modalities))
8497 .filter(m => m.architecture.output_modalities.includes('embeddings'));
98+ .sort((a, b) => a?.id && b?.id && a.id.localeCompare(b.id));
8599
86100 return res.json(embeddingModels);
87101 } catch (error) {
src/prompt-converters.js+40 -0
@@ -1156,3 +1156,43 @@ export function calculateGoogleBudgetTokens(maxTokens, reasoningEffort, model) {
11561156
11571157 return null;
11581158}
1159+
1160+/**
1161+ * Embed media content in OpenRouter messages.
1162+ * @param {object[]} messages Array of messages
1163+ */
1164+export function embedOpenRouterMedia(messages) {
1165+ if (!Array.isArray(messages)) {
1166+ return;
1167+ }
1168+
1169+ for (const message of messages) {
1170+ if (!Array.isArray(message.content)) {
1171+ continue;
1172+ }
1173+
1174+ for (const contentPart of message.content) {
1175+ if (contentPart?.type === 'video_url' && contentPart.video_url?.url?.startsWith('data:')) {
1176+ contentPart.type = 'input_video';
1177+ }
1178+
1179+ if (contentPart?.type === 'audio_url' && contentPart.audio_url?.url?.startsWith('data:')) {
1180+ const formatMap = {
1181+ 'audio/mpeg': 'mp3',
1182+ 'audio/wav': 'wav',
1183+ };
1184+
1185+ const [header, base64Data] = contentPart.audio_url.url.split(',');
1186+ const mimeType = header.match(/data:([^;]+)/)?.[1] || 'audio/mpeg';
1187+
1188+ contentPart.type = 'input_audio';
1189+ contentPart.input_audio = {
1190+ format: formatMap[mimeType] || 'mp3',
1191+ data: base64Data,
1192+ };
1193+
1194+ delete contentPart.audio_url;
1195+ }
1196+ }
1197+ }
1198+}