OpenRouter: add as image generation source

6fccccca6d7911555aba55e9b555f48ec925e0c5

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: b3708087536929469b2989d2364fabf0b160c0f1
3 files changed, +158 -31Showing whitespace changes
public/scripts/extensions/stable-diffusion/index.js+59 -0
@@ -91,6 +91,7 @@ const sources = {
9191 xai: 'xai',
9292 google: 'google',
9393 zai: 'zai',
94+ openrouter: 'openrouter',
9495};
9596
9697const initiators = {
@@ -1325,6 +1326,7 @@ async function onModelChange() {
13251326 sources.google,
13261327 sources.chutes,
13271328 sources.zai,
1329+ sources.openrouter,
13281330 ];
13291331
13301332 if (cloudSources.includes(extension_settings.sd.source)) {
@@ -1561,6 +1563,9 @@ async function loadSamplers() {
15611563 case sources.zai:
15621564 samplers = ['N/A'];
15631565 break;
1566+ case sources.openrouter:
1567+ samplers = ['N/A'];
1568+ break;
15641569 }
15651570
15661571 for (const sampler of samplers) {
@@ -1769,6 +1774,9 @@ async function loadModels() {
17691774 case sources.zai:
17701775 models = await loadZaiModels();
17711776 break;
1777+ case sources.openrouter:
1778+ models = await loadOpenRouterModels();
1779+ break;
17721780 }
17731781
17741782 if (extension_settings.sd.source === sources.electronhub) {
@@ -2253,6 +2261,19 @@ async function loadZaiModels() {
22532261 return ['cogview-4-250304'].map(name => ({ value: name, text: name }));
22542262}
22552263
2264+async function loadOpenRouterModels() {
2265+ const result = await fetch('/api/openrouter/models/image', {
2266+ method: 'POST',
2267+ headers: getRequestHeaders({ omitContentType: true }),
2268+ });
2269+
2270+ if (result.ok) {
2271+ return await result.json();
2272+ }
2273+
2274+ return [];
2275+}
2276+
22562277function loadNovelSchedulers() {
22572278 return ['karras', 'native', 'exponential', 'polyexponential'];
22582279}
@@ -2347,6 +2368,9 @@ async function loadSchedulers() {
23472368 case sources.zai:
23482369 schedulers = ['N/A'];
23492370 break;
2371+ case sources.openrouter:
2372+ schedulers = ['N/A'];
2373+ break;
23502374 }
23512375
23522376 for (const scheduler of schedulers) {
@@ -2453,6 +2477,9 @@ async function loadVaes() {
24532477 case sources.zai:
24542478 vaes = ['N/A'];
24552479 break;
2480+ case sources.openrouter:
2481+ vaes = ['N/A'];
2482+ break;
24562483 }
24572484
24582485 for (const vae of vaes) {
@@ -3051,6 +3078,9 @@ async function sendGenerationRequest(generationType, prompt, additionalNegativeP
30513078 case sources.zai:
30523079 result = await generateZaiImage(prefixedPrompt, signal);
30533080 break;
3081+ case sources.openrouter:
3082+ result = await generateOpenRouterImage(prefixedPrompt, signal);
3083+ break;
30543084 }
30553085
30563086 if (!result.data) {
@@ -4161,6 +4191,33 @@ async function generateZaiImage(prompt, signal) {
41614191 throw new Error(text);
41624192}
41634193
4194+/**
4195+ * Generates an image using the OpenRouter API.
4196+ * @param {string} prompt The main instruction used to guide the image generation.
4197+ * @param {AbortSignal} signal An AbortSignal object that can be used to cancel the request.
4198+ * @returns {Promise<{format: string, data: string}>}
4199+ */
4200+async function generateOpenRouterImage(prompt, signal) {
4201+ const result = await fetch('/api/openrouter/image/generate', {
4202+ method: 'POST',
4203+ headers: getRequestHeaders(),
4204+ signal: signal,
4205+ body: JSON.stringify({
4206+ model: extension_settings.sd.model,
4207+ prompt: prompt,
4208+ aspect_ratio: getClosestAspectRatio(extension_settings.sd.width, extension_settings.sd.height, 'stability'),
4209+ }),
4210+ });
4211+
4212+ if (result.ok) {
4213+ const data = await result.json();
4214+ return { format: 'jpg', data: data.image };
4215+ }
4216+
4217+ const text = await result.text();
4218+ throw new Error(text);
4219+}
4220+
41644221async function onComfyOpenWorkflowEditorClick() {
41654222 let workflow = await (await fetch('/api/sd/comfy/workflow', {
41664223 method: 'POST',
@@ -4469,6 +4526,8 @@ function isValidState() {
44694526 return secret_state[SECRET_KEYS.MAKERSUITE] || secret_state[SECRET_KEYS.VERTEXAI] || secret_state[SECRET_KEYS.VERTEXAI_SERVICE_ACCOUNT];
44704527 case sources.zai:
44714528 return secret_state[SECRET_KEYS.ZAI];
4529+ case sources.openrouter:
4530+ return secret_state[SECRET_KEYS.OPENROUTER];
44724531 default:
44734532 return false;
44744533 }
public/scripts/extensions/stable-diffusion/settings.html+1 -0
@@ -50,6 +50,7 @@
5050 <option value="nanogpt">NanoGPT</option>
5151 <option value="novel">NovelAI Diffusion</option>
5252 <option value="openai">OpenAI</option>
53+ <option value="openrouter">OpenRouter</option>
5354 <option value="pollinations">Pollinations</option>
5455 <option value="vlad">SD.Next (vladmandic)</option>
5556 <option value="stability">Stability AI</option>
src/endpoints/openrouter.js+98 -31
@@ -1,5 +1,7 @@
11import express from 'express';
22import fetch from 'node-fetch';
3+import mime from 'mime-types';
4+import { readSecret, SECRET_KEYS } from './secrets.js';
35
46export const router = express.Router();
57const API_OPENROUTER = 'https://openrouter.ai/api/v1';
@@ -30,18 +32,23 @@ router.post('/models/providers', async (req, res) => {
3032 }
3133});
3234
33-router.post('/models/multimodal', async (_req, res) => {
35+/**
34- try {
36+ * Fetches and filters models from OpenRouter API based on modality criteria.
35- // The endpoint is available without authentication
37+ * @param {string} endpoint - The API endpoint to fetch from
36- const response = await fetch(`${API_OPENROUTER}/models`, {
38+ * @param {string} inputModality - Required input modality
39+ * @param {string} outputModality - Required output modality
40+ * @param {boolean} [idsOnly=false] - Whether to return only model IDs
41+ * @returns {Promise<any[]>} Filtered models or model IDs
42+ */
43+async function fetchModelsByModality(endpoint, inputModality, outputModality, idsOnly = false) {
44+ const response = await fetch(`${API_OPENROUTER}${endpoint}`, {
3745 method: 'GET',
38- headers: {
46+ headers: { 'Accept': 'application/json' },
39- 'Accept': 'application/json',
40- },
4147 });
4248
4349 if (!response.ok) {
44- return res.json([]);
50+ console.warn('OpenRouter API request failed', response.statusText);
51+ return [];
4552 }
4653
4754 /** @type {any} */
@@ -49,18 +56,23 @@ router.post('/models/multimodal', async (_req, res) => {
4956
5057 if (!Array.isArray(data?.data)) {
5158 console.warn('OpenRouter API response was not an array');
5259 return res.json([]);
5360 }
5461
5562 const multimodalModelsfiltered = data.data
5663 .filter(m => Array.isArray(m?.architecture?.input_modalities))
5764 .filter(m => m.architecture.input_modalities.includes('image'inputModality))
5865 .filter(m => Array.isArray(m?.architecture?.output_modalities))
5966 .filter(m => m.architecture.output_modalities.includes('text'outputModality))
6067 .sort((a, b) => a?.id && b?.id &&? a.id.localeCompare(b.id) : 0);
61- .map(m => m.id);
68+
69+ return idsOnly ? filtered.map(m => m.id) : filtered;
70+}
6271
63- return res.json(multimodalModels);
72+router.post('/models/multimodal', async (_req, res) => {
73+ try {
74+ const models = await fetchModelsByModality('/models', 'image', 'text', true);
75+ return res.json(models);
6476 } catch (error) {
6577 console.error(error);
6678 return res.sendStatus(500);
@@ -69,35 +81,90 @@ router.post('/models/multimodal', async (_req, res) => {
6981
7082router.post('/models/embedding', async (_req, res) => {
7183 try {
72- // The endpoint is available without authentication
84+ const models = await fetchModelsByModality('/embeddings/models', 'text', 'embeddings');
73- const response = await fetch(`${API_OPENROUTER}/embeddings/models`, {
85+ return res.json(models);
74- method: 'GET',
86+ } catch (error) {
87+ console.error(error);
88+ return res.sendStatus(500);
89+ }
90+});
91+
92+router.post('/models/image', async (_req, res) => {
93+ try {
94+ const models = await fetchModelsByModality('/models', 'text', 'image');
95+ return res.json(models.map(m => ({ value: m.id, text: m.name || m.id })));
96+ } catch (error) {
97+ console.error(error);
98+ return res.sendStatus(500);
99+ }
100+});
101+
102+router.post('/image/generate', async (req, res) => {
103+ try {
104+ const key = readSecret(req.user.directories, SECRET_KEYS.OPENROUTER);
105+
106+ if (!key) {
107+ console.warn('OpenRouter API key not found');
108+ return res.status(400).json({ error: 'OpenRouter API key not found' });
109+ }
110+
111+ console.debug('OpenRouter image generation request', req.body);
112+
113+ const { model, prompt } = req.body;
114+
115+ if (!model || !prompt) {
116+ return res.status(400).json({ error: 'Model and prompt are required' });
117+ }
118+
119+ const response = await fetch(`${API_OPENROUTER}/chat/completions`, {
120+ method: 'POST',
75121 headers: {
76122 'AcceptContent-Type': 'application/json',
123+ 'Authorization': `Bearer ${key}`,
124+ },
125+ body: JSON.stringify({
126+ model: model,
127+ messages: [
128+ {
129+ role: 'user',
130+ content: prompt,
77131 },
132+ ],
133+ modalities: ['image', 'text'],
134+ image_config: {
135+ aspect_ratio: req.body.aspect_ratio || '1:1',
136+ },
137+ }),
78138 });
79139
80140 if (!response.ok) {
81141 console.warn('OpenRouter APIimage requestgeneration failed', await response.statusTexttext());
82142 return res.jsonsendStatus([]500);
83143 }
84144
85145 /** @type {any} */
86146 const data = await response.json();
87147
88- if (!Array.isArray(data?.data)) {
148+ const imageUrl = data?.choices?.[0]?.message?.images?.[0]?.image_url?.url;
89- console.warn('OpenRouter API response was not an array');
149+
90- return res.json([]);
150+ if (!imageUrl) {
151+ console.warn('No image URL found in OpenRouter response', data);
152+ return res.sendStatus(500);
91153 }
92154
93- const embeddingModels = data.data
155+ const [mimeType, base64Data] = /^data:(.*);base64,(.*)$/.exec(imageUrl)?.slice(1) || [];
94- .filter(m => Array.isArray(m?.architecture?.input_modalities))
156+
95- .filter(m => m.architecture.input_modalities.includes('text'))
157+ if (!mimeType || !base64Data) {
96- .filter(m => Array.isArray(m?.architecture?.output_modalities))
158+ console.warn('Invalid image data format', imageUrl);
97- .filter(m => m.architecture.output_modalities.includes('embeddings'))
159+ return res.sendStatus(500);
98- .sort((a, b) => a?.id && b?.id && a.id.localeCompare(b.id));
160+ }
161+
162+ const result = {
163+ format: mime.extension(mimeType) || 'png',
164+ image: base64Data,
165+ };
99166
100167 return res.json(embeddingModelsresult);
101168 } catch (error) {
102169 console.error(error);
103170 return res.sendStatus(500);