Captioning: Load external model lists for OpenRouter, AIML and Pollinations

ad960197b725d6857fb99b34a5368130400206ec

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: ad516c864ace0b2efd4eba000f3cee30b04c2c9c
3 files changed, +78 -68Showing whitespace changes
public/scripts/extensions/caption/index.js+23 -12
@@ -488,8 +488,13 @@ jQuery(async function () {
488488 $('#img_file').on('change', (e) => onSelectImage(e.originalEvent, '', false));
489489 }
490490 async function switchMultimodalBlocks() {
491491 await addOpenRouterModelsaddRemoteEndpointModels();
492492 const isMultimodal = extension_settings.caption.source === 'multimodal';
493+ if (!extension_settings.caption.multimodal_model) {
494+ const dropdown = $('#caption_multimodal_model');
495+ const options = dropdown.find(`option[data-type="${extension_settings.caption.multimodal_api}"]`);
496+ extension_settings.caption.multimodal_model = String(options.first().val());
497+ }
493498 $('#caption_multimodal_block').toggle(isMultimodal);
494499 $('#caption_prompt_block').toggle(isMultimodal);
495500 $('#caption_multimodal_api').val(extension_settings.caption.multimodal_api);
@@ -504,16 +509,18 @@ jQuery(async function () {
504509 const html = await renderExtensionTemplateAsync('caption', 'settings', { TEMPLATE_DEFAULT, PROMPT_DEFAULT });
505510 $('#caption_container').append(html);
506511 }
507- async function addOpenRouterModels() {
512+
513+ async function addRemoteEndpointModels() {
514+ async function processEndpoint(api, url) {
508515 const dropdown = document.getElementById('caption_multimodal_model');
509516 if (!(dropdown instanceof HTMLSelectElement)) {
510517 return;
511518 }
512519 if (extension_settings.caption.source !== 'multimodal' || extension_settings.caption.multimodal_api !== 'openrouter'api) {
513520 return;
514521 }
515522 const options = Array.from(dropdown.options);
516523 const response = await fetch('/api/openrouter/models/multimodal'url, {
517524 method: 'POST',
518525 headers: getRequestHeaders(),
519526 });
@@ -522,19 +529,24 @@ jQuery(async function () {
522529 }
523530 const modelIds = await response.json();
524531 if (Array.isArray(modelIds) && modelIds.length > 0) {
525532 modelIds.sort().forEach((modelId) => {
526533 if (!modelId || typeof modelId !== 'string' || options.some(o => o.value === modelId)) {
527534 return;
528535 }
529536 const option = document.createElement('option');
530537 option.value = modelId;
531538 option.textContent = modelId;
532539 option.dataset.type = 'openrouter'api;
533540 dropdown.add(option);
534541 });
535542 }
536543 }
537544
545+ await processEndpoint('openrouter', '/api/openrouter/models/multimodal');
546+ await processEndpoint('aimlapi', '/api/backends/chat-completions/aimlapi/models/multimodal');
547+ await processEndpoint('pollinations', '/api/backends/chat-completions/pollinations/models/multimodal');
548+ }
549+
538550 await addSettings();
539551 addPictureSendForm();
540552 addSendPictureButton();
@@ -550,9 +562,9 @@ jQuery(async function () {
550562 $('#caption_prompt').val(extension_settings.caption.prompt);
551563 $('#caption_template').val(extension_settings.caption.template);
552564 $('#caption_refine_mode').on('input', onRefineModeInput);
553565 $('#caption_source').on('change', async () => {
554566 extension_settings.caption.source = String($('#caption_source').val());
555567 await switchMultimodalBlocks();
556568 saveSettingsDebounced();
557569 });
558570 $('#caption_prompt').on('input', () => {
@@ -581,13 +593,12 @@ jQuery(async function () {
581593 $('#ollama_download_model').trigger('click');
582594 $('#dialogue_popup_input').val(presetModel);
583595 });
584596 $('#caption_multimodal_api').on('change', async () => {
585597 const api = String($('#caption_multimodal_api').val());
586- const model = String($(`#caption_multimodal_model option[data-type="${api}"]`).first().val());
587598 extension_settings.caption.multimodal_api = api;
588599 extension_settings.caption.multimodal_model = model'';
600+ await switchMultimodalBlocks();
589601 saveSettingsDebounced();
590- switchMultimodalBlocks();
591602 });
592603 $('#caption_multimodal_model').on('change', () => {
593604 extension_settings.caption.multimodal_model = String($('#caption_multimodal_model').val());
public/scripts/extensions/caption/settings.html+1 -56
@@ -40,6 +40,7 @@
4040 <div class="flex1 flex-container flexFlowColumn flexNoGap">
4141 <label for="caption_multimodal_model" data-i18n="Model">Model</label>
4242 <select id="caption_multimodal_model" class="flex1 text_pole">
43+ <!-- AI/ML API, OpenRouter, Pollinations are added externally by JavaScript -->
4344 <option data-type="cohere" value="c4ai-aya-vision-8b">c4ai-aya-vision-8b</option>
4445 <option data-type="cohere" value="c4ai-aya-vision-32b">c4ai-aya-vision-32b</option>
4546 <option data-type="cohere" value="command-a-vision-07-2025">command-a-vision-07-2025</option>
@@ -150,50 +151,6 @@
150151 <option data-type="groq" value="llama-3.2-11b-vision-preview">llama-3.2-11b-vision-preview</option>
151152 <option data-type="groq" value="llama-3.2-90b-vision-preview">llama-3.2-90b-vision-preview</option>
152153 <option data-type="groq" value="llava-v1.5-7b-4096-preview">llava-v1.5-7b-4096-preview</option>
153- <option data-type="aimlapi" value="gpt-4o-2024-05-13">gpt-4o-2024-05-13</option>
154- <option data-type="aimlapi" value="gpt-4o-2024-08-06">gpt-4o-2024-08-06</option>
155- <option data-type="aimlapi" value="gpt-4o">gpt-4o</option>
156- <option data-type="aimlapi" value="gpt-4o-mini">gpt-4o-mini</option>
157- <option data-type="aimlapi" value="gpt-4o-mini-2024-07-18">gpt-4o-mini-2024-07-18</option>
158- <option data-type="aimlapi" value="chatgpt-4o-latest">chatgpt-4o-latest</option>
159- <option data-type="aimlapi" value="gpt-4-turbo">gpt-4-turbo</option>
160- <option data-type="aimlapi" value="claude-3-opus-20240229">claude-3-opus-20240229</option>
161- <option data-type="aimlapi" value="claude-3-sonnet-20240229">claude-3-sonnet-20240229</option>
162- <option data-type="aimlapi" value="claude-3-haiku-20240307">claude-3-haiku-20240307</option>
163- <option data-type="aimlapi" value="claude-3-5-sonnet-20240620">claude-3-5-sonnet-20240620</option>
164- <option data-type="aimlapi" value="google/gemini-1.5-flash">google/gemini-1.5-flash</option>
165- <option data-type="aimlapi" value="google/gemini-1.5-pro">google/gemini-1.5-pro</option>
166- <option data-type="aimlapi" value="google/gemini-2.0-flash">google/gemini-2.0-flash</option>
167- <option data-type="aimlapi" value="google/gemini-2.0-flash-exp">google/gemini-2.0-flash-exp</option>
168- <option data-type="aimlapi" value="google/gemini-2.5-flash-preview">google/gemini-2.5-flash-preview</option>
169- <option data-type="aimlapi" value="meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo">meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo</option>
170- <option data-type="aimlapi" value="meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo">meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo</option>
171- <option data-type="openrouter" value="openai/gpt-4-vision-preview">openai/gpt-4-vision-preview</option>
172- <option data-type="openrouter" value="openai/gpt-4o">openai/gpt-4o</option>
173- <option data-type="openrouter" value="openai/gpt-4o-2024-05-13">openai/gpt-4o-2024-05-13</option>
174- <option data-type="openrouter" value="openai/gpt-4o-2024-08-06">openai/gpt-4o-2024-08-06</option>
175- <option data-type="openrouter" value="openai/gpt-4-turbo">openai/gpt-4-turbo</option>
176- <option data-type="openrouter" value="openai/gpt-4o-mini">openai/gpt-4o-mini</option>
177- <option data-type="openrouter" value="openai/gpt-4o-mini-2024-07-18">openai/gpt-4o-mini-2024-07-18</option>
178- <option data-type="openrouter" value="openai/chatgpt-4o-latest">openai/chatgpt-4o-latest</option>
179- <option data-type="openrouter" value="haotian-liu/llava-13b">haotian-liu/llava-13b</option>
180- <option data-type="openrouter" value="fireworks/firellava-13b">fireworks/firellava-13b</option>
181- <option data-type="openrouter" value="anthropic/claude-3.5-sonnet">anthropic/claude-3.5-sonnet</option>
182- <option data-type="openrouter" value="anthropic/claude-3-haiku">anthropic/claude-3-haiku</option>
183- <option data-type="openrouter" value="anthropic/claude-3-sonnet">anthropic/claude-3-sonnet</option>
184- <option data-type="openrouter" value="anthropic/claude-3-opus">anthropic/claude-3-opus</option>
185- <option data-type="openrouter" value="anthropic/claude-3.5-sonnet:beta">anthropic/claude-3.5-sonnet:beta</option>
186- <option data-type="openrouter" value="anthropic/claude-3-haiku:beta">anthropic/claude-3-haiku:beta</option>
187- <option data-type="openrouter" value="anthropic/claude-3-sonnet:beta">anthropic/claude-3-sonnet:beta</option>
188- <option data-type="openrouter" value="anthropic/claude-3-opus:beta">anthropic/claude-3-opus:beta</option>
189- <option data-type="openrouter" value="nousresearch/nous-hermes-2-vision-7b">nousresearch/nous-hermes-2-vision-7b</option>
190- <option data-type="openrouter" value="google/gemini-flash-8b-1.5-exp">google/gemini-flash-8b-1.5-exp</option>
191- <option data-type="openrouter" value="google/gemini-flash-1.5">google/gemini-flash-1.5</option>
192- <option data-type="openrouter" value="google/gemini-flash-1.5-exp">google/gemini-flash-1.5-exp</option>
193- <option data-type="openrouter" value="google/gemini-pro-1.5">google/gemini-pro-1.5</option>
194- <option data-type="openrouter" value="google/gemini-pro-1.5-exp">google/gemini-pro-1.5-exp</option>
195- <option data-type="openrouter" value="google/gemini-pro-vision">google/gemini-pro-vision</option>
196- <option data-type="openrouter" value="liuhaotian/llava-yi-34b">liuhaotian/llava-yi-34b</option>
197154 <option data-type="ollama" value="ollama_current" data-i18n="currently_selected">[Currently selected]</option>
198155 <option data-type="ollama" value="bakllava">bakllava</option>
199156 <option data-type="ollama" value="llava">llava</option>
@@ -208,18 +165,6 @@
208165 <option data-type="xai" value="grok-4-0709">grok-4-0709</option>
209166 <option data-type="xai" value="grok-2-vision-1212">grok-2-vision-1212</option>
210167 <option data-type="xai" value="grok-vision-beta">grok-vision-beta</option>
211- <option data-type="pollinations" value="openai">openai</option>
212- <option data-type="pollinations" value="openai-fast">openai-fast</option>
213- <option data-type="pollinations" value="openai-large">openai-large</option>
214- <option data-type="pollinations" value="openai-roblox">openai-roblox</option>
215- <option data-type="pollinations" value="mistral">mistral</option>
216- <option data-type="pollinations" value="unity">unity</option>
217- <option data-type="pollinations" value="mirexa">mirexa</option>
218- <option data-type="pollinations" value="searchgpt">searchgpt</option>
219- <option data-type="pollinations" value="evil">evil</option>
220- <option data-type="pollinations" value="phi">phi</option>
221- <option data-type="pollinations" value="sur">sur</option>
222- <option data-type="pollinations" value="bidara">bidara</option>
223168 </select>
224169 </div>
225170 <div data-type="ollama">
src/endpoints/backends/chat-completions.js+54 -0
@@ -1835,3 +1835,57 @@ router.post('/generate', function (request, response) {
18351835 }
18361836 }
18371837});
1838+
1839+const pollinations = express.Router();
1840+
1841+pollinations.post('/models/multimodal', async (_req, res) => {
1842+ try {
1843+ const response = await fetch('https://text.pollinations.ai/models');
1844+
1845+ if (!response.ok) {
1846+ return res.json([]);
1847+ }
1848+
1849+ /** @type {any} */
1850+ const data = await response.json();
1851+
1852+ if (!Array.isArray(data)) {
1853+ return res.json([]);
1854+ }
1855+
1856+ const multimodalModels = data.filter(m => m?.vision).map(m => m.name);
1857+ return res.json(multimodalModels);
1858+ } catch (error) {
1859+ console.error(error);
1860+ return res.sendStatus(500);
1861+ }
1862+});
1863+
1864+router.use('/pollinations', pollinations);
1865+
1866+const aimlapi = express.Router();
1867+
1868+aimlapi.post('/models/multimodal', async (_req, res) => {
1869+ try {
1870+ const response = await fetch('https://api.aimlapi.com/v1/models');
1871+
1872+ if (!response.ok) {
1873+ return res.json([]);
1874+ }
1875+
1876+ /** @type {any} */
1877+ const data = await response.json();
1878+
1879+ if (!Array.isArray(data?.data)) {
1880+ return res.json([]);
1881+ }
1882+
1883+ const multimodalModels = data.data.filter(m => m?.features?.includes('openai/chat-completion.vision')).map(m => m.id);
1884+ return res.json(multimodalModels);
1885+ } catch (error) {
1886+ console.error(error);
1887+ return res.sendStatus(500);
1888+ }
1889+});
1890+
1891+router.use('/aimlapi', aimlapi);