Captioning: Load external model lists for OpenRouter, AIML and Pollinations

ad960197b725d6857fb99b34a5368130400206ec

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: ad516c864ace0b2efd4eba000f3cee30b04c2c9c
3 files changed, +78 -68Showing whitespace changes
public/scripts/extensions/caption/index.js+23 -12
@@ -488,8 +488,13 @@ jQuery(async function () {
488 $('#img_file').on('change', (e) => onSelectImage(e.originalEvent, '', false));488 $('#img_file').on('change', (e) => onSelectImage(e.originalEvent, '', false));
489 }489 }
490 async function switchMultimodalBlocks() {490 async function switchMultimodalBlocks() {
491 await addOpenRouterModels();491 await addRemoteEndpointModels();
492 const isMultimodal = extension_settings.caption.source === 'multimodal';492 const isMultimodal = extension_settings.caption.source === 'multimodal';
493 if (!extension_settings.caption.multimodal_model) {
494 const dropdown = $('#caption_multimodal_model');
495 const options = dropdown.find(`option[data-type="${extension_settings.caption.multimodal_api}"]`);
496 extension_settings.caption.multimodal_model = String(options.first().val());
497 }
493 $('#caption_multimodal_block').toggle(isMultimodal);498 $('#caption_multimodal_block').toggle(isMultimodal);
494 $('#caption_prompt_block').toggle(isMultimodal);499 $('#caption_prompt_block').toggle(isMultimodal);
495 $('#caption_multimodal_api').val(extension_settings.caption.multimodal_api);500 $('#caption_multimodal_api').val(extension_settings.caption.multimodal_api);
@@ -504,16 +509,18 @@ jQuery(async function () {
504 const html = await renderExtensionTemplateAsync('caption', 'settings', { TEMPLATE_DEFAULT, PROMPT_DEFAULT });509 const html = await renderExtensionTemplateAsync('caption', 'settings', { TEMPLATE_DEFAULT, PROMPT_DEFAULT });
505 $('#caption_container').append(html);510 $('#caption_container').append(html);
506 }511 }
507 async function addOpenRouterModels() {512
513 async function addRemoteEndpointModels() {
514 async function processEndpoint(api, url) {
508 const dropdown = document.getElementById('caption_multimodal_model');515 const dropdown = document.getElementById('caption_multimodal_model');
509 if (!(dropdown instanceof HTMLSelectElement)) {516 if (!(dropdown instanceof HTMLSelectElement)) {
510 return;517 return;
511 }518 }
512 if (extension_settings.caption.source !== 'multimodal' || extension_settings.caption.multimodal_api !== 'openrouter') {519 if (extension_settings.caption.source !== 'multimodal' || extension_settings.caption.multimodal_api !== api) {
513 return;520 return;
514 }521 }
515 const options = Array.from(dropdown.options);522 const options = Array.from(dropdown.options);
516 const response = await fetch('/api/openrouter/models/multimodal', {523 const response = await fetch(url, {
517 method: 'POST',524 method: 'POST',
518 headers: getRequestHeaders(),525 headers: getRequestHeaders(),
519 });526 });
@@ -522,19 +529,24 @@ jQuery(async function () {
522 }529 }
523 const modelIds = await response.json();530 const modelIds = await response.json();
524 if (Array.isArray(modelIds) && modelIds.length > 0) {531 if (Array.isArray(modelIds) && modelIds.length > 0) {
525 modelIds.forEach((modelId) => {532 modelIds.sort().forEach((modelId) => {
526 if (!modelId || typeof modelId !== 'string' || options.some(o => o.value === modelId)) {533 if (!modelId || typeof modelId !== 'string' || options.some(o => o.value === modelId)) {
527 return;534 return;
528 }535 }
529 const option = document.createElement('option');536 const option = document.createElement('option');
530 option.value = modelId;537 option.value = modelId;
531 option.textContent = modelId;538 option.textContent = modelId;
532 option.dataset.type = 'openrouter';539 option.dataset.type = api;
533 dropdown.add(option);540 dropdown.add(option);
534 });541 });
535 }542 }
536 }543 }
537544
545 await processEndpoint('openrouter', '/api/openrouter/models/multimodal');
546 await processEndpoint('aimlapi', '/api/backends/chat-completions/aimlapi/models/multimodal');
547 await processEndpoint('pollinations', '/api/backends/chat-completions/pollinations/models/multimodal');
548 }
549
538 await addSettings();550 await addSettings();
539 addPictureSendForm();551 addPictureSendForm();
540 addSendPictureButton();552 addSendPictureButton();
@@ -550,9 +562,9 @@ jQuery(async function () {
550 $('#caption_prompt').val(extension_settings.caption.prompt);562 $('#caption_prompt').val(extension_settings.caption.prompt);
551 $('#caption_template').val(extension_settings.caption.template);563 $('#caption_template').val(extension_settings.caption.template);
552 $('#caption_refine_mode').on('input', onRefineModeInput);564 $('#caption_refine_mode').on('input', onRefineModeInput);
553 $('#caption_source').on('change', () => {565 $('#caption_source').on('change', async () => {
554 extension_settings.caption.source = String($('#caption_source').val());566 extension_settings.caption.source = String($('#caption_source').val());
555 switchMultimodalBlocks();567 await switchMultimodalBlocks();
556 saveSettingsDebounced();568 saveSettingsDebounced();
557 });569 });
558 $('#caption_prompt').on('input', () => {570 $('#caption_prompt').on('input', () => {
@@ -581,13 +593,12 @@ jQuery(async function () {
581 $('#ollama_download_model').trigger('click');593 $('#ollama_download_model').trigger('click');
582 $('#dialogue_popup_input').val(presetModel);594 $('#dialogue_popup_input').val(presetModel);
583 });595 });
584 $('#caption_multimodal_api').on('change', () => {596 $('#caption_multimodal_api').on('change', async () => {
585 const api = String($('#caption_multimodal_api').val());597 const api = String($('#caption_multimodal_api').val());
586 const model = String($(`#caption_multimodal_model option[data-type="${api}"]`).first().val());
587 extension_settings.caption.multimodal_api = api;598 extension_settings.caption.multimodal_api = api;
588 extension_settings.caption.multimodal_model = model;599 extension_settings.caption.multimodal_model = '';
600 await switchMultimodalBlocks();
589 saveSettingsDebounced();601 saveSettingsDebounced();
590 switchMultimodalBlocks();
591 });602 });
592 $('#caption_multimodal_model').on('change', () => {603 $('#caption_multimodal_model').on('change', () => {
593 extension_settings.caption.multimodal_model = String($('#caption_multimodal_model').val());604 extension_settings.caption.multimodal_model = String($('#caption_multimodal_model').val());
public/scripts/extensions/caption/settings.html+1 -56
@@ -40,6 +40,7 @@
40 <div class="flex1 flex-container flexFlowColumn flexNoGap">40 <div class="flex1 flex-container flexFlowColumn flexNoGap">
41 <label for="caption_multimodal_model" data-i18n="Model">Model</label>41 <label for="caption_multimodal_model" data-i18n="Model">Model</label>
42 <select id="caption_multimodal_model" class="flex1 text_pole">42 <select id="caption_multimodal_model" class="flex1 text_pole">
43 <!-- AI/ML API, OpenRouter, Pollinations are added externally by JavaScript -->
43 <option data-type="cohere" value="c4ai-aya-vision-8b">c4ai-aya-vision-8b</option>44 <option data-type="cohere" value="c4ai-aya-vision-8b">c4ai-aya-vision-8b</option>
44 <option data-type="cohere" value="c4ai-aya-vision-32b">c4ai-aya-vision-32b</option>45 <option data-type="cohere" value="c4ai-aya-vision-32b">c4ai-aya-vision-32b</option>
45 <option data-type="cohere" value="command-a-vision-07-2025">command-a-vision-07-2025</option>46 <option data-type="cohere" value="command-a-vision-07-2025">command-a-vision-07-2025</option>
@@ -150,50 +151,6 @@
150 <option data-type="groq" value="llama-3.2-11b-vision-preview">llama-3.2-11b-vision-preview</option>151 <option data-type="groq" value="llama-3.2-11b-vision-preview">llama-3.2-11b-vision-preview</option>
151 <option data-type="groq" value="llama-3.2-90b-vision-preview">llama-3.2-90b-vision-preview</option>152 <option data-type="groq" value="llama-3.2-90b-vision-preview">llama-3.2-90b-vision-preview</option>
152 <option data-type="groq" value="llava-v1.5-7b-4096-preview">llava-v1.5-7b-4096-preview</option>153 <option data-type="groq" value="llava-v1.5-7b-4096-preview">llava-v1.5-7b-4096-preview</option>
153 <option data-type="aimlapi" value="gpt-4o-2024-05-13">gpt-4o-2024-05-13</option>
154 <option data-type="aimlapi" value="gpt-4o-2024-08-06">gpt-4o-2024-08-06</option>
155 <option data-type="aimlapi" value="gpt-4o">gpt-4o</option>
156 <option data-type="aimlapi" value="gpt-4o-mini">gpt-4o-mini</option>
157 <option data-type="aimlapi" value="gpt-4o-mini-2024-07-18">gpt-4o-mini-2024-07-18</option>
158 <option data-type="aimlapi" value="chatgpt-4o-latest">chatgpt-4o-latest</option>
159 <option data-type="aimlapi" value="gpt-4-turbo">gpt-4-turbo</option>
160 <option data-type="aimlapi" value="claude-3-opus-20240229">claude-3-opus-20240229</option>
161 <option data-type="aimlapi" value="claude-3-sonnet-20240229">claude-3-sonnet-20240229</option>
162 <option data-type="aimlapi" value="claude-3-haiku-20240307">claude-3-haiku-20240307</option>
163 <option data-type="aimlapi" value="claude-3-5-sonnet-20240620">claude-3-5-sonnet-20240620</option>
164 <option data-type="aimlapi" value="google/gemini-1.5-flash">google/gemini-1.5-flash</option>
165 <option data-type="aimlapi" value="google/gemini-1.5-pro">google/gemini-1.5-pro</option>
166 <option data-type="aimlapi" value="google/gemini-2.0-flash">google/gemini-2.0-flash</option>
167 <option data-type="aimlapi" value="google/gemini-2.0-flash-exp">google/gemini-2.0-flash-exp</option>
168 <option data-type="aimlapi" value="google/gemini-2.5-flash-preview">google/gemini-2.5-flash-preview</option>
169 <option data-type="aimlapi" value="meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo">meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo</option>
170 <option data-type="aimlapi" value="meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo">meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo</option>
171 <option data-type="openrouter" value="openai/gpt-4-vision-preview">openai/gpt-4-vision-preview</option>
172 <option data-type="openrouter" value="openai/gpt-4o">openai/gpt-4o</option>
173 <option data-type="openrouter" value="openai/gpt-4o-2024-05-13">openai/gpt-4o-2024-05-13</option>
174 <option data-type="openrouter" value="openai/gpt-4o-2024-08-06">openai/gpt-4o-2024-08-06</option>
175 <option data-type="openrouter" value="openai/gpt-4-turbo">openai/gpt-4-turbo</option>
176 <option data-type="openrouter" value="openai/gpt-4o-mini">openai/gpt-4o-mini</option>
177 <option data-type="openrouter" value="openai/gpt-4o-mini-2024-07-18">openai/gpt-4o-mini-2024-07-18</option>
178 <option data-type="openrouter" value="openai/chatgpt-4o-latest">openai/chatgpt-4o-latest</option>
179 <option data-type="openrouter" value="haotian-liu/llava-13b">haotian-liu/llava-13b</option>
180 <option data-type="openrouter" value="fireworks/firellava-13b">fireworks/firellava-13b</option>
181 <option data-type="openrouter" value="anthropic/claude-3.5-sonnet">anthropic/claude-3.5-sonnet</option>
182 <option data-type="openrouter" value="anthropic/claude-3-haiku">anthropic/claude-3-haiku</option>
183 <option data-type="openrouter" value="anthropic/claude-3-sonnet">anthropic/claude-3-sonnet</option>
184 <option data-type="openrouter" value="anthropic/claude-3-opus">anthropic/claude-3-opus</option>
185 <option data-type="openrouter" value="anthropic/claude-3.5-sonnet:beta">anthropic/claude-3.5-sonnet:beta</option>
186 <option data-type="openrouter" value="anthropic/claude-3-haiku:beta">anthropic/claude-3-haiku:beta</option>
187 <option data-type="openrouter" value="anthropic/claude-3-sonnet:beta">anthropic/claude-3-sonnet:beta</option>
188 <option data-type="openrouter" value="anthropic/claude-3-opus:beta">anthropic/claude-3-opus:beta</option>
189 <option data-type="openrouter" value="nousresearch/nous-hermes-2-vision-7b">nousresearch/nous-hermes-2-vision-7b</option>
190 <option data-type="openrouter" value="google/gemini-flash-8b-1.5-exp">google/gemini-flash-8b-1.5-exp</option>
191 <option data-type="openrouter" value="google/gemini-flash-1.5">google/gemini-flash-1.5</option>
192 <option data-type="openrouter" value="google/gemini-flash-1.5-exp">google/gemini-flash-1.5-exp</option>
193 <option data-type="openrouter" value="google/gemini-pro-1.5">google/gemini-pro-1.5</option>
194 <option data-type="openrouter" value="google/gemini-pro-1.5-exp">google/gemini-pro-1.5-exp</option>
195 <option data-type="openrouter" value="google/gemini-pro-vision">google/gemini-pro-vision</option>
196 <option data-type="openrouter" value="liuhaotian/llava-yi-34b">liuhaotian/llava-yi-34b</option>
197 <option data-type="ollama" value="ollama_current" data-i18n="currently_selected">[Currently selected]</option>154 <option data-type="ollama" value="ollama_current" data-i18n="currently_selected">[Currently selected]</option>
198 <option data-type="ollama" value="bakllava">bakllava</option>155 <option data-type="ollama" value="bakllava">bakllava</option>
199 <option data-type="ollama" value="llava">llava</option>156 <option data-type="ollama" value="llava">llava</option>
@@ -208,18 +165,6 @@
208 <option data-type="xai" value="grok-4-0709">grok-4-0709</option>165 <option data-type="xai" value="grok-4-0709">grok-4-0709</option>
209 <option data-type="xai" value="grok-2-vision-1212">grok-2-vision-1212</option>166 <option data-type="xai" value="grok-2-vision-1212">grok-2-vision-1212</option>
210 <option data-type="xai" value="grok-vision-beta">grok-vision-beta</option>167 <option data-type="xai" value="grok-vision-beta">grok-vision-beta</option>
211 <option data-type="pollinations" value="openai">openai</option>
212 <option data-type="pollinations" value="openai-fast">openai-fast</option>
213 <option data-type="pollinations" value="openai-large">openai-large</option>
214 <option data-type="pollinations" value="openai-roblox">openai-roblox</option>
215 <option data-type="pollinations" value="mistral">mistral</option>
216 <option data-type="pollinations" value="unity">unity</option>
217 <option data-type="pollinations" value="mirexa">mirexa</option>
218 <option data-type="pollinations" value="searchgpt">searchgpt</option>
219 <option data-type="pollinations" value="evil">evil</option>
220 <option data-type="pollinations" value="phi">phi</option>
221 <option data-type="pollinations" value="sur">sur</option>
222 <option data-type="pollinations" value="bidara">bidara</option>
223 </select>168 </select>
224 </div>169 </div>
225 <div data-type="ollama">170 <div data-type="ollama">
src/endpoints/backends/chat-completions.js+54 -0
@@ -1835,3 +1835,57 @@ router.post('/generate', function (request, response) {
1835 }1835 }
1836 }1836 }
1837});1837});
1838
1839const pollinations = express.Router();
1840
1841pollinations.post('/models/multimodal', async (_req, res) => {
1842 try {
1843 const response = await fetch('https://text.pollinations.ai/models');
1844
1845 if (!response.ok) {
1846 return res.json([]);
1847 }
1848
1849 /** @type {any} */
1850 const data = await response.json();
1851
1852 if (!Array.isArray(data)) {
1853 return res.json([]);
1854 }
1855
1856 const multimodalModels = data.filter(m => m?.vision).map(m => m.name);
1857 return res.json(multimodalModels);
1858 } catch (error) {
1859 console.error(error);
1860 return res.sendStatus(500);
1861 }
1862});
1863
1864router.use('/pollinations', pollinations);
1865
1866const aimlapi = express.Router();
1867
1868aimlapi.post('/models/multimodal', async (_req, res) => {
1869 try {
1870 const response = await fetch('https://api.aimlapi.com/v1/models');
1871
1872 if (!response.ok) {
1873 return res.json([]);
1874 }
1875
1876 /** @type {any} */
1877 const data = await response.json();
1878
1879 if (!Array.isArray(data?.data)) {
1880 return res.json([]);
1881 }
1882
1883 const multimodalModels = data.data.filter(m => m?.features?.includes('openai/chat-completion.vision')).map(m => m.id);
1884 return res.json(multimodalModels);
1885 } catch (error) {
1886 console.error(error);
1887 return res.sendStatus(500);
1888 }
1889});
1890
1891router.use('/aimlapi', aimlapi);