Auto-add OpenRouter captioning models

6393eda81ee991290ac05521ed59dbf2fa8825eb

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: 44cc81fa73766155eeed9cf3874c268bb5439fcb

public/scripts/extensions/caption/index.js+54 -22 Viewed

 import { ensureImageFormatSupported, getBase64Async, isTrueBoolean, saveBase64AsFile } from '../../utils.js';
 import { getContext, getApiUrl, doExtrasFetch, extension_settings, modules, renderExtensionTemplateAsync } from '../../extensions.js';
 import { appendMediaToMessage, callPopup, eventSource, event_types, getRequestHeaders, main_api, saveChatConditional, saveSettingsDebounced, substituteParamsExtended } from '../../../script.js';
 import { getMessageTimeStamp } from '../../RossAscends-mods.js';
 import { SECRET_KEYS, secret_state } from '../../secrets.js';
 import { getMultimodalCaption } from '../shared.js';

         $('#form_sheld').append(imgForm);
         $('#img_file').on('change', (e) => onSelectImage(e.originalEvent, '', false));
+    }
     async function switchMultimodalBlocks() {
 +        await addOpenRouterModels();
         const isMultimodal = extension_settings.caption.source === 'multimodal';
 -        $('#caption_ollama_pull').on('click', (e) => {
 -            const presetModel = extension_settings.caption.multimodal_model !== 'ollama_current' ? extension_settings.caption.multimodal_model : '';
 -            e.preventDefault();
 -            $('#ollama_download_model').trigger('click');
 -            $('#dialogue_popup_input').val(presetModel);
 -        });
         $('#caption_multimodal_block').toggle(isMultimodal);
         $('#caption_prompt_block').toggle(isMultimodal);
         $('#caption_multimodal_api').val(extension_settings.caption.multimodal_api);

             const types = type.split(',');
             $(this).toggle(types.includes(extension_settings.caption.multimodal_api));
         });
 -        $('#caption_multimodal_api').on('change', () => {
 -            const api = String($('#caption_multimodal_api').val());
 -            const model = String($(`#caption_multimodal_model option[data-type="${api}"]`).first().val());
 -            extension_settings.caption.multimodal_api = api;
 -            extension_settings.caption.multimodal_model = model;
 -            saveSettingsDebounced();
 -            switchMultimodalBlocks();
 -        });
 -        $('#caption_multimodal_model').on('change', () => {
 -            extension_settings.caption.multimodal_model = String($('#caption_multimodal_model').val());
 -            saveSettingsDebounced();
 -        });
+    }
     async function addSettings() {
         const html = await renderExtensionTemplateAsync('caption', 'settings', { TEMPLATE_DEFAULT, PROMPT_DEFAULT });
         $('#caption_container').append(html);
+    }
 +    async function addOpenRouterModels() {
 +        const dropdown = document.getElementById('caption_multimodal_model');
 +        if (!(dropdown instanceof HTMLSelectElement)) {
 +            return;
 +        }
 +        if (extension_settings.caption.source !== 'multimodal' || extension_settings.caption.multimodal_api !== 'openrouter') {
 +            return;
 +        }
 +        const options = Array.from(dropdown.options);
 +        const response = await fetch('/api/openrouter/models/multimodal', {
 +            method: 'POST',
 +            headers: getRequestHeaders(),
 +        });
 +        if (!response.ok) {
 +            return;
 +        }
 +        const modelIds = await response.json();
 +        if (Array.isArray(modelIds) && modelIds.length > 0) {
 +            modelIds.forEach((modelId) => {
 +                if (!modelId || typeof modelId !== 'string' || options.some(o => o.value === modelId)) {
 +                    return;
 +                }
 +                const option = document.createElement('option');
 +                option.value = modelId;
 +                option.textContent = modelId;
 +                option.dataset.type = 'openrouter';
 +                dropdown.add(option);
 +            });
 +        }
 +    }
     await addSettings();
     addPictureSendForm();
     addSendPictureButton();
     setImageIcon();
     migrateSettings();
     await switchMultimodalBlocks();
     $('#caption_refine_mode').prop('checked', !!(extension_settings.caption.refine_mode));
     $('#caption_allow_reverse_proxy').prop('checked', !!(extension_settings.caption.allow_reverse_proxy));

         extension_settings.caption.auto_mode = !!$('#caption_auto_mode').prop('checked');
         saveSettingsDebounced();
     });
 +    $('#caption_ollama_pull').on('click', (e) => {
 +        const presetModel = extension_settings.caption.multimodal_model !== 'ollama_current' ? extension_settings.caption.multimodal_model : '';
 +        e.preventDefault();
 +        $('#ollama_download_model').trigger('click');
 +        $('#dialogue_popup_input').val(presetModel);
 +    });
 +    $('#caption_multimodal_api').on('change', () => {
 +        const api = String($('#caption_multimodal_api').val());
 +        const model = String($(`#caption_multimodal_model option[data-type="${api}"]`).first().val());
 +        extension_settings.caption.multimodal_api = api;
 +        extension_settings.caption.multimodal_model = model;
 +        saveSettingsDebounced();
 +        switchMultimodalBlocks();
 +    });
 +    $('#caption_multimodal_model').on('change', () => {
 +        extension_settings.caption.multimodal_model = String($('#caption_multimodal_model').val());
 +        saveSettingsDebounced();
 +    });
     const onMessageEvent = async (index) => {
         if (!extension_settings.caption.auto_mode) {

+        }
     });
     SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'caption',
 +        name: 'caption',
         callback: captionCommandCallback,
         returns: 'caption',
         namedArgumentList: [

server.js+3 -0 Viewed

 // Ooba/OpenAI text completions
 app.use('/api/backends/text-completions', require('./src/endpoints/backends/text-completions').router);
 +// OpenRouter
 +app.use('/api/openrouter', require('./src/endpoints/openrouter').router);
++
 // KoboldAI
 app.use('/api/backends/kobold', require('./src/endpoints/backends/kobold').router);

src/endpoints/openrouter.js+32 -0 Viewed

 +const express = require('express');
 +const { jsonParser } = require('../express-common');
++
 +const router = express.Router();
 +const API_OPENROUTER = 'https://openrouter.ai/api/v1';
++
 +router.post('/models/multimodal', jsonParser, async (_req, res) => {
 +    try {
 +        // The endpoint is available without authentication
 +        const response = await fetch(`${API_OPENROUTER}/models`, {
 +            method: 'GET',
 +            headers: {
 +                'Accept': 'application/json',
 +            },
 +        });
++
 +        if (!response.ok) {
 +            return res.json([]);
 +        }
++
 +        const data = await response.json();
 +        const models = data?.data || [];
 +        const multimodalModels = models.filter(m => m?.architecture?.modality === 'text+image->text').map(m => m.id);
++
 +        return res.json(multimodalModels);
 +    } catch (error) {
 +        console.error(error);
 +        return res.sendStatus(500);
 +    }
 +});
++
 +module.exports = { router };