Merge pull request #2654 from ayancey/openai-compatible-tts OpenAI Compatible API TTS Provider

33916aa493399bebe2d3e327ce91c5abdae295aa

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

Parents: 406acb4312a2da30ea6e282b085112eed62653e5 d0b368f29cc4f11b3c2dd4d6c2a896f0b70f42de

public/scripts/extensions/stable-diffusion/index.js+16 -5 Viewed

 import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from '../../slash-commands/SlashCommandArgument.js';
 import { debounce_timeout } from '../../constants.js';
 import { SlashCommandEnumValue } from '../../slash-commands/SlashCommandEnumValue.js';
-import { POPUP_TYPE, Popup, callGenericPopup } from '../../popup.js';
+import { POPUP_RESULT, POPUP_TYPE, Popup, callGenericPopup } from '../../popup.js';
 export { MODULE_NAME };
 const MODULE_NAME = 'sd';

 async function onStabilityKeyClick() {
     const popupText = 'Stability AI API Key:';
-    const key = await callGenericPopup(popupText, POPUP_TYPE.INPUT);
+    const key = await callGenericPopup(popupText, POPUP_TYPE.INPUT, '', {
+        customButtons: [{
+            text: 'Remove Key',
+            appendAtEnd: true,
+            result: POPUP_RESULT.NEGATIVE,
+            action: async () => {
+                await writeSecret(SECRET_KEYS.STABILITY, '');
+                toastr.success('API Key removed');
+                await loadSettingOptions();
+            },
+        }],
+    });
     if (!key) {
         return;

     if (result.ok) {
         const data = await result.json();
         // Default format is 'jpg'
         let format = 'jpg';
         // Check if a format is specified in the result
         if (data.format) {
             format = data.format.toLowerCase();
+        }
         return { format: format, data: data.images[0] };
     } else {
         const text = await result.text();

public/scripts/extensions/translate/index.js+13 -2 Viewed

     updateMessageBlock,
 } from '../../../script.js';
 import { extension_settings, getContext, renderExtensionTemplateAsync } from '../../extensions.js';
-import { POPUP_TYPE, callGenericPopup } from '../../popup.js';
+import { POPUP_RESULT, POPUP_TYPE, callGenericPopup } from '../../popup.js';
 import { findSecret, secret_state, writeSecret } from '../../secrets.js';
 import { SlashCommand } from '../../slash-commands/SlashCommand.js';
 import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from '../../slash-commands/SlashCommandArgument.js';

         const secretKey = extension_settings.translate.provider + '_url';
         const savedUrl = secret_state[secretKey] ? await findSecret(secretKey) : '';
-        const url = await callGenericPopup(popupText, POPUP_TYPE.INPUT, savedUrl);
+        const url = await callGenericPopup(popupText, POPUP_TYPE.INPUT, savedUrl,{
+            customButtons: [{
+                text: 'Remove URL',
+                appendAtEnd: true,
+                result: POPUP_RESULT.NEGATIVE,
+                action: async () => {
+                    await writeSecret(secretKey, '');
+                    toastr.success('API URL removed');
+                    $('#translate_url_button').toggleClass('success', !!secret_state[secretKey]);
+                },
+            }],
+        });
         if (url == false || url == '') {
             return;

public/scripts/extensions/tts/azure.js+14 -2 Viewed

 import { getRequestHeaders } from '../../../script.js';
-import { POPUP_TYPE, callGenericPopup } from '../../popup.js';
+import { POPUP_RESULT, POPUP_TYPE, callGenericPopup } from '../../popup.js';
 import { SECRET_KEYS, findSecret, secret_state, writeSecret } from '../../secrets.js';
 import { getPreviewString, saveTtsProviderSettings } from './index.js';
 export { AzureTtsProvider };

             const popupText = 'Azure TTS API Key';
             const savedKey = secret_state[SECRET_KEYS.AZURE_TTS] ? await findSecret(SECRET_KEYS.AZURE_TTS) : '';
-            const key = await callGenericPopup(popupText, POPUP_TYPE.INPUT, savedKey);
+            const key = await callGenericPopup(popupText, POPUP_TYPE.INPUT, savedKey, {
+                customButtons: [{
+                    text: 'Remove Key',
+                    appendAtEnd: true,
+                    result: POPUP_RESULT.NEGATIVE,
+                    action: async () => {
+                        await writeSecret(SECRET_KEYS.AZURE_TTS, '');
+                        $('#azure_tts_key').toggleClass('success', secret_state[SECRET_KEYS.AZURE_TTS]);
+                        toastr.success('API Key removed');
+                        await this.onRefreshClick();
+                    },
+                }],
+            });
             if (key == false || key == '') {
                 return;

public/scripts/extensions/tts/index.js+11 -9 Viewed

 import { NovelTtsProvider } from './novel.js';
 import { power_user } from '../../power-user.js';
 import { OpenAITtsProvider } from './openai.js';
+import { OpenAICompatibleTtsProvider } from './openai-compatible.js';
 import { XTTSTtsProvider } from './xtts.js';
 import { VITSTtsProvider } from './vits.js';
 import { GSVITtsProvider } from './gsvi.js';

+}
 const ttsProviders = {
-    ElevenLabs: ElevenLabsTtsProvider,
+    AllTalk: AllTalkTtsProvider,
-    Silero: SileroTtsProvider,
+    Azure: AzureTtsProvider,
-    XTTSv2: XTTSTtsProvider,
-    VITS: VITSTtsProvider,
-    GSVI: GSVITtsProvider,
-    SBVits2: SBVits2TtsProvider,
-    System: SystemTtsProvider,
     Coqui: CoquiTtsProvider,
     Edge: EdgeTtsProvider,
+    ElevenLabs: ElevenLabsTtsProvider,
+    GSVI: GSVITtsProvider,
     Novel: NovelTtsProvider,
     OpenAI: OpenAITtsProvider,
-    AllTalk: AllTalkTtsProvider,
+    'OpenAI Compatible': OpenAICompatibleTtsProvider,
+    SBVits2: SBVits2TtsProvider,
+    Silero: SileroTtsProvider,
     SpeechT5: SpeechT5TtsProvider,
-    Azure: AzureTtsProvider,
+    System: SystemTtsProvider,
+    VITS: VITSTtsProvider,
+    XTTSv2: XTTSTtsProvider,
 };
 let ttsProvider;
 let ttsProviderName;

public/scripts/extensions/tts/openai-compatible.js+193 -0 Viewed

+import { getRequestHeaders } from '../../../script.js';
+import { callGenericPopup, POPUP_RESULT, POPUP_TYPE } from '../../popup.js';
+import { findSecret, SECRET_KEYS, secret_state, writeSecret } from '../../secrets.js';
+import { getPreviewString, saveTtsProviderSettings } from './index.js';
+export { OpenAICompatibleTtsProvider };
+class OpenAICompatibleTtsProvider {
+    settings;
+    voices = [];
+    separator = ' . ';
+    audioElement = document.createElement('audio');
+    defaultSettings = {
+        voiceMap: {},
+        model: 'tts-1',
+        speed: 1,
+        available_voices: ['alloy', 'echo', 'fable', 'onyx', 'nova', 'shimmer'],
+        provider_endpoint: 'http://127.0.0.1:8000/v1/audio/speech',
+    };
+    get settingsHtml() {
+        let html = `
+        <label for="openai_compatible_tts_endpoint">Provider Endpoint:</label>
+        <div class="flex-container alignItemsCenter">
+            <div class="flex1">
+                <input id="openai_compatible_tts_endpoint" type="text" class="text_pole" maxlength="250" value="${this.defaultSettings.provider_endpoint}"/>
+            </div>
+            <div id="openai_compatible_tts_key" class="menu_button menu_button_icon">
+                <i class="fa-solid fa-key"></i>
+                <span>API Key</span>
+            </div>
+        </div>
+        <label for="openai_compatible_model">Model:</label>
+        <input id="openai_compatible_model" type="text" class="text_pole" maxlength="250" value="${this.defaultSettings.model}"/>
+        <label for="openai_compatible_tts_voices">Available Voices (comma separated):</label>
+        <input id="openai_compatible_tts_voices" type="text" class="text_pole" maxlength="250" value="${this.defaultSettings.available_voices.join()}"/>
+        <label for="openai_compatible_tts_speed">Speed: <span id="openai_compatible_tts_speed_output"></span></label>
+        <input type="range" id="openai_compatible_tts_speed" value="1" min="0.25" max="4" step="0.05">`;
+        return html;
+    }
+    async loadSettings(settings) {
+        // Populate Provider UI given input settings
+        if (Object.keys(settings).length == 0) {
+            console.info('Using default TTS Provider settings');
+        }
+        // Only accept keys defined in defaultSettings
+        this.settings = this.defaultSettings;
+        for (const key in settings) {
+            if (key in this.settings) {
+                this.settings[key] = settings[key];
+            } else {
+                throw `Invalid setting passed to TTS Provider: ${key}`;
+            }
+        }
+        $('#openai_compatible_tts_endpoint').val(this.settings.provider_endpoint);
+        $('#openai_compatible_tts_endpoint').on('input', () => { this.onSettingsChange(); });
+        $('#openai_compatible_model').val(this.defaultSettings.model);
+        $('#openai_compatible_model').on('input', () => { this.onSettingsChange(); });
+        $('#openai_compatible_tts_voices').val(this.settings.available_voices.join());
+        $('#openai_compatible_tts_voices').on('input', () => { this.onSettingsChange(); });
+        $('#openai_compatible_tts_speed').val(this.settings.speed);
+        $('#openai_compatible_tts_speed').on('input', () => {
+            this.onSettingsChange();
+        });
+        $('#openai_compatible_tts_speed_output').text(this.settings.speed);
+        $('#openai_compatible_tts_key').toggleClass('success', secret_state[SECRET_KEYS.CUSTOM_OPENAI_TTS]);
+        $('#openai_compatible_tts_key').on('click', async () => {
+            const popupText = 'OpenAI-compatible TTS API Key';
+            const savedKey = secret_state[SECRET_KEYS.CUSTOM_OPENAI_TTS] ? await findSecret(SECRET_KEYS.CUSTOM_OPENAI_TTS) : '';
+            const key = await callGenericPopup(popupText, POPUP_TYPE.INPUT, savedKey, {
+                customButtons: [{
+                    text: 'Remove Key',
+                    appendAtEnd: true,
+                    result: POPUP_RESULT.NEGATIVE,
+                    action: async () => {
+                        await writeSecret(SECRET_KEYS.CUSTOM_OPENAI_TTS, '');
+                        $('#openai_compatible_tts_key').toggleClass('success', secret_state[SECRET_KEYS.CUSTOM_OPENAI_TTS]);
+                        toastr.success('API Key removed');
+                        await this.onRefreshClick();
+                    },
+                }],
+            });
+            if (key == false || key == '') {
+                return;
+            }
+            await writeSecret(SECRET_KEYS.CUSTOM_OPENAI_TTS, String(key));
+            toastr.success('API Key saved');
+            $('#openai_compatible_tts_key').toggleClass('success', secret_state[SECRET_KEYS.CUSTOM_OPENAI_TTS]);
+            await this.onRefreshClick();
+        });
+        await this.checkReady();
+        console.debug('OpenAI Compatible TTS: Settings loaded');
+    }
+    onSettingsChange() {
+        // Update dynamically
+        this.settings.provider_endpoint = String($('#openai_compatible_tts_endpoint').val());
+        this.settings.model = String($('#openai_compatible_model').val());
+        this.settings.available_voices = String($('#openai_compatible_tts_voices').val()).split(',');
+        this.settings.speed = Number($('#openai_compatible_tts_speed').val());
+        $('#openai_compatible_tts_speed_output').text(this.settings.speed);
+        saveTtsProviderSettings();
+    }
+    async checkReady() {
+        await this.fetchTtsVoiceObjects();
+    }
+    async onRefreshClick() {
+        return;
+    }
+    async getVoice(voiceName) {
+        if (this.voices.length == 0) {
+            this.voices = await this.fetchTtsVoiceObjects();
+        }
+        const match = this.voices.filter(
+            oaicVoice => oaicVoice.name == voiceName,
+        )[0];
+        if (!match) {
+            throw `TTS Voice name ${voiceName} not found`;
+        }
+        return match;
+    }
+    async generateTts(text, voiceId) {
+        const response = await this.fetchTtsGeneration(text, voiceId);
+        return response;
+    }
+    async fetchTtsVoiceObjects() {
+        return this.settings.available_voices.map(v => {
+            return { name: v, voice_id: v, lang: 'en-US' };
+        });
+    }
+    async previewTtsVoice(voiceId) {
+        this.audioElement.pause();
+        this.audioElement.currentTime = 0;
+        const text = getPreviewString('en-US');
+        const response = await this.fetchTtsGeneration(text, voiceId);
+        if (!response.ok) {
+            throw new Error(`HTTP ${response.status}`);
+        }
+        const audio = await response.blob();
+        const url = URL.createObjectURL(audio);
+        this.audioElement.src = url;
+        this.audioElement.play();
+        this.audioElement.onended = () => URL.revokeObjectURL(url);
+    }
+    async fetchTtsGeneration(inputText, voiceId) {
+        console.info(`Generating new TTS for voice_id ${voiceId}`);
+        const response = await fetch('/api/openai/custom/generate-voice', {
+            method: 'POST',
+            headers: getRequestHeaders(),
+            body: JSON.stringify({
+                provider_endpoint: this.settings.provider_endpoint,
+                model: this.settings.model,
+                input: inputText,
+                voice: voiceId,
+                response_format: 'mp3',
+                speed: this.settings.speed,
+            }),
+        });
+        if (!response.ok) {
+            toastr.error(response.statusText, 'TTS Generation Failed');
+            throw new Error(`HTTP ${response.status}: ${await response.text()}`);
+        }
+        return response;
+    }
+}

public/scripts/extensions/vectors/index.js+25 -7 Viewed

 import { SlashCommandParser } from '../../slash-commands/SlashCommandParser.js';
 import { SlashCommand } from '../../slash-commands/SlashCommand.js';
 import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from '../../slash-commands/SlashCommandArgument.js';
+import { callGenericPopup, POPUP_RESULT, POPUP_TYPE } from '../../popup.js';
 const MODULE_NAME = 'vectors';

         saveSettingsDebounced();
         toggleSettings();
     });
-    $('#api_key_nomicai').on('change', () => {
+    $('#api_key_nomicai').on('click', async () => {
-        const nomicKey = String($('#api_key_nomicai').val()).trim();
+        const popupText = 'NomicAI API Key:';
-        if (nomicKey.length) {
+        const key = await callGenericPopup(popupText, POPUP_TYPE.INPUT, '', {
-            writeSecret(SECRET_KEYS.NOMICAI, nomicKey);
+            customButtons: [{
+                text: 'Remove Key',
+                appendAtEnd: true,
+                result: POPUP_RESULT.NEGATIVE,
+                action: async () => {
+                    await writeSecret(SECRET_KEYS.NOMICAI, '');
+                    toastr.success('API Key removed');
+                    $('#api_key_nomicai').toggleClass('success', !!secret_state[SECRET_KEYS.NOMICAI]);
+                    saveSettingsDebounced();
+                },
+            }],
+        });
+        if (!key) {
+            return;
+        }
+        await writeSecret(SECRET_KEYS.NOMICAI, String(key));
+        $('#api_key_nomicai').toggleClass('success', !!secret_state[SECRET_KEYS.NOMICAI]);
+        toastr.success('API Key saved');
         saveSettingsDebounced();
     });
     $('#vectors_togetherai_model').val(settings.togetherai_model).on('change', () => {

         $('#dialogue_popup_input').val(presetModel);
     });
-    const validSecret = !!secret_state[SECRET_KEYS.NOMICAI];
+    $('#api_key_nomicai').toggleClass('success', !!secret_state[SECRET_KEYS.NOMICAI]);
-    const placeholder = validSecret ? '✔️ Key saved' : '❌ Missing key';
-    $('#api_key_nomicai').attr('placeholder', placeholder);
     toggleSettings();
     eventSource.on(event_types.MESSAGE_DELETED, onChatEvent);

public/scripts/extensions/vectors/settings.html+5 -9 Viewed

                 </span>
             </small>
-            <div class="flex-container flexFlowColumn" id="nomicai_apiKey">
+            <div class="flex-container alignItemsCenter" id="nomicai_apiKey">
-                <label for="api_key_nomicai">
+                <label for="api_key_nomicai" class="flex1">
                     <span data-i18n="NomicAI API Key">NomicAI API Key</span>
                 </label>
-                <div class="flex-container">
+                <div id="api_key_nomicai" class="menu_button menu_button_icon">
-                    <input id="api_key_nomicai" name="api_key_nomicai" class="text_pole flex1 wide100p" maxlength="500" size="35" type="text" autocomplete="off">
+                    <i class="fa-solid fa-key"></i>
-                    <div title="Clear your API key" class="menu_button fa-solid fa-circle-xmark clear-api-key" data-key="api_key_nomicai">
+                    <span data-i18n="Click to set">Click to set</span>
-                    </div>
-                </div>
-                <div data-for="api_key_nomicai" class="neutral_warning" data-i18n="For privacy reasons, your API key will be hidden after you reload the page.">
-                    For privacy reasons, your API key will be hidden after you reload the page.
                 </div>
             </div>

public/scripts/secrets.js+2 -1 Viewed

     HUGGINGFACE: 'api_key_huggingface',
     STABILITY: 'api_key_stability',
     BLOCKENTROPY: 'api_key_blockentropy',
+    CUSTOM_OPENAI_TTS: 'api_key_custom_openai_tts',
 };
 const INPUT_MAP = {

@@ -127,7 +128,7 @@ export async function writeSecret(key, value) {
127	const text = await response.text();	128	const text = await response.text();
128		129
129	if (text == 'ok') {	130	if (text == 'ok') {
130	secret_state[key] = true;	131	secret_state[key] = !!value;
131	updateSecretDisplay();	132	updateSecretDisplay();
132	}	133	}
133	}	134	}

src/endpoints/openai.js+44 -0 Viewed

+    }
 });
+const custom = express.Router();
+custom.post('/generate-voice', jsonParser, async (request, response) => {
+    try {
+        const key = readSecret(request.user.directories, SECRET_KEYS.CUSTOM_OPENAI_TTS);
+        const { input, provider_endpoint, response_format, voice, speed, model } = request.body;
+        if (!provider_endpoint) {
+            console.log('No OpenAI-compatible TTS provider endpoint provided');
+            return response.sendStatus(400);
+        }
+        const result = await fetch(provider_endpoint, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                Authorization: `Bearer ${key ?? ''}`,
+            },
+            body: JSON.stringify({
+                input: input ?? '',
+                response_format: response_format ?? 'mp3',
+                voice: voice ?? 'alloy',
+                speed: speed ?? 1,
+                model: model ?? 'tts-1',
+            }),
+        });
+        if (!result.ok) {
+            const text = await result.text();
+            console.log('OpenAI request failed', result.statusText, text);
+            return response.status(500).send(text);
+        }
+        const buffer = await result.arrayBuffer();
+        response.setHeader('Content-Type', 'audio/mpeg');
+        return response.send(Buffer.from(buffer));
+    } catch (error) {
+        console.error('OpenAI TTS generation failed', error);
+        response.status(500).send('Internal server error');
+    }
+});
+router.use('/custom', custom);
 module.exports = { router };

src/endpoints/secrets.js+1 -0 Viewed

     HUGGINGFACE: 'api_key_huggingface',
     STABILITY: 'api_key_stability',
     BLOCKENTROPY: 'api_key_blockentropy',
+    CUSTOM_OPENAI_TTS: 'api_key_custom_openai_tts',
 };
 // These are the keys that are safe to expose, even if allowKeysExposure is false