Merge pull request #2654 from ayancey/openai-compatible-tts OpenAI Compatible API TTS Provider

33916aa493399bebe2d3e327ce91c5abdae295aa

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: 406acb4312a2da30ea6e282b085112eed62653e5 d0b368f29cc4f11b3c2dd4d6c2a896f0b70f42de
10 files changed, +324 -35Ignore whitespace
public/scripts/extensions/stable-diffusion/index.js+16 -5
@@ -30,7 +30,7 @@ import { SlashCommand } from '../../slash-commands/SlashCommand.js';
3030import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from '../../slash-commands/SlashCommandArgument.js';
3131import { debounce_timeout } from '../../constants.js';
3232import { SlashCommandEnumValue } from '../../slash-commands/SlashCommandEnumValue.js';
3333import { POPUP_RESULT, POPUP_TYPE, Popup, callGenericPopup } from '../../popup.js';
3434export { MODULE_NAME };
3535
3636const MODULE_NAME = 'sd';
@@ -1097,7 +1097,18 @@ function onComfyWorkflowChange() {
10971097
10981098async function onStabilityKeyClick() {
10991099 const popupText = 'Stability AI API Key:';
11001100 const key = await callGenericPopup(popupText, POPUP_TYPE.INPUT);, '', {
1101+ customButtons: [{
1102+ text: 'Remove Key',
1103+ appendAtEnd: true,
1104+ result: POPUP_RESULT.NEGATIVE,
1105+ action: async () => {
1106+ await writeSecret(SECRET_KEYS.STABILITY, '');
1107+ toastr.success('API Key removed');
1108+ await loadSettingOptions();
1109+ },
1110+ }],
1111+ });
11011112
11021113 if (!key) {
11031114 return;
@@ -2693,15 +2704,15 @@ async function generateBlockEntropyImage(prompt, negativePrompt, signal) {
26932704
26942705 if (result.ok) {
26952706 const data = await result.json();
2696-
2707+
26972708 // Default format is 'jpg'
26982709 let format = 'jpg';
2699-
2710+
27002711 // Check if a format is specified in the result
27012712 if (data.format) {
27022713 format = data.format.toLowerCase();
27032714 }
2704-
2715+
27052716 return { format: format, data: data.images[0] };
27062717 } else {
27072718 const text = await result.text();
public/scripts/extensions/translate/index.js+13 -2
@@ -10,7 +10,7 @@ import {
1010 updateMessageBlock,
1111} from '../../../script.js';
1212import { extension_settings, getContext, renderExtensionTemplateAsync } from '../../extensions.js';
1313import { POPUP_RESULT, POPUP_TYPE, callGenericPopup } from '../../popup.js';
1414import { findSecret, secret_state, writeSecret } from '../../secrets.js';
1515import { SlashCommand } from '../../slash-commands/SlashCommand.js';
1616import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from '../../slash-commands/SlashCommandArgument.js';
@@ -621,7 +621,18 @@ jQuery(async () => {
621621 const secretKey = extension_settings.translate.provider + '_url';
622622 const savedUrl = secret_state[secretKey] ? await findSecret(secretKey) : '';
623623
624624 const url = await callGenericPopup(popupText, POPUP_TYPE.INPUT, savedUrl);,{
625+ customButtons: [{
626+ text: 'Remove URL',
627+ appendAtEnd: true,
628+ result: POPUP_RESULT.NEGATIVE,
629+ action: async () => {
630+ await writeSecret(secretKey, '');
631+ toastr.success('API URL removed');
632+ $('#translate_url_button').toggleClass('success', !!secret_state[secretKey]);
633+ },
634+ }],
635+ });
625636
626637 if (url == false || url == '') {
627638 return;
public/scripts/extensions/tts/azure.js+14 -2
@@ -1,5 +1,5 @@
11import { getRequestHeaders } from '../../../script.js';
22import { POPUP_RESULT, POPUP_TYPE, callGenericPopup } from '../../popup.js';
33import { SECRET_KEYS, findSecret, secret_state, writeSecret } from '../../secrets.js';
44import { getPreviewString, saveTtsProviderSettings } from './index.js';
55export { AzureTtsProvider };
@@ -70,7 +70,19 @@ class AzureTtsProvider {
7070 const popupText = 'Azure TTS API Key';
7171 const savedKey = secret_state[SECRET_KEYS.AZURE_TTS] ? await findSecret(SECRET_KEYS.AZURE_TTS) : '';
7272
7373 const key = await callGenericPopup(popupText, POPUP_TYPE.INPUT, savedKey);, {
74+ customButtons: [{
75+ text: 'Remove Key',
76+ appendAtEnd: true,
77+ result: POPUP_RESULT.NEGATIVE,
78+ action: async () => {
79+ await writeSecret(SECRET_KEYS.AZURE_TTS, '');
80+ $('#azure_tts_key').toggleClass('success', secret_state[SECRET_KEYS.AZURE_TTS]);
81+ toastr.success('API Key removed');
82+ await this.onRefreshClick();
83+ },
84+ }],
85+ });
7486
7587 if (key == false || key == '') {
7688 return;
public/scripts/extensions/tts/index.js+11 -9
@@ -9,6 +9,7 @@ import { SystemTtsProvider } from './system.js';
99import { NovelTtsProvider } from './novel.js';
1010import { power_user } from '../../power-user.js';
1111import { OpenAITtsProvider } from './openai.js';
12+import { OpenAICompatibleTtsProvider } from './openai-compatible.js';
1213import { XTTSTtsProvider } from './xtts.js';
1314import { VITSTtsProvider } from './vits.js';
1415import { GSVITtsProvider } from './gsvi.js';
@@ -82,20 +83,21 @@ export function getPreviewString(lang) {
8283}
8384
8485const ttsProviders = {
8586 ElevenLabsAllTalk: ElevenLabsTtsProviderAllTalkTtsProvider,
8687 SileroAzure: SileroTtsProviderAzureTtsProvider,
87- XTTSv2: XTTSTtsProvider,
88- VITS: VITSTtsProvider,
89- GSVI: GSVITtsProvider,
90- SBVits2: SBVits2TtsProvider,
91- System: SystemTtsProvider,
9288 Coqui: CoquiTtsProvider,
9389 Edge: EdgeTtsProvider,
90+ ElevenLabs: ElevenLabsTtsProvider,
91+ GSVI: GSVITtsProvider,
9492 Novel: NovelTtsProvider,
9593 OpenAI: OpenAITtsProvider,
9694 AllTalk'OpenAI Compatible': AllTalkTtsProviderOpenAICompatibleTtsProvider,
95+ SBVits2: SBVits2TtsProvider,
96+ Silero: SileroTtsProvider,
9797 SpeechT5: SpeechT5TtsProvider,
9898 AzureSystem: AzureTtsProviderSystemTtsProvider,
99+ VITS: VITSTtsProvider,
100+ XTTSv2: XTTSTtsProvider,
99101};
100102let ttsProvider;
101103let ttsProviderName;
public/scripts/extensions/tts/openai-compatible.js+193 -0
@@ -0,0 +1,193 @@
1+import { getRequestHeaders } from '../../../script.js';
2+import { callGenericPopup, POPUP_RESULT, POPUP_TYPE } from '../../popup.js';
3+import { findSecret, SECRET_KEYS, secret_state, writeSecret } from '../../secrets.js';
4+import { getPreviewString, saveTtsProviderSettings } from './index.js';
5+
6+export { OpenAICompatibleTtsProvider };
7+
8+class OpenAICompatibleTtsProvider {
9+ settings;
10+ voices = [];
11+ separator = ' . ';
12+
13+ audioElement = document.createElement('audio');
14+
15+ defaultSettings = {
16+ voiceMap: {},
17+ model: 'tts-1',
18+ speed: 1,
19+ available_voices: ['alloy', 'echo', 'fable', 'onyx', 'nova', 'shimmer'],
20+ provider_endpoint: 'http://127.0.0.1:8000/v1/audio/speech',
21+ };
22+
23+ get settingsHtml() {
24+ let html = `
25+ <label for="openai_compatible_tts_endpoint">Provider Endpoint:</label>
26+ <div class="flex-container alignItemsCenter">
27+ <div class="flex1">
28+ <input id="openai_compatible_tts_endpoint" type="text" class="text_pole" maxlength="250" value="${this.defaultSettings.provider_endpoint}"/>
29+ </div>
30+ <div id="openai_compatible_tts_key" class="menu_button menu_button_icon">
31+ <i class="fa-solid fa-key"></i>
32+ <span>API Key</span>
33+ </div>
34+ </div>
35+ <label for="openai_compatible_model">Model:</label>
36+ <input id="openai_compatible_model" type="text" class="text_pole" maxlength="250" value="${this.defaultSettings.model}"/>
37+ <label for="openai_compatible_tts_voices">Available Voices (comma separated):</label>
38+ <input id="openai_compatible_tts_voices" type="text" class="text_pole" maxlength="250" value="${this.defaultSettings.available_voices.join()}"/>
39+ <label for="openai_compatible_tts_speed">Speed: <span id="openai_compatible_tts_speed_output"></span></label>
40+ <input type="range" id="openai_compatible_tts_speed" value="1" min="0.25" max="4" step="0.05">`;
41+ return html;
42+ }
43+
44+ async loadSettings(settings) {
45+ // Populate Provider UI given input settings
46+ if (Object.keys(settings).length == 0) {
47+ console.info('Using default TTS Provider settings');
48+ }
49+
50+ // Only accept keys defined in defaultSettings
51+ this.settings = this.defaultSettings;
52+
53+ for (const key in settings) {
54+ if (key in this.settings) {
55+ this.settings[key] = settings[key];
56+ } else {
57+ throw `Invalid setting passed to TTS Provider: ${key}`;
58+ }
59+ }
60+
61+ $('#openai_compatible_tts_endpoint').val(this.settings.provider_endpoint);
62+ $('#openai_compatible_tts_endpoint').on('input', () => { this.onSettingsChange(); });
63+
64+ $('#openai_compatible_model').val(this.defaultSettings.model);
65+ $('#openai_compatible_model').on('input', () => { this.onSettingsChange(); });
66+
67+ $('#openai_compatible_tts_voices').val(this.settings.available_voices.join());
68+ $('#openai_compatible_tts_voices').on('input', () => { this.onSettingsChange(); });
69+
70+ $('#openai_compatible_tts_speed').val(this.settings.speed);
71+ $('#openai_compatible_tts_speed').on('input', () => {
72+ this.onSettingsChange();
73+ });
74+
75+ $('#openai_compatible_tts_speed_output').text(this.settings.speed);
76+
77+ $('#openai_compatible_tts_key').toggleClass('success', secret_state[SECRET_KEYS.CUSTOM_OPENAI_TTS]);
78+ $('#openai_compatible_tts_key').on('click', async () => {
79+ const popupText = 'OpenAI-compatible TTS API Key';
80+ const savedKey = secret_state[SECRET_KEYS.CUSTOM_OPENAI_TTS] ? await findSecret(SECRET_KEYS.CUSTOM_OPENAI_TTS) : '';
81+
82+ const key = await callGenericPopup(popupText, POPUP_TYPE.INPUT, savedKey, {
83+ customButtons: [{
84+ text: 'Remove Key',
85+ appendAtEnd: true,
86+ result: POPUP_RESULT.NEGATIVE,
87+ action: async () => {
88+ await writeSecret(SECRET_KEYS.CUSTOM_OPENAI_TTS, '');
89+ $('#openai_compatible_tts_key').toggleClass('success', secret_state[SECRET_KEYS.CUSTOM_OPENAI_TTS]);
90+ toastr.success('API Key removed');
91+ await this.onRefreshClick();
92+ },
93+ }],
94+ });
95+
96+ if (key == false || key == '') {
97+ return;
98+ }
99+
100+ await writeSecret(SECRET_KEYS.CUSTOM_OPENAI_TTS, String(key));
101+
102+ toastr.success('API Key saved');
103+ $('#openai_compatible_tts_key').toggleClass('success', secret_state[SECRET_KEYS.CUSTOM_OPENAI_TTS]);
104+ await this.onRefreshClick();
105+ });
106+
107+ await this.checkReady();
108+
109+ console.debug('OpenAI Compatible TTS: Settings loaded');
110+ }
111+
112+ onSettingsChange() {
113+ // Update dynamically
114+ this.settings.provider_endpoint = String($('#openai_compatible_tts_endpoint').val());
115+ this.settings.model = String($('#openai_compatible_model').val());
116+ this.settings.available_voices = String($('#openai_compatible_tts_voices').val()).split(',');
117+ this.settings.speed = Number($('#openai_compatible_tts_speed').val());
118+ $('#openai_compatible_tts_speed_output').text(this.settings.speed);
119+ saveTtsProviderSettings();
120+ }
121+
122+ async checkReady() {
123+ await this.fetchTtsVoiceObjects();
124+ }
125+
126+ async onRefreshClick() {
127+ return;
128+ }
129+
130+ async getVoice(voiceName) {
131+ if (this.voices.length == 0) {
132+ this.voices = await this.fetchTtsVoiceObjects();
133+ }
134+ const match = this.voices.filter(
135+ oaicVoice => oaicVoice.name == voiceName,
136+ )[0];
137+ if (!match) {
138+ throw `TTS Voice name ${voiceName} not found`;
139+ }
140+ return match;
141+ }
142+
143+ async generateTts(text, voiceId) {
144+ const response = await this.fetchTtsGeneration(text, voiceId);
145+ return response;
146+ }
147+
148+ async fetchTtsVoiceObjects() {
149+ return this.settings.available_voices.map(v => {
150+ return { name: v, voice_id: v, lang: 'en-US' };
151+ });
152+ }
153+
154+ async previewTtsVoice(voiceId) {
155+ this.audioElement.pause();
156+ this.audioElement.currentTime = 0;
157+
158+ const text = getPreviewString('en-US');
159+ const response = await this.fetchTtsGeneration(text, voiceId);
160+ if (!response.ok) {
161+ throw new Error(`HTTP ${response.status}`);
162+ }
163+
164+ const audio = await response.blob();
165+ const url = URL.createObjectURL(audio);
166+ this.audioElement.src = url;
167+ this.audioElement.play();
168+ this.audioElement.onended = () => URL.revokeObjectURL(url);
169+ }
170+
171+ async fetchTtsGeneration(inputText, voiceId) {
172+ console.info(`Generating new TTS for voice_id ${voiceId}`);
173+ const response = await fetch('/api/openai/custom/generate-voice', {
174+ method: 'POST',
175+ headers: getRequestHeaders(),
176+ body: JSON.stringify({
177+ provider_endpoint: this.settings.provider_endpoint,
178+ model: this.settings.model,
179+ input: inputText,
180+ voice: voiceId,
181+ response_format: 'mp3',
182+ speed: this.settings.speed,
183+ }),
184+ });
185+
186+ if (!response.ok) {
187+ toastr.error(response.statusText, 'TTS Generation Failed');
188+ throw new Error(`HTTP ${response.status}: ${await response.text()}`);
189+ }
190+
191+ return response;
192+ }
193+}
public/scripts/extensions/vectors/index.js+25 -7
@@ -30,6 +30,7 @@ import { textgen_types, textgenerationwebui_settings } from '../../textgen-setti
3030import { SlashCommandParser } from '../../slash-commands/SlashCommandParser.js';
3131import { SlashCommand } from '../../slash-commands/SlashCommand.js';
3232import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from '../../slash-commands/SlashCommandArgument.js';
33+import { callGenericPopup, POPUP_RESULT, POPUP_TYPE } from '../../popup.js';
3334
3435const MODULE_NAME = 'vectors';
3536
@@ -1299,11 +1300,30 @@ jQuery(async () => {
12991300 saveSettingsDebounced();
13001301 toggleSettings();
13011302 });
13021303 $('#api_key_nomicai').on('changeclick', async () => {
1303- const nomicKey = String($('#api_key_nomicai').val()).trim();
1304+ const popupText = 'NomicAI API Key:';
1304- if (nomicKey.length) {
1305+ const key = await callGenericPopup(popupText, POPUP_TYPE.INPUT, '', {
1305- writeSecret(SECRET_KEYS.NOMICAI, nomicKey);
1306+ customButtons: [{
1307+ text: 'Remove Key',
1308+ appendAtEnd: true,
1309+ result: POPUP_RESULT.NEGATIVE,
1310+ action: async () => {
1311+ await writeSecret(SECRET_KEYS.NOMICAI, '');
1312+ toastr.success('API Key removed');
1313+ $('#api_key_nomicai').toggleClass('success', !!secret_state[SECRET_KEYS.NOMICAI]);
1314+ saveSettingsDebounced();
1315+ },
1316+ }],
1317+ });
1318+
1319+ if (!key) {
1320+ return;
13061321 }
1322+
1323+ await writeSecret(SECRET_KEYS.NOMICAI, String(key));
1324+ $('#api_key_nomicai').toggleClass('success', !!secret_state[SECRET_KEYS.NOMICAI]);
1325+
1326+ toastr.success('API Key saved');
13071327 saveSettingsDebounced();
13081328 });
13091329 $('#vectors_togetherai_model').val(settings.togetherai_model).on('change', () => {
@@ -1531,9 +1551,7 @@ jQuery(async () => {
15311551 $('#dialogue_popup_input').val(presetModel);
15321552 });
15331553
15341554 const validSecret =$('#api_key_nomicai').toggleClass('success', !!secret_state[SECRET_KEYS.NOMICAI]);
1535- const placeholder = validSecret ? '✔️ Key saved' : '❌ Missing key';
1536- $('#api_key_nomicai').attr('placeholder', placeholder);
15371555
15381556 toggleSettings();
15391557 eventSource.on(event_types.MESSAGE_DELETED, onChatEvent);
public/scripts/extensions/vectors/settings.html+5 -9
@@ -103,17 +103,13 @@
103103 </span>
104104 </small>
105105
106106 <div class="flex-container flexFlowColumnalignItemsCenter" id="nomicai_apiKey">
107107 <label for="api_key_nomicai" class="flex1">
108108 <span data-i18n="NomicAI API Key">NomicAI API Key</span>
109109 </label>
110110 <div id="api_key_nomicai" class="flex-containermenu_button menu_button_icon">
111- <input id="api_key_nomicai" name="api_key_nomicai" class="text_pole flex1 wide100p" maxlength="500" size="35" type="text" autocomplete="off">
111+ <i class="fa-solid fa-key"></i>
112- <div title="Clear your API key" class="menu_button fa-solid fa-circle-xmark clear-api-key" data-key="api_key_nomicai">
112+ <span data-i18n="Click to set">Click to set</span>
113- </div>
114- </div>
115- <div data-for="api_key_nomicai" class="neutral_warning" data-i18n="For privacy reasons, your API key will be hidden after you reload the page.">
116- For privacy reasons, your API key will be hidden after you reload the page.
117113 </div>
118114 </div>
119115
public/scripts/secrets.js+2 -1
@@ -33,6 +33,7 @@ export const SECRET_KEYS = {
3333 HUGGINGFACE: 'api_key_huggingface',
3434 STABILITY: 'api_key_stability',
3535 BLOCKENTROPY: 'api_key_blockentropy',
36+ CUSTOM_OPENAI_TTS: 'api_key_custom_openai_tts',
3637};
3738
3839const INPUT_MAP = {
@@ -127,7 +128,7 @@ export async function writeSecret(key, value) {
127128 const text = await response.text();
128129
129130 if (text == 'ok') {
130131 secret_state[key] = true!!value;
131132 updateSecretDisplay();
132133 }
133134 }
src/endpoints/openai.js+44 -0
@@ -282,4 +282,48 @@ router.post('/generate-image', jsonParser, async (request, response) => {
282282 }
283283});
284284
285+const custom = express.Router();
286+
287+custom.post('/generate-voice', jsonParser, async (request, response) => {
288+ try {
289+ const key = readSecret(request.user.directories, SECRET_KEYS.CUSTOM_OPENAI_TTS);
290+ const { input, provider_endpoint, response_format, voice, speed, model } = request.body;
291+
292+ if (!provider_endpoint) {
293+ console.log('No OpenAI-compatible TTS provider endpoint provided');
294+ return response.sendStatus(400);
295+ }
296+
297+ const result = await fetch(provider_endpoint, {
298+ method: 'POST',
299+ headers: {
300+ 'Content-Type': 'application/json',
301+ Authorization: `Bearer ${key ?? ''}`,
302+ },
303+ body: JSON.stringify({
304+ input: input ?? '',
305+ response_format: response_format ?? 'mp3',
306+ voice: voice ?? 'alloy',
307+ speed: speed ?? 1,
308+ model: model ?? 'tts-1',
309+ }),
310+ });
311+
312+ if (!result.ok) {
313+ const text = await result.text();
314+ console.log('OpenAI request failed', result.statusText, text);
315+ return response.status(500).send(text);
316+ }
317+
318+ const buffer = await result.arrayBuffer();
319+ response.setHeader('Content-Type', 'audio/mpeg');
320+ return response.send(Buffer.from(buffer));
321+ } catch (error) {
322+ console.error('OpenAI TTS generation failed', error);
323+ response.status(500).send('Internal server error');
324+ }
325+});
326+
327+router.use('/custom', custom);
328+
285329module.exports = { router };
src/endpoints/secrets.js+1 -0
@@ -45,6 +45,7 @@ const SECRET_KEYS = {
4545 HUGGINGFACE: 'api_key_huggingface',
4646 STABILITY: 'api_key_stability',
4747 BLOCKENTROPY: 'api_key_blockentropy',
48+ CUSTOM_OPENAI_TTS: 'api_key_custom_openai_tts',
4849};
4950
5051// These are the keys that are safe to expose, even if allowKeysExposure is false