Quick and dirty attempt at implementing an OpenAI compatible TTS voice provider

ab42d6ff823bb33008270c45488145a824eab6e9

Alex Yancey <me@alexyancey.com>

Parents: 406acb4312a2da30ea6e282b085112eed62653e5
2 files changed, +156 -0Ignore whitespace
public/scripts/extensions/tts/index.js+2 -0
@@ -9,6 +9,7 @@ import { SystemTtsProvider } from './system.js';
99import { NovelTtsProvider } from './novel.js';
1010import { power_user } from '../../power-user.js';
1111import { OpenAITtsProvider } from './openai.js';
12+import { OpenAICompatibleTtsProvider } from './openai-compatible.js';
1213import { XTTSTtsProvider } from './xtts.js';
1314import { VITSTtsProvider } from './vits.js';
1415import { GSVITtsProvider } from './gsvi.js';
@@ -96,6 +97,7 @@ const ttsProviders = {
9697 AllTalk: AllTalkTtsProvider,
9798 SpeechT5: SpeechT5TtsProvider,
9899 Azure: AzureTtsProvider,
100+ OpenAICompatible: OpenAICompatibleTtsProvider,
99101};
100102let ttsProvider;
101103let ttsProviderName;
public/scripts/extensions/tts/openai-compatible.js+154 -0
@@ -0,0 +1,154 @@
1+import { getRequestHeaders } from '../../../script.js';
2+import { getPreviewString, saveTtsProviderSettings } from './index.js';
3+
4+export { OpenAICompatibleTtsProvider };
5+
6+class OpenAICompatibleTtsProvider {
7+ settings;
8+ voices = [];
9+ separator = ' . ';
10+
11+ audioElement = document.createElement('audio');
12+
13+ defaultSettings = {
14+ voiceMap: {},
15+ model: 'tts-1',
16+ speed: 1,
17+ available_voices: ['alloy', 'echo', 'fable', 'onyx', 'nova', 'shimmer'],
18+ provider_endpoint: 'http://127.0.0.1:8000/v1/audio/speech',
19+ };
20+
21+ get settingsHtml() {
22+ let html = `
23+ <label for="openai_compatible_tts_endpoint">Provider Endpoint:</label>
24+ <input id="openai_compatible_tts_endpoint" type="text" class="text_pole" maxlength="250" value="${this.defaultSettings.provider_endpoint}"/>
25+ <label for="openai_compatible_model">Model:</label>
26+ <input id="openai_compatible_model" type="text" class="text_pole" maxlength="250" value="${this.defaultSettings.model}"/>
27+ <label for="openai_compatible_tts_voices">Available Voices (comma separated):</label>
28+ <input id="openai_compatible_tts_voices" type="text" class="text_pole" maxlength="250" value="${this.defaultSettings.available_voices.join()}"/>
29+ <label for="openai_compatible_tts_speed">Speed: <span id="openai_compatible_tts_speed_output"></span></label>
30+ <input type="range" id="openai_compatible_tts_speed" value="1" min="0.25" max="4" step="0.05">`;
31+ return html;
32+ }
33+
34+ async loadSettings(settings) {
35+ // Populate Provider UI given input settings
36+ if (Object.keys(settings).length == 0) {
37+ console.info('Using default TTS Provider settings');
38+ }
39+
40+ // Only accept keys defined in defaultSettings
41+ this.settings = this.defaultSettings;
42+
43+ for (const key in settings) {
44+ if (key in this.settings) {
45+ this.settings[key] = settings[key];
46+ } else {
47+ throw `Invalid setting passed to TTS Provider: ${key}`;
48+ }
49+ }
50+
51+ $('#openai_compatible_tts_endpoint').val(this.settings.provider_endpoint);
52+ $('#openai_compatible_tts_endpoint').on('input', () => { this.onSettingsChange(); });
53+
54+ $('#openai_compatible_model').val(this.defaultSettings.model);
55+ $('#openai_compatible_model').on('input', () => { this.onSettingsChange(); });
56+
57+ $('#openai_compatible_tts_voices').val(this.settings.available_voices.join());
58+ $('#openai_compatible_tts_voices').on('input', () => { this.onSettingsChange(); });
59+
60+ $('#openai_compatible_tts_speed').val(this.settings.speed);
61+ $('#openai_compatible_tts_speed').on('input', () => {
62+ this.onSettingsChange();
63+ });
64+
65+ $('#openai_compatible_tts_speed_output').text(this.settings.speed);
66+
67+ this.refreshSession();
68+
69+ await this.checkReady();
70+
71+ console.debug('OpenAI Compatible TTS: Settings loaded');
72+ }
73+
74+ onSettingsChange() {
75+ // Update dynamically
76+ this.settings.provider_endpoint = String($('#openai_compatible_tts_endpoint').val());
77+ this.settings.model = String($('#openai_compatible_model').val());
78+ this.settings.available_voices = $('#openai_compatible_tts_voices').val().split(',');
79+ this.settings.speed = Number($('#openai_compatible_tts_speed').val());
80+ $('#openai_compatible_tts_speed_output').text(this.settings.speed);
81+ saveTtsProviderSettings();
82+ }
83+
84+ async checkReady() {
85+ await this.fetchTtsVoiceObjects();
86+ }
87+
88+ async onRefreshClick() {
89+ return;
90+ }
91+
92+ async getVoice(voiceName) {
93+ if (this.voices.length == 0) {
94+ this.voices = await this.fetchTtsVoiceObjects();
95+ }
96+ const match = this.voices.filter(
97+ oaicVoice => oaicVoice.name == voiceName,
98+ )[0];
99+ if (!match) {
100+ throw `TTS Voice name ${voiceName} not found`;
101+ }
102+ return match;
103+ }
104+
105+ async generateTts(text, voiceId) {
106+ const response = await this.fetchTtsGeneration(text, voiceId);
107+ return response;
108+ }
109+
110+ async fetchTtsVoiceObjects() {
111+ return this.settings.available_voices.map(v => {
112+ return { name: v, voice_id: v, lang: 'en-US' };
113+ });
114+ }
115+
116+ async previewTtsVoice(voiceId) {
117+ this.audioElement.pause();
118+ this.audioElement.currentTime = 0;
119+
120+ const text = getPreviewString('en-US');
121+ const response = await this.fetchTtsGeneration(text, voiceId);
122+ if (!response.ok) {
123+ throw new Error(`HTTP ${response.status}`);
124+ }
125+
126+ const audio = await response.blob();
127+ const url = URL.createObjectURL(audio);
128+ this.audioElement.src = url;
129+ this.audioElement.play();
130+ this.audioElement.onended = () => URL.revokeObjectURL(url);
131+ }
132+
133+ async fetchTtsGeneration(inputText, voiceId) {
134+ console.info(`Generating new TTS for voice_id ${voiceId}`);
135+ const response = await fetch(this.settings.provider_endpoint, {
136+ method: 'POST',
137+ headers: getRequestHeaders(),
138+ body: JSON.stringify({
139+ 'model': this.settings.model,
140+ 'input': inputText,
141+ 'voice': voiceId,
142+ 'response_format': 'mp3',
143+ 'speed': this.settings.speed,
144+ }),
145+ });
146+
147+ if (!response.ok) {
148+ toastr.error(response.statusText, 'TTS Generation Failed');
149+ throw new Error(`HTTP ${response.status}: ${await response.text()}`);
150+ }
151+
152+ return response;
153+ }
154+}