TTS: Add support for VITS/W2V2-VITS/Bert-VITS2 (#2439) * add VITSTtsProvider * add options * fix params, drop reference_audio_path * post with body * improve preview lang * add space in label

bb39e852b88eca7cc5ba5caf7ac8565ec2e285fb

Risenafis <91325858+Risenafis@users.noreply.github.com>

Signed
Parents: 75099d3a2260476b877d904119a0241902beeeaa
2 files changed, +406 -0Ignore whitespace
public/scripts/extensions/tts/index.js+2 -0
@@ -10,6 +10,7 @@ import { NovelTtsProvider } from './novel.js';
1010import { power_user } from '../../power-user.js';
1111import { OpenAITtsProvider } from './openai.js';
1212import { XTTSTtsProvider } from './xtts.js';
13+import { VITSTtsProvider } from './vits.js';
1314import { GSVITtsProvider } from './gsvi.js';
1415import { SBVits2TtsProvider } from './sbvits2.js';
1516import { AllTalkTtsProvider } from './alltalk.js';
@@ -83,6 +84,7 @@ const ttsProviders = {
8384 ElevenLabs: ElevenLabsTtsProvider,
8485 Silero: SileroTtsProvider,
8586 XTTSv2: XTTSTtsProvider,
87+ VITS: VITSTtsProvider,
8688 GSVI: GSVITtsProvider,
8789 SBVits2: SBVits2TtsProvider,
8890 System: SystemTtsProvider,
public/scripts/extensions/tts/vits.js+404 -0
@@ -0,0 +1,404 @@
1+import { getPreviewString, saveTtsProviderSettings } from './index.js';
2+
3+export { VITSTtsProvider };
4+
5+class VITSTtsProvider {
6+ //########//
7+ // Config //
8+ //########//
9+
10+ settings;
11+ ready = false;
12+ voices = [];
13+ separator = '. ';
14+ audioElement = document.createElement('audio');
15+
16+ /**
17+ * Perform any text processing before passing to TTS engine.
18+ * @param {string} text Input text
19+ * @returns {string} Processed text
20+ */
21+ processText(text) {
22+ return text;
23+ }
24+
25+ audioFormats = ['wav', 'ogg', 'silk', 'mp3', 'flac'];
26+
27+ languageLabels = {
28+ 'Auto': 'auto',
29+ 'Chinese': 'zh',
30+ 'English': 'en',
31+ 'Japanese': 'ja',
32+ 'Korean': 'ko',
33+ };
34+
35+ langKey2LangCode = {
36+ 'zh': 'zh-CN',
37+ 'en': 'en-US',
38+ 'ja': 'ja-JP',
39+ 'ko': 'ko-KR',
40+ };
41+
42+ modelTypes = {
43+ VITS: 'VITS',
44+ W2V2_VITS: 'W2V2-VITS',
45+ BERT_VITS2: 'BERT-VITS2',
46+ };
47+
48+ defaultSettings = {
49+ provider_endpoint: 'http://localhost:23456',
50+ format: 'wav',
51+ lang: 'auto',
52+ length: 1.0,
53+ noise: 0.33,
54+ noisew: 0.4,
55+ segment_size: 50,
56+ streaming: false,
57+ dim_emotion: 0,
58+ sdp_ratio: 0.2,
59+ emotion: 0,
60+ text_prompt: '',
61+ style_text: '',
62+ style_weight: 1,
63+ };
64+
65+ get settingsHtml() {
66+ let html = `
67+ <label for="vits_lang">Text Language</label>
68+ <select id="vits_lang">`;
69+
70+ for (let language in this.languageLabels) {
71+ if (this.languageLabels[language] == this.settings?.lang) {
72+ html += `<option value="${this.languageLabels[language]}" selected="selected">${language}</option>`;
73+ continue;
74+ }
75+ html += `<option value="${this.languageLabels[language]}">${language}</option>`;
76+ }
77+
78+ html += `
79+ </select>
80+ <label>VITS / W2V2-VITS / Bert-VITS2 Settings:</label><br/>
81+ <label for="vits_endpoint">Provider Endpoint:</label>
82+ <input id="vits_endpoint" type="text" class="text_pole" maxlength="250" value="${this.defaultSettings.provider_endpoint}"/>
83+ <span>Use <a target="_blank" href="https://github.com/Artrajz/vits-simple-api">vits-simple-api</a>.</span><br/>
84+
85+ <label for="vits_format">Audio format:</label>
86+ <select id="vits_format">`;
87+
88+ for (let format of this.audioFormats) {
89+ if (format == this.settings?.format) {
90+ html += `<option value="${format}" selected="selected">${format}</option>`;
91+ continue;
92+ }
93+ html += `<option value="${format}">${format}</option>`;
94+ }
95+
96+ html += `
97+ </select>
98+ <label for="vits_length">Audio length: <span id="vits_length_output">${this.defaultSettings.length}</span></label>
99+ <input id="vits_length" type="range" value="${this.defaultSettings.length}" min="0.0" max="5" step="0.01" />
100+
101+ <label for="vits_noise">Noise: <span id="vits_noise_output">${this.defaultSettings.noise}</span></label>
102+ <input id="vits_noise" type="range" value="${this.defaultSettings.noise}" min="0.1" max="2" step="0.01" />
103+
104+ <label for="vits_noisew">SDP noise: <span id="vits_noisew_output">${this.defaultSettings.noisew}</span></label>
105+ <input id="vits_noisew" type="range" value="${this.defaultSettings.noisew}" min="0.1" max="2" step="0.01" />
106+
107+ <label for="vits_segment_size">Segment Size: <span id="vits_segment_size_output">${this.defaultSettings.segment_size}</span></label>
108+ <input id="vits_segment_size" type="range" value="${this.defaultSettings.segment_size}" min="0" max="1000" step="1" />
109+
110+ <label for="vits_streaming" class="checkbox_label">
111+ <input id="vits_streaming" type="checkbox" />
112+ <span>Streaming</span>
113+ </label>
114+
115+ <label>W2V2-VITS Settings:</label><br/>
116+ <label for="vits_dim_emotion">Dimensional emotion:</label>
117+ <input id="vits_dim_emotion" type="number" class="text_pole" min="0" max="5457" step="1" value="${this.defaultSettings.dim_emotion}"/>
118+
119+ <label>BERT-VITS2 Settings:</label><br/>
120+ <label for="vits_sdp_ratio">sdp_ratio: <span id="vits_sdp_ratio_output">${this.defaultSettings.sdp_ratio}</span></label>
121+ <input id="vits_sdp_ratio" type="range" value="${this.defaultSettings.sdp_ratio}" min="0.0" max="1" step="0.01" />
122+
123+ <label for="vits_emotion">emotion: <span id="vits_emotion_output">${this.defaultSettings.emotion}</span></label>
124+ <input id="vits_emotion" type="range" value="${this.defaultSettings.emotion}" min="0" max="9" step="1" />
125+
126+ <label for="vits_text_prompt">Text Prompt:</label>
127+ <input id="vits_text_prompt" type="text" class="text_pole" maxlength="512" value="${this.defaultSettings.text_prompt}"/>
128+
129+ <label for="vits_style_text">Style text:</label>
130+ <input id="vits_style_text" type="text" class="text_pole" maxlength="512" value="${this.defaultSettings.style_text}"/>
131+
132+ <label for="vits_style_weight">Style weight <span id="vits_style_weight_output">${this.defaultSettings.style_weight}</span></label>
133+ <input id="vits_style_weight" type="range" value="${this.defaultSettings.style_weight}" min="0" max="1" step="0.01" />
134+ `;
135+
136+ return html;
137+ }
138+
139+ onSettingsChange() {
140+ // Used when provider settings are updated from UI
141+ this.settings.provider_endpoint = $('#vits_endpoint').val();
142+ this.settings.lang = $('#vits_lang').val();
143+ this.settings.format = $('#vits_format').val();
144+ this.settings.dim_emotion = $('#vits_dim_emotion').val();
145+ this.settings.text_prompt = $('#vits_text_prompt').val();
146+ this.settings.style_text = $('#vits_style_text').val();
147+
148+ // Update the default TTS settings based on input fields
149+ this.settings.length = $('#vits_length').val();
150+ this.settings.noise = $('#vits_noise').val();
151+ this.settings.noisew = $('#vits_noisew').val();
152+ this.settings.segment_size = $('#vits_segment_size').val();
153+ this.settings.streaming = $('#vits_streaming').is(':checked');
154+ this.settings.sdp_ratio = $('#vits_sdp_ratio').val();
155+ this.settings.emotion = $('#vits_emotion').val();
156+ this.settings.style_weight = $('#vits_style_weight').val();
157+
158+ // Update the UI to reflect changes
159+ $('#vits_length_output').text(this.settings.length);
160+ $('#vits_noise_output').text(this.settings.noise);
161+ $('#vits_noisew_output').text(this.settings.noisew);
162+ $('#vits_segment_size_output').text(this.settings.segment_size);
163+ $('#vits_sdp_ratio_output').text(this.settings.sdp_ratio);
164+ $('#vits_emotion_output').text(this.settings.emotion);
165+ $('#vits_style_weight_output').text(this.settings.style_weight);
166+
167+ saveTtsProviderSettings();
168+ this.changeTTSSettings();
169+ }
170+
171+ async loadSettings(settings) {
172+ // Pupulate Provider UI given input settings
173+ if (Object.keys(settings).length == 0) {
174+ console.info('Using default TTS Provider settings');
175+ }
176+
177+ // Only accept keys defined in defaultSettings
178+ this.settings = this.defaultSettings;
179+
180+ for (const key in settings) {
181+ if (key in this.settings) {
182+ this.settings[key] = settings[key];
183+ } else {
184+ console.debug(`Ignoring non-user-configurable setting: ${key}`);
185+ }
186+ }
187+
188+ // Set initial values from the settings
189+ $('#vits_endpoint').val(this.settings.provider_endpoint);
190+ $('#vits_lang').val(this.settings.lang);
191+ $('#vits_format').val(this.settings.format);
192+ $('#vits_length').val(this.settings.length);
193+ $('#vits_noise').val(this.settings.noise);
194+ $('#vits_noisew').val(this.settings.noisew);
195+ $('#vits_segment_size').val(this.settings.segment_size);
196+ $('#vits_streaming').prop('checked', this.settings.streaming);
197+ $('#vits_dim_emotion').val(this.settings.dim_emotion);
198+ $('#vits_sdp_ratio').val(this.settings.sdp_ratio);
199+ $('#vits_emotion').val(this.settings.emotion);
200+ $('#vits_text_prompt').val(this.settings.text_prompt);
201+ $('#vits_style_text').val(this.settings.style_text);
202+ $('#vits_style_weight').val(this.settings.style_weight);
203+
204+ // Update the UI to reflect changes
205+ $('#vits_length_output').text(this.settings.length);
206+ $('#vits_noise_output').text(this.settings.noise);
207+ $('#vits_noisew_output').text(this.settings.noisew);
208+ $('#vits_segment_size_output').text(this.settings.segment_size);
209+ $('#vits_sdp_ratio_output').text(this.settings.sdp_ratio);
210+ $('#vits_emotion_output').text(this.settings.emotion);
211+ $('#vits_style_weight_output').text(this.settings.style_weight);
212+
213+ // Register input/change event listeners to update settings on user interaction
214+ $('#vits_endpoint').on('input', () => { this.onSettingsChange(); });
215+ $('#vits_lang').on('change', () => { this.onSettingsChange(); });
216+ $('#vits_format').on('change', () => { this.onSettingsChange(); });
217+ $('#vits_length').on('change', () => { this.onSettingsChange(); });
218+ $('#vits_noise').on('change', () => { this.onSettingsChange(); });
219+ $('#vits_noisew').on('change', () => { this.onSettingsChange(); });
220+ $('#vits_segment_size').on('change', () => { this.onSettingsChange(); });
221+ $('#vits_streaming').on('change', () => { this.onSettingsChange(); });
222+ $('#vits_dim_emotion').on('change', () => { this.onSettingsChange(); });
223+ $('#vits_sdp_ratio').on('change', () => { this.onSettingsChange(); });
224+ $('#vits_emotion').on('change', () => { this.onSettingsChange(); });
225+ $('#vits_text_prompt').on('change', () => { this.onSettingsChange(); });
226+ $('#vits_style_text').on('change', () => { this.onSettingsChange(); });
227+ $('#vits_style_weight').on('change', () => { this.onSettingsChange(); });
228+
229+ await this.checkReady();
230+
231+ console.info('VITS: Settings loaded');
232+ }
233+
234+ // Perform a simple readiness check by trying to fetch voiceIds
235+ async checkReady() {
236+ await Promise.allSettled([this.fetchTtsVoiceObjects(), this.changeTTSSettings()]);
237+ }
238+
239+ async onRefreshClick() {
240+ return;
241+ }
242+
243+ //#################//
244+ // TTS Interfaces //
245+ //#################//
246+
247+ async getVoice(voiceName) {
248+ if (this.voices.length == 0) {
249+ this.voices = await this.fetchTtsVoiceObjects();
250+ }
251+ const match = this.voices.filter(
252+ v => v.name == voiceName,
253+ )[0];
254+ if (!match) {
255+ throw `TTS Voice name ${voiceName} not found`;
256+ }
257+ return match;
258+ }
259+
260+ async getVoiceById(voiceId) {
261+ if (this.voices.length == 0) {
262+ this.voices = await this.fetchTtsVoiceObjects();
263+ }
264+ const match = this.voices.filter(
265+ v => v.voice_id == voiceId,
266+ )[0];
267+ if (!match) {
268+ throw `TTS Voice id ${voiceId} not found`;
269+ }
270+ return match;
271+ }
272+
273+ async generateTts(text, voiceId) {
274+ const response = await this.fetchTtsGeneration(text, voiceId);
275+ return response;
276+ }
277+
278+ //###########//
279+ // API CALLS //
280+ //###########//
281+ async fetchTtsVoiceObjects() {
282+ const response = await fetch(`${this.settings.provider_endpoint}/voice/speakers`);
283+ if (!response.ok) {
284+ throw new Error(`HTTP ${response.status}: ${await response.json()}`);
285+ }
286+ const jsonData = await response.json();
287+ const voices = [];
288+
289+ const addVoices = (modelType) => {
290+ jsonData[modelType].forEach(voice => {
291+ voices.push({
292+ name: `[${modelType}] ${voice.name} (${voice.lang})`,
293+ voice_id: `${modelType}&${voice.id}`,
294+ preview_url: false,
295+ lang: voice.lang,
296+ });
297+ });
298+ };
299+ for (const key in this.modelTypes) {
300+ addVoices(this.modelTypes[key]);
301+ }
302+
303+ this.voices = voices; // Assign to the class property
304+ return voices; // Also return this list
305+ }
306+
307+ // Each time a parameter is changed, we change the configuration
308+ async changeTTSSettings() {
309+ }
310+
311+ /**
312+ * Fetch TTS generation from the API.
313+ * @param {string} inputText Text to generate TTS for
314+ * @param {string} voiceId Voice ID to use (model_type&speaker_id))
315+ * @returns {Promise<Response|string>} Fetch response
316+ */
317+ async fetchTtsGeneration(inputText, voiceId, lang = null, forceNoStreaming = false) {
318+ console.info(`Generating new TTS for voice_id ${voiceId}`);
319+
320+ const streaming = !forceNoStreaming && this.settings.streaming;
321+ const [model_type, speaker_id] = voiceId.split('&');
322+ const params = new URLSearchParams();
323+ params.append('text', inputText);
324+ params.append('id', speaker_id);
325+ if (streaming) {
326+ params.append('streaming', streaming);
327+ // Streaming response only supports MP3
328+ }
329+ else {
330+ params.append('format', this.settings.format);
331+ }
332+ params.append('lang', lang ?? this.settings.lang);
333+ params.append('length', this.settings.length);
334+ params.append('noise', this.settings.noise);
335+ params.append('noisew', this.settings.noisew);
336+ params.append('segment_size', this.settings.segment_size);
337+
338+ if (model_type == this.modelTypes.W2V2_VITS) {
339+ params.append('emotion', this.settings.dim_emotion);
340+ }
341+ else if (model_type == this.modelTypes.BERT_VITS2) {
342+ params.append('sdp_ratio', this.settings.sdp_ratio);
343+ params.append('emotion', this.settings.emotion);
344+ if (this.settings.text_prompt) {
345+ params.append('text_prompt', this.settings.text_prompt);
346+ }
347+ if (this.settings.style_text) {
348+ params.append('style_text', this.settings.style_text);
349+ params.append('style_weight', this.settings.style_weight);
350+ }
351+ }
352+
353+ const url = `${this.settings.provider_endpoint}/voice/${model_type.toLowerCase()}`;
354+
355+ if (streaming) {
356+ return url + `?${params.toString()}`;
357+ }
358+
359+ const response = await fetch(
360+ url,
361+ {
362+ method: 'POST',
363+ headers: {
364+ 'Content-Type': 'application/x-www-form-urlencoded',
365+ },
366+ body: params,
367+ },
368+ );
369+ if (!response.ok) {
370+ toastr.error(response.statusText, 'TTS Generation Failed');
371+ throw new Error(`HTTP ${response.status}: ${await response.text()}`);
372+ }
373+ return response;
374+ }
375+
376+ /**
377+ * Preview TTS for a given voice ID.
378+ * @param {string} id Voice ID
379+ */
380+ async previewTtsVoice(id) {
381+ this.audioElement.pause();
382+ this.audioElement.currentTime = 0;
383+ const voice = await this.getVoiceById(id);
384+ const lang = voice.lang.includes(this.settings.lang) ? this.settings.lang : voice.lang[0];
385+
386+ let lang_code = this.langKey2LangCode[lang];
387+ const text = getPreviewString(lang_code);
388+ const response = await this.fetchTtsGeneration(text, id, lang, true);
389+ if (typeof response != 'string') {
390+ if (!response.ok) {
391+ throw new Error(`HTTP ${response.status}: ${await response.text()}`);
392+ }
393+ const audio = await response.blob();
394+ const url = URL.createObjectURL(audio);
395+ this.audioElement.src = url;
396+ this.audioElement.play();
397+ }
398+ }
399+
400+ // Interface not used
401+ async fetchTtsFromHistory(history_item_id) {
402+ return Promise.resolve(history_item_id);
403+ }
404+}