TTS: Add support for VITS/W2V2-VITS/Bert-VITS2 (#2439) * add VITSTtsProvider * add options * fix params, drop reference_audio_path * post with body * improve preview lang * add space in label

bb39e852b88eca7cc5ba5caf7ac8565ec2e285fb

Risenafis <91325858+Risenafis@users.noreply.github.com>

Signed
Parents: 75099d3a2260476b877d904119a0241902beeeaa
2 files changed, +406 -0Ignore whitespace
public/scripts/extensions/tts/index.js+2 -0
@@ -10,6 +10,7 @@ import { NovelTtsProvider } from './novel.js';
10import { power_user } from '../../power-user.js';10import { power_user } from '../../power-user.js';
11import { OpenAITtsProvider } from './openai.js';11import { OpenAITtsProvider } from './openai.js';
12import { XTTSTtsProvider } from './xtts.js';12import { XTTSTtsProvider } from './xtts.js';
13import { VITSTtsProvider } from './vits.js';
13import { GSVITtsProvider } from './gsvi.js';14import { GSVITtsProvider } from './gsvi.js';
14import { SBVits2TtsProvider } from './sbvits2.js';15import { SBVits2TtsProvider } from './sbvits2.js';
15import { AllTalkTtsProvider } from './alltalk.js';16import { AllTalkTtsProvider } from './alltalk.js';
@@ -83,6 +84,7 @@ const ttsProviders = {
83 ElevenLabs: ElevenLabsTtsProvider,84 ElevenLabs: ElevenLabsTtsProvider,
84 Silero: SileroTtsProvider,85 Silero: SileroTtsProvider,
85 XTTSv2: XTTSTtsProvider,86 XTTSv2: XTTSTtsProvider,
87 VITS: VITSTtsProvider,
86 GSVI: GSVITtsProvider,88 GSVI: GSVITtsProvider,
87 SBVits2: SBVits2TtsProvider,89 SBVits2: SBVits2TtsProvider,
88 System: SystemTtsProvider,90 System: SystemTtsProvider,
public/scripts/extensions/tts/vits.js+404 -0
@@ -0,0 +1,404 @@
1import { getPreviewString, saveTtsProviderSettings } from './index.js';
2
3export { VITSTtsProvider };
4
5class VITSTtsProvider {
6 //########//
7 // Config //
8 //########//
9
10 settings;
11 ready = false;
12 voices = [];
13 separator = '. ';
14 audioElement = document.createElement('audio');
15
16 /**
17 * Perform any text processing before passing to TTS engine.
18 * @param {string} text Input text
19 * @returns {string} Processed text
20 */
21 processText(text) {
22 return text;
23 }
24
25 audioFormats = ['wav', 'ogg', 'silk', 'mp3', 'flac'];
26
27 languageLabels = {
28 'Auto': 'auto',
29 'Chinese': 'zh',
30 'English': 'en',
31 'Japanese': 'ja',
32 'Korean': 'ko',
33 };
34
35 langKey2LangCode = {
36 'zh': 'zh-CN',
37 'en': 'en-US',
38 'ja': 'ja-JP',
39 'ko': 'ko-KR',
40 };
41
42 modelTypes = {
43 VITS: 'VITS',
44 W2V2_VITS: 'W2V2-VITS',
45 BERT_VITS2: 'BERT-VITS2',
46 };
47
48 defaultSettings = {
49 provider_endpoint: 'http://localhost:23456',
50 format: 'wav',
51 lang: 'auto',
52 length: 1.0,
53 noise: 0.33,
54 noisew: 0.4,
55 segment_size: 50,
56 streaming: false,
57 dim_emotion: 0,
58 sdp_ratio: 0.2,
59 emotion: 0,
60 text_prompt: '',
61 style_text: '',
62 style_weight: 1,
63 };
64
65 get settingsHtml() {
66 let html = `
67 <label for="vits_lang">Text Language</label>
68 <select id="vits_lang">`;
69
70 for (let language in this.languageLabels) {
71 if (this.languageLabels[language] == this.settings?.lang) {
72 html += `<option value="${this.languageLabels[language]}" selected="selected">${language}</option>`;
73 continue;
74 }
75 html += `<option value="${this.languageLabels[language]}">${language}</option>`;
76 }
77
78 html += `
79 </select>
80 <label>VITS / W2V2-VITS / Bert-VITS2 Settings:</label><br/>
81 <label for="vits_endpoint">Provider Endpoint:</label>
82 <input id="vits_endpoint" type="text" class="text_pole" maxlength="250" value="${this.defaultSettings.provider_endpoint}"/>
83 <span>Use <a target="_blank" href="https://github.com/Artrajz/vits-simple-api">vits-simple-api</a>.</span><br/>
84
85 <label for="vits_format">Audio format:</label>
86 <select id="vits_format">`;
87
88 for (let format of this.audioFormats) {
89 if (format == this.settings?.format) {
90 html += `<option value="${format}" selected="selected">${format}</option>`;
91 continue;
92 }
93 html += `<option value="${format}">${format}</option>`;
94 }
95
96 html += `
97 </select>
98 <label for="vits_length">Audio length: <span id="vits_length_output">${this.defaultSettings.length}</span></label>
99 <input id="vits_length" type="range" value="${this.defaultSettings.length}" min="0.0" max="5" step="0.01" />
100
101 <label for="vits_noise">Noise: <span id="vits_noise_output">${this.defaultSettings.noise}</span></label>
102 <input id="vits_noise" type="range" value="${this.defaultSettings.noise}" min="0.1" max="2" step="0.01" />
103
104 <label for="vits_noisew">SDP noise: <span id="vits_noisew_output">${this.defaultSettings.noisew}</span></label>
105 <input id="vits_noisew" type="range" value="${this.defaultSettings.noisew}" min="0.1" max="2" step="0.01" />
106
107 <label for="vits_segment_size">Segment Size: <span id="vits_segment_size_output">${this.defaultSettings.segment_size}</span></label>
108 <input id="vits_segment_size" type="range" value="${this.defaultSettings.segment_size}" min="0" max="1000" step="1" />
109
110 <label for="vits_streaming" class="checkbox_label">
111 <input id="vits_streaming" type="checkbox" />
112 <span>Streaming</span>
113 </label>
114
115 <label>W2V2-VITS Settings:</label><br/>
116 <label for="vits_dim_emotion">Dimensional emotion:</label>
117 <input id="vits_dim_emotion" type="number" class="text_pole" min="0" max="5457" step="1" value="${this.defaultSettings.dim_emotion}"/>
118
119 <label>BERT-VITS2 Settings:</label><br/>
120 <label for="vits_sdp_ratio">sdp_ratio: <span id="vits_sdp_ratio_output">${this.defaultSettings.sdp_ratio}</span></label>
121 <input id="vits_sdp_ratio" type="range" value="${this.defaultSettings.sdp_ratio}" min="0.0" max="1" step="0.01" />
122
123 <label for="vits_emotion">emotion: <span id="vits_emotion_output">${this.defaultSettings.emotion}</span></label>
124 <input id="vits_emotion" type="range" value="${this.defaultSettings.emotion}" min="0" max="9" step="1" />
125
126 <label for="vits_text_prompt">Text Prompt:</label>
127 <input id="vits_text_prompt" type="text" class="text_pole" maxlength="512" value="${this.defaultSettings.text_prompt}"/>
128
129 <label for="vits_style_text">Style text:</label>
130 <input id="vits_style_text" type="text" class="text_pole" maxlength="512" value="${this.defaultSettings.style_text}"/>
131
132 <label for="vits_style_weight">Style weight <span id="vits_style_weight_output">${this.defaultSettings.style_weight}</span></label>
133 <input id="vits_style_weight" type="range" value="${this.defaultSettings.style_weight}" min="0" max="1" step="0.01" />
134 `;
135
136 return html;
137 }
138
139 onSettingsChange() {
140 // Used when provider settings are updated from UI
141 this.settings.provider_endpoint = $('#vits_endpoint').val();
142 this.settings.lang = $('#vits_lang').val();
143 this.settings.format = $('#vits_format').val();
144 this.settings.dim_emotion = $('#vits_dim_emotion').val();
145 this.settings.text_prompt = $('#vits_text_prompt').val();
146 this.settings.style_text = $('#vits_style_text').val();
147
148 // Update the default TTS settings based on input fields
149 this.settings.length = $('#vits_length').val();
150 this.settings.noise = $('#vits_noise').val();
151 this.settings.noisew = $('#vits_noisew').val();
152 this.settings.segment_size = $('#vits_segment_size').val();
153 this.settings.streaming = $('#vits_streaming').is(':checked');
154 this.settings.sdp_ratio = $('#vits_sdp_ratio').val();
155 this.settings.emotion = $('#vits_emotion').val();
156 this.settings.style_weight = $('#vits_style_weight').val();
157
158 // Update the UI to reflect changes
159 $('#vits_length_output').text(this.settings.length);
160 $('#vits_noise_output').text(this.settings.noise);
161 $('#vits_noisew_output').text(this.settings.noisew);
162 $('#vits_segment_size_output').text(this.settings.segment_size);
163 $('#vits_sdp_ratio_output').text(this.settings.sdp_ratio);
164 $('#vits_emotion_output').text(this.settings.emotion);
165 $('#vits_style_weight_output').text(this.settings.style_weight);
166
167 saveTtsProviderSettings();
168 this.changeTTSSettings();
169 }
170
171 async loadSettings(settings) {
172 // Pupulate Provider UI given input settings
173 if (Object.keys(settings).length == 0) {
174 console.info('Using default TTS Provider settings');
175 }
176
177 // Only accept keys defined in defaultSettings
178 this.settings = this.defaultSettings;
179
180 for (const key in settings) {
181 if (key in this.settings) {
182 this.settings[key] = settings[key];
183 } else {
184 console.debug(`Ignoring non-user-configurable setting: ${key}`);
185 }
186 }
187
188 // Set initial values from the settings
189 $('#vits_endpoint').val(this.settings.provider_endpoint);
190 $('#vits_lang').val(this.settings.lang);
191 $('#vits_format').val(this.settings.format);
192 $('#vits_length').val(this.settings.length);
193 $('#vits_noise').val(this.settings.noise);
194 $('#vits_noisew').val(this.settings.noisew);
195 $('#vits_segment_size').val(this.settings.segment_size);
196 $('#vits_streaming').prop('checked', this.settings.streaming);
197 $('#vits_dim_emotion').val(this.settings.dim_emotion);
198 $('#vits_sdp_ratio').val(this.settings.sdp_ratio);
199 $('#vits_emotion').val(this.settings.emotion);
200 $('#vits_text_prompt').val(this.settings.text_prompt);
201 $('#vits_style_text').val(this.settings.style_text);
202 $('#vits_style_weight').val(this.settings.style_weight);
203
204 // Update the UI to reflect changes
205 $('#vits_length_output').text(this.settings.length);
206 $('#vits_noise_output').text(this.settings.noise);
207 $('#vits_noisew_output').text(this.settings.noisew);
208 $('#vits_segment_size_output').text(this.settings.segment_size);
209 $('#vits_sdp_ratio_output').text(this.settings.sdp_ratio);
210 $('#vits_emotion_output').text(this.settings.emotion);
211 $('#vits_style_weight_output').text(this.settings.style_weight);
212
213 // Register input/change event listeners to update settings on user interaction
214 $('#vits_endpoint').on('input', () => { this.onSettingsChange(); });
215 $('#vits_lang').on('change', () => { this.onSettingsChange(); });
216 $('#vits_format').on('change', () => { this.onSettingsChange(); });
217 $('#vits_length').on('change', () => { this.onSettingsChange(); });
218 $('#vits_noise').on('change', () => { this.onSettingsChange(); });
219 $('#vits_noisew').on('change', () => { this.onSettingsChange(); });
220 $('#vits_segment_size').on('change', () => { this.onSettingsChange(); });
221 $('#vits_streaming').on('change', () => { this.onSettingsChange(); });
222 $('#vits_dim_emotion').on('change', () => { this.onSettingsChange(); });
223 $('#vits_sdp_ratio').on('change', () => { this.onSettingsChange(); });
224 $('#vits_emotion').on('change', () => { this.onSettingsChange(); });
225 $('#vits_text_prompt').on('change', () => { this.onSettingsChange(); });
226 $('#vits_style_text').on('change', () => { this.onSettingsChange(); });
227 $('#vits_style_weight').on('change', () => { this.onSettingsChange(); });
228
229 await this.checkReady();
230
231 console.info('VITS: Settings loaded');
232 }
233
234 // Perform a simple readiness check by trying to fetch voiceIds
235 async checkReady() {
236 await Promise.allSettled([this.fetchTtsVoiceObjects(), this.changeTTSSettings()]);
237 }
238
239 async onRefreshClick() {
240 return;
241 }
242
243 //#################//
244 // TTS Interfaces //
245 //#################//
246
247 async getVoice(voiceName) {
248 if (this.voices.length == 0) {
249 this.voices = await this.fetchTtsVoiceObjects();
250 }
251 const match = this.voices.filter(
252 v => v.name == voiceName,
253 )[0];
254 if (!match) {
255 throw `TTS Voice name ${voiceName} not found`;
256 }
257 return match;
258 }
259
260 async getVoiceById(voiceId) {
261 if (this.voices.length == 0) {
262 this.voices = await this.fetchTtsVoiceObjects();
263 }
264 const match = this.voices.filter(
265 v => v.voice_id == voiceId,
266 )[0];
267 if (!match) {
268 throw `TTS Voice id ${voiceId} not found`;
269 }
270 return match;
271 }
272
273 async generateTts(text, voiceId) {
274 const response = await this.fetchTtsGeneration(text, voiceId);
275 return response;
276 }
277
278 //###########//
279 // API CALLS //
280 //###########//
281 async fetchTtsVoiceObjects() {
282 const response = await fetch(`${this.settings.provider_endpoint}/voice/speakers`);
283 if (!response.ok) {
284 throw new Error(`HTTP ${response.status}: ${await response.json()}`);
285 }
286 const jsonData = await response.json();
287 const voices = [];
288
289 const addVoices = (modelType) => {
290 jsonData[modelType].forEach(voice => {
291 voices.push({
292 name: `[${modelType}] ${voice.name} (${voice.lang})`,
293 voice_id: `${modelType}&${voice.id}`,
294 preview_url: false,
295 lang: voice.lang,
296 });
297 });
298 };
299 for (const key in this.modelTypes) {
300 addVoices(this.modelTypes[key]);
301 }
302
303 this.voices = voices; // Assign to the class property
304 return voices; // Also return this list
305 }
306
307 // Each time a parameter is changed, we change the configuration
308 async changeTTSSettings() {
309 }
310
311 /**
312 * Fetch TTS generation from the API.
313 * @param {string} inputText Text to generate TTS for
314 * @param {string} voiceId Voice ID to use (model_type&speaker_id))
315 * @returns {Promise<Response|string>} Fetch response
316 */
317 async fetchTtsGeneration(inputText, voiceId, lang = null, forceNoStreaming = false) {
318 console.info(`Generating new TTS for voice_id ${voiceId}`);
319
320 const streaming = !forceNoStreaming && this.settings.streaming;
321 const [model_type, speaker_id] = voiceId.split('&');
322 const params = new URLSearchParams();
323 params.append('text', inputText);
324 params.append('id', speaker_id);
325 if (streaming) {
326 params.append('streaming', streaming);
327 // Streaming response only supports MP3
328 }
329 else {
330 params.append('format', this.settings.format);
331 }
332 params.append('lang', lang ?? this.settings.lang);
333 params.append('length', this.settings.length);
334 params.append('noise', this.settings.noise);
335 params.append('noisew', this.settings.noisew);
336 params.append('segment_size', this.settings.segment_size);
337
338 if (model_type == this.modelTypes.W2V2_VITS) {
339 params.append('emotion', this.settings.dim_emotion);
340 }
341 else if (model_type == this.modelTypes.BERT_VITS2) {
342 params.append('sdp_ratio', this.settings.sdp_ratio);
343 params.append('emotion', this.settings.emotion);
344 if (this.settings.text_prompt) {
345 params.append('text_prompt', this.settings.text_prompt);
346 }
347 if (this.settings.style_text) {
348 params.append('style_text', this.settings.style_text);
349 params.append('style_weight', this.settings.style_weight);
350 }
351 }
352
353 const url = `${this.settings.provider_endpoint}/voice/${model_type.toLowerCase()}`;
354
355 if (streaming) {
356 return url + `?${params.toString()}`;
357 }
358
359 const response = await fetch(
360 url,
361 {
362 method: 'POST',
363 headers: {
364 'Content-Type': 'application/x-www-form-urlencoded',
365 },
366 body: params,
367 },
368 );
369 if (!response.ok) {
370 toastr.error(response.statusText, 'TTS Generation Failed');
371 throw new Error(`HTTP ${response.status}: ${await response.text()}`);
372 }
373 return response;
374 }
375
376 /**
377 * Preview TTS for a given voice ID.
378 * @param {string} id Voice ID
379 */
380 async previewTtsVoice(id) {
381 this.audioElement.pause();
382 this.audioElement.currentTime = 0;
383 const voice = await this.getVoiceById(id);
384 const lang = voice.lang.includes(this.settings.lang) ? this.settings.lang : voice.lang[0];
385
386 let lang_code = this.langKey2LangCode[lang];
387 const text = getPreviewString(lang_code);
388 const response = await this.fetchTtsGeneration(text, id, lang, true);
389 if (typeof response != 'string') {
390 if (!response.ok) {
391 throw new Error(`HTTP ${response.status}: ${await response.text()}`);
392 }
393 const audio = await response.blob();
394 const url = URL.createObjectURL(audio);
395 this.audioElement.src = url;
396 this.audioElement.play();
397 }
398 }
399
400 // Interface not used
401 async fetchTtsFromHistory(history_item_id) {
402 return Promise.resolve(history_item_id);
403 }
404}