Blame Raw
· · · 566 lines (24.2 KB)
0 contributors
1import { getPreviewString, saveTtsProviderSettings } from './index.js';
2
3export { TtsWebuiProvider };
4
5class TtsWebuiProvider {
6 settings;
7 voices = [];
8 separator = ' . ';
9
10 audioElement = document.createElement('audio');
11 audioContext = null;
12 audioWorkletNode = null;
13 currentVolume = 1.0; // Track current volume
14
15 defaultSettings = {
16 voiceMap: {},
17 model: 'chatterbox',
18 speed: 1,
19 volume: 1.0,
20 available_voices: [''],
21 provider_endpoint: 'http://127.0.0.1:7778/v1/audio/speech',
22 streaming: true,
23 stream_chunk_size: 100,
24 desired_length: 80,
25 max_length: 200,
26 halve_first_chunk: true,
27 exaggeration: 0.5,
28 cfg_weight: 0.5,
29 temperature: 0.8,
30 device: 'auto',
31 dtype: 'float32',
32 cpu_offload: false,
33 chunked: true,
34 cache_voice: false,
35 tokens_per_slice: 1000,
36 remove_milliseconds: 45,
37 remove_milliseconds_start: 25,
38 chunk_overlap_method: 'zero',
39 seed: -1,
40 };
41
42 get settingsHtml() {
43 let html = `
44 <h4 class="textAlignCenter">TTS WebUI Settings</h4>
45
46 <div class="flex gap10px marginBot10 alignItemsFlexEnd">
47 <div class="flex1 flexFlowColumn">
48 <label for="tts_webui_endpoint">Provider Endpoint:</label>
49 <input id="tts_webui_endpoint" type="text" class="text_pole" maxlength="500" value="${this.defaultSettings.provider_endpoint}"/>
50 </div>
51 </div>
52
53 <div class="flex gap10px marginBot10">
54 <div class="flex1 flexFlowColumn">
55 <label for="tts_webui_model">Model:</label>
56 <input id="tts_webui_model" type="text" class="text_pole" maxlength="500" value="${this.defaultSettings.model}"/>
57 </div>
58 <div class="flex1 flexFlowColumn">
59 <label for="tts_webui_voices">Available Voices (comma separated):</label>
60 <input id="tts_webui_voices" type="text" class="text_pole" value="${this.defaultSettings.available_voices.join()}"/>
61 </div>
62 </div>
63
64 <div class="flex gap10px marginBot10">
65 <div class="flex1 flexFlowColumn">
66 <label for="tts_webui_streaming" class="checkbox_label alignItemsCenter flexGap5">
67 <input id="tts_webui_streaming" type="checkbox" />
68 <span>Streaming</span>
69 </label>
70 </div>
71 <div class="flex1 flexFlowColumn">
72 <label for="tts_webui_volume">Volume: <span id="tts_webui_volume_output">${this.defaultSettings.volume}</span></label>
73 <input type="range" id="tts_webui_volume" value="${this.defaultSettings.volume}" min="0" max="2" step="0.1">
74 </div>
75 </div>
76
77 <hr>
78 <h4 class="textAlignCenter">Generation Settings</h4>
79
80 <div class="flex gap10px marginBot10">
81 <div class="flex1 flexFlowColumn">
82 <label for="tts_webui_exaggeration">Exaggeration: <span id="tts_webui_exaggeration_output">${this.defaultSettings.exaggeration}</span></label>
83 <input id="tts_webui_exaggeration" type="range" value="${this.defaultSettings.exaggeration}" min="0" max="2" step="0.1" />
84 </div>
85 <div class="flex1 flexFlowColumn">
86 <label for="tts_webui_cfg_weight">CFG Weight: <span id="tts_webui_cfg_weight_output">${this.defaultSettings.cfg_weight}</span></label>
87 <input id="tts_webui_cfg_weight" type="range" value="${this.defaultSettings.cfg_weight}" min="0" max="2" step="0.1" />
88 </div>
89 </div>
90
91 <div class="flex gap10px marginBot10">
92 <div class="flex1 flexFlowColumn">
93 <label for="tts_webui_temperature">Temperature: <span id="tts_webui_temperature_output">${this.defaultSettings.temperature}</span></label>
94 <input id="tts_webui_temperature" type="range" value="${this.defaultSettings.temperature}" min="0" max="2" step="0.1" />
95 </div>
96 <div class="flex1 flexFlowColumn">
97 <label for="tts_webui_seed">Seed (-1 for random):</label>
98 <input id="tts_webui_seed" type="text" class="text_pole" value="${this.defaultSettings.seed}"/>
99 </div>
100 </div>
101
102 <hr>
103 <h4 class="textAlignCenter">Chunking</h4>
104
105 <div class="flex gap10px marginBot10">
106 <div class="flex1 flexFlowColumn">
107 <label for="tts_webui_chunked" class="checkbox_label alignItemsCenter flexGap5">
108 <input id="tts_webui_chunked" type="checkbox" />
109 <span>Split prompt into chunks</span>
110 </label>
111 </div>
112 <div class="flex1 flexFlowColumn">
113 <label for="tts_webui_halve_first_chunk" class="checkbox_label alignItemsCenter flexGap5">
114 <input id="tts_webui_halve_first_chunk" type="checkbox" />
115 <span>Halve First Chunk</span>
116 </label>
117 </div>
118 </div>
119
120 <div class="flex gap10px marginBot10">
121 <div class="flex1 flexFlowColumn">
122 <label for="tts_webui_desired_length">Desired Length: <span id="tts_webui_desired_length_output">${this.defaultSettings.desired_length}</span></label>
123 <input id="tts_webui_desired_length" type="range" value="${this.defaultSettings.desired_length}" min="25" max="300" step="5" />
124 </div>
125 <div class="flex1 flexFlowColumn">
126 <label for="tts_webui_max_length">Max Length: <span id="tts_webui_max_length_output">${this.defaultSettings.max_length}</span></label>
127 <input id="tts_webui_max_length" type="range" value="${this.defaultSettings.max_length}" min="50" max="450" step="5" />
128 </div>
129 </div>
130
131 <hr>
132 <h4 class="textAlignCenter">Model</h4>
133
134 <div class="flex gap10px marginBot10">
135 <div class="flex1 flexFlowColumn">
136 <label for="tts_webui_device">Device:</label>
137 <select id="tts_webui_device">
138 <option value="auto" ${this.defaultSettings.device === 'auto' ? 'selected' : ''}>Auto</option>
139 <option value="cuda" ${this.defaultSettings.device === 'cuda' ? 'selected' : ''}>CUDA</option>
140 <option value="mps" ${this.defaultSettings.device === 'mps' ? 'selected' : ''}>MPS</option>
141 <option value="cpu" ${this.defaultSettings.device === 'cpu' ? 'selected' : ''}>CPU</option>
142 </select>
143 </div>
144 <div class="flex1 flexFlowColumn">
145 <label for="tts_webui_dtype">Data Type:</label>
146 <select id="tts_webui_dtype">
147 <option value="float32" ${this.defaultSettings.dtype === 'float32' ? 'selected' : ''}>Float32</option>
148 <option value="float16" ${this.defaultSettings.dtype === 'float16' ? 'selected' : ''}>Float16</option>
149 <option value="bfloat16" ${this.defaultSettings.dtype === 'bfloat16' ? 'selected' : ''}>BFloat16</option>
150 </select>
151 </div>
152 </div>
153
154 <div class="flex gap10px marginBot10">
155 <div class="flex1 flexFlowColumn">
156 <label for="tts_webui_cpu_offload" class="checkbox_label alignItemsCenter flexGap5">
157 <input id="tts_webui_cpu_offload" type="checkbox" />
158 <span>CPU Offload</span>
159 </label>
160 </div>
161 <div class="flex1">
162 <!-- Empty for spacing -->
163 </div>
164 </div>
165
166 <hr>
167 <h4 class="textAlignCenter">Streaming (Advanced Settings)</h4>
168
169 <div class="flex gap10px marginBot10">
170 <div class="flex1 flexFlowColumn">
171 <label for="tts_webui_tokens_per_slice">Tokens Per Slice: <span id="tts_webui_tokens_per_slice_output">${this.defaultSettings.tokens_per_slice}</span></label>
172 <input id="tts_webui_tokens_per_slice" type="range" value="${this.defaultSettings.tokens_per_slice}" min="15" max="1000" step="1" />
173 </div>
174 <div class="flex1 flexFlowColumn">
175 <label for="tts_webui_chunk_overlap_method">Chunk Overlap Method:</label>
176 <select id="tts_webui_chunk_overlap_method">
177 <option value="zero" ${this.defaultSettings.chunk_overlap_method === 'zero' ? 'selected' : ''}>Zero</option>
178 <option value="full" ${this.defaultSettings.chunk_overlap_method === 'full' ? 'selected' : ''}>Full</option>
179 </select>
180 </div>
181 </div>
182
183 <div class="flex gap10px marginBot10">
184 <div class="flex1 flexFlowColumn">
185 <label for="tts_webui_remove_milliseconds">Remove Milliseconds: <span id="tts_webui_remove_milliseconds_output">${this.defaultSettings.remove_milliseconds}</span></label>
186 <input id="tts_webui_remove_milliseconds" type="range" value="${this.defaultSettings.remove_milliseconds}" min="0" max="100" step="1" />
187 </div>
188 <div class="flex1 flexFlowColumn">
189 <label for="tts_webui_remove_milliseconds_start">Remove Milliseconds Start: <span id="tts_webui_remove_milliseconds_start_output">${this.defaultSettings.remove_milliseconds_start}</span></label>
190 <input id="tts_webui_remove_milliseconds_start" type="range" value="${this.defaultSettings.remove_milliseconds_start}" min="0" max="100" step="1" />
191 </div>
192 </div>`;
193 return html;
194 }
195
196 async loadSettings(settings) {
197 // Populate Provider UI given input settings
198 if (Object.keys(settings).length == 0) {
199 console.info('Using default TTS Provider settings');
200 }
201
202 // Only accept keys defined in defaultSettings
203 this.settings = this.defaultSettings;
204
205 for (const key in settings) {
206 if (key in this.settings) {
207 this.settings[key] = settings[key];
208 } else {
209 throw `Invalid setting passed to TTS Provider: ${key}`;
210 }
211 }
212
213 $('#tts_webui_endpoint').val(this.settings.provider_endpoint);
214 $('#tts_webui_endpoint').on('input', () => { this.onSettingsChange(); });
215
216 $('#tts_webui_model').val(this.settings.model);
217 $('#tts_webui_model').on('input', () => { this.onSettingsChange(); });
218
219 $('#tts_webui_voices').val(this.settings.available_voices.join());
220 $('#tts_webui_voices').on('input', () => { this.onSettingsChange(); });
221
222 $('#tts_webui_streaming').prop('checked', this.settings.streaming);
223 $('#tts_webui_streaming').on('change', () => { this.onSettingsChange(); });
224
225 $('#tts_webui_volume').val(this.settings.volume);
226 $('#tts_webui_volume').on('input', () => {
227 this.onSettingsChange();
228 });
229
230 $('#tts_webui_stream_chunk_size').val(this.settings.stream_chunk_size);
231 $('#tts_webui_stream_chunk_size').on('input', () => { this.onSettingsChange(); });
232
233 $('#tts_webui_desired_length').val(this.settings.desired_length);
234 $('#tts_webui_desired_length').on('input', () => { this.onSettingsChange(); });
235
236 $('#tts_webui_max_length').val(this.settings.max_length);
237 $('#tts_webui_max_length').on('input', () => { this.onSettingsChange(); });
238
239 $('#tts_webui_halve_first_chunk').prop('checked', this.settings.halve_first_chunk);
240 $('#tts_webui_halve_first_chunk').on('change', () => { this.onSettingsChange(); });
241
242 $('#tts_webui_exaggeration').val(this.settings.exaggeration);
243 $('#tts_webui_exaggeration').on('input', () => { this.onSettingsChange(); });
244
245 $('#tts_webui_cfg_weight').val(this.settings.cfg_weight);
246 $('#tts_webui_cfg_weight').on('input', () => { this.onSettingsChange(); });
247
248 $('#tts_webui_temperature').val(this.settings.temperature);
249 $('#tts_webui_temperature').on('input', () => { this.onSettingsChange(); });
250
251 $('#tts_webui_device').val(this.settings.device);
252 $('#tts_webui_device').on('change', () => { this.onSettingsChange(); });
253
254 $('#tts_webui_dtype').val(this.settings.dtype);
255 $('#tts_webui_dtype').on('change', () => { this.onSettingsChange(); });
256
257 $('#tts_webui_cpu_offload').prop('checked', this.settings.cpu_offload);
258 $('#tts_webui_cpu_offload').on('change', () => { this.onSettingsChange(); });
259
260 $('#tts_webui_chunked').prop('checked', this.settings.chunked);
261 $('#tts_webui_chunked').on('change', () => { this.onSettingsChange(); });
262
263 $('#tts_webui_tokens_per_slice').val(this.settings.tokens_per_slice);
264 $('#tts_webui_tokens_per_slice').on('input', () => { this.onSettingsChange(); });
265
266 $('#tts_webui_remove_milliseconds').val(this.settings.remove_milliseconds);
267 $('#tts_webui_remove_milliseconds').on('input', () => { this.onSettingsChange(); });
268
269 $('#tts_webui_remove_milliseconds_start').val(this.settings.remove_milliseconds_start);
270 $('#tts_webui_remove_milliseconds_start').on('input', () => { this.onSettingsChange(); });
271
272 $('#tts_webui_chunk_overlap_method').val(this.settings.chunk_overlap_method);
273 $('#tts_webui_chunk_overlap_method').on('change', () => { this.onSettingsChange(); });
274
275 $('#tts_webui_seed').val(this.settings.seed);
276 $('#tts_webui_seed').on('input', () => { this.onSettingsChange(); });
277
278 // Update output labels
279 $('#tts_webui_volume_output').text(this.settings.volume);
280 $('#tts_webui_desired_length_output').text(this.settings.desired_length);
281 $('#tts_webui_max_length_output').text(this.settings.max_length);
282 $('#tts_webui_exaggeration_output').text(this.settings.exaggeration);
283 $('#tts_webui_cfg_weight_output').text(this.settings.cfg_weight);
284 $('#tts_webui_temperature_output').text(this.settings.temperature);
285 $('#tts_webui_tokens_per_slice_output').text(this.settings.tokens_per_slice);
286 $('#tts_webui_remove_milliseconds_output').text(this.settings.remove_milliseconds);
287 $('#tts_webui_remove_milliseconds_start_output').text(this.settings.remove_milliseconds_start);
288
289 await this.checkReady();
290
291 console.debug('OpenAI Compatible TTS: Settings loaded');
292 }
293
294 onSettingsChange() {
295 // Update dynamically
296 this.settings.provider_endpoint = String($('#tts_webui_endpoint').val());
297 this.settings.model = String($('#tts_webui_model').val());
298 this.settings.available_voices = String($('#tts_webui_voices').val()).split(',');
299 this.settings.volume = Number($('#tts_webui_volume').val());
300 this.settings.streaming = $('#tts_webui_streaming').is(':checked');
301 this.settings.stream_chunk_size = Number($('#tts_webui_stream_chunk_size').val());
302 this.settings.desired_length = Number($('#tts_webui_desired_length').val());
303 this.settings.max_length = Number($('#tts_webui_max_length').val());
304 this.settings.halve_first_chunk = $('#tts_webui_halve_first_chunk').is(':checked');
305 this.settings.exaggeration = Number($('#tts_webui_exaggeration').val());
306 this.settings.cfg_weight = Number($('#tts_webui_cfg_weight').val());
307 this.settings.temperature = Number($('#tts_webui_temperature').val());
308 this.settings.device = String($('#tts_webui_device').val());
309 this.settings.dtype = String($('#tts_webui_dtype').val());
310 this.settings.cpu_offload = $('#tts_webui_cpu_offload').is(':checked');
311 this.settings.chunked = $('#tts_webui_chunked').is(':checked');
312 this.settings.tokens_per_slice = Number($('#tts_webui_tokens_per_slice').val());
313 this.settings.remove_milliseconds = Number($('#tts_webui_remove_milliseconds').val());
314 this.settings.remove_milliseconds_start = Number($('#tts_webui_remove_milliseconds_start').val());
315 this.settings.chunk_overlap_method = String($('#tts_webui_chunk_overlap_method').val());
316 this.settings.seed = parseInt($('#tts_webui_seed').val()) || -1;
317
318 // Apply volume change immediately
319 this.setVolume(this.settings.volume);
320
321 // Update output labels
322 $('#tts_webui_volume_output').text(this.settings.volume);
323 $('#tts_webui_desired_length_output').text(this.settings.desired_length);
324 $('#tts_webui_max_length_output').text(this.settings.max_length);
325 $('#tts_webui_exaggeration_output').text(this.settings.exaggeration);
326 $('#tts_webui_cfg_weight_output').text(this.settings.cfg_weight);
327 $('#tts_webui_temperature_output').text(this.settings.temperature);
328 $('#tts_webui_tokens_per_slice_output').text(this.settings.tokens_per_slice);
329 $('#tts_webui_remove_milliseconds_output').text(this.settings.remove_milliseconds);
330 $('#tts_webui_remove_milliseconds_start_output').text(this.settings.remove_milliseconds_start);
331
332 saveTtsProviderSettings();
333 }
334
335 async checkReady() {
336 await this.fetchTtsVoiceObjects();
337 }
338
339 async onRefreshClick() {
340 await this.fetchTtsVoiceObjects();
341 console.info('TTS voices refreshed');
342 }
343
344 async getVoice(voiceName) {
345 if (this.voices.length == 0) {
346 this.voices = await this.fetchTtsVoiceObjects();
347 }
348 const match = this.voices.filter(
349 oaicVoice => oaicVoice.name == voiceName,
350 )[0];
351 if (!match) {
352 throw `TTS Voice name ${voiceName} not found`;
353 }
354 return match;
355 }
356
357 async generateTts(text, voiceId) {
358 const response = await this.fetchTtsGeneration(text, voiceId);
359
360 if (this.settings.streaming) {
361 // Stream audio in real-time
362 await this.processStreamingAudio(response);
363 // Return empty string since audio is already played via AudioWorklet
364 return '';
365 }
366
367 return response;
368 }
369
370 async fetchTtsVoiceObjects() {
371 // Try to fetch voices from the provider endpoint
372 try {
373 const voicesEndpoint = this.settings.provider_endpoint.replace('/speech', '/voices/' + this.settings.model);
374 const response = await fetch(voicesEndpoint);
375
376 if (!response.ok) {
377 throw new Error(`HTTP ${response.status}`);
378 }
379
380 const responseJson = await response.json();
381 console.info('Discovered voices from provider:', responseJson);
382
383 this.voices = responseJson.voices.map(({ value, label }) => ({
384 name: label,
385 voice_id: value,
386 lang: 'en-US',
387 }));
388
389 return this.voices;
390 } catch (error) {
391 console.warn('Voice discovery failed, using configured voices:', error);
392 }
393
394 // Fallback to configured voices
395 this.voices = this.settings.available_voices.map(name => ({
396 name, voice_id: name, lang: 'en-US',
397 }));
398
399 return this.voices;
400 }
401
402 async initAudioWorklet(wavSampleRate) {
403 this.audioContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: wavSampleRate });
404
405 // Load the PCM processor from separate file
406 const processorUrl = './scripts/extensions/tts/lib/pcm-processor.js';
407 await this.audioContext.audioWorklet.addModule(processorUrl);
408 this.audioWorkletNode = new AudioWorkletNode(this.audioContext, 'pcm-processor');
409 this.audioWorkletNode.connect(this.audioContext.destination);
410 }
411
412 parseWavHeader(buffer) {
413 const view = new DataView(buffer);
414 // Sample rate is at bytes 24-27 (little endian)
415 const sampleRate = view.getUint32(24, true);
416 // Number of channels is at bytes 22-23 (little endian)
417 const channels = view.getUint16(22, true);
418 // Bits per sample is at bytes 34-35 (little endian)
419 const bitsPerSample = view.getUint16(34, true);
420
421 return { sampleRate, channels, bitsPerSample };
422 }
423
424 async processStreamingAudio(response) {
425 if (!response.ok) {
426 throw new Error(`HTTP ${response.status}`);
427 }
428
429 const reader = response.body.getReader();
430 let headerParsed = false;
431 let wavInfo = null;
432
433 const processStream = async ({ done, value }) => {
434 if (done) {
435 return;
436 }
437
438 if (!headerParsed) {
439 // Parse WAV header to get sample rate
440 wavInfo = this.parseWavHeader(value.buffer);
441 console.log('WAV Info:', wavInfo);
442
443 // Initialize AudioWorklet with correct sample rate
444 await this.initAudioWorklet(wavInfo.sampleRate);
445
446 // Skip WAV header (first 44 bytes typically)
447 const pcmData = value.slice(44);
448 this.audioWorkletNode.port.postMessage({ pcmData });
449 headerParsed = true;
450
451 const next = await reader.read();
452 return processStream(next);
453 }
454
455 // Send PCM data to AudioWorklet for immediate playback
456 this.audioWorkletNode.port.postMessage({ pcmData: value });
457 const next = await reader.read();
458 return processStream(next);
459 };
460
461 const firstChunk = await reader.read();
462 await processStream(firstChunk);
463 }
464
465 async previewTtsVoice(voiceId) {
466 this.audioElement.pause();
467 this.audioElement.currentTime = 0;
468
469 const text = getPreviewString('en-US');
470 const response = await this.fetchTtsGeneration(text, voiceId);
471
472 if (this.settings.streaming) {
473 // Use shared streaming method
474 await this.processStreamingAudio(response);
475 } else {
476 // For non-streaming, response is a fetch Response object
477 if (!response.ok) {
478 throw new Error(`HTTP ${response.status}`);
479 }
480
481 const audio = await response.blob();
482 const url = URL.createObjectURL(audio);
483 this.audioElement.src = url;
484 this.audioElement.play();
485 this.audioElement.onended = () => URL.revokeObjectURL(url);
486 }
487 }
488
489 async fetchTtsGeneration(inputText, voiceId) {
490 console.info(`Generating new TTS for voice_id ${voiceId}`);
491
492 const settings = this.settings;
493 const streaming = settings.streaming;
494
495 const chatterboxParams = [
496 'desired_length',
497 'max_length',
498 'halve_first_chunk',
499 'exaggeration',
500 'cfg_weight',
501 'temperature',
502 'device',
503 'dtype',
504 'cpu_offload',
505 'chunked',
506 'cache_voice',
507 'tokens_per_slice',
508 'remove_milliseconds',
509 'remove_milliseconds_start',
510 'chunk_overlap_method',
511 'seed',
512 ];
513 const getParams = settings => Object.fromEntries(
514 Object.entries(settings).filter(([key]) =>
515 chatterboxParams.includes(key),
516 ),
517 );
518
519 const requestBody = {
520 model: settings.model,
521 voice: voiceId,
522 input: inputText,
523 response_format: 'wav',
524 speed: settings.speed,
525 stream: streaming,
526 params: getParams(settings),
527 };
528
529 const headers = {
530 'Content-Type': 'application/json',
531 'Cache-Control': streaming ? 'no-cache' : undefined,
532 };
533
534 if (streaming) {
535 headers['Cache-Control'] = 'no-cache';
536 }
537
538 const response = await fetch(settings.provider_endpoint, {
539 method: 'POST',
540 headers,
541 body: JSON.stringify(requestBody),
542 });
543
544 if (!response.ok) {
545 toastr.error(response.statusText, 'TTS Generation Failed');
546 throw new Error(
547 `HTTP ${response.status}: ${await response.text()}`,
548 );
549 }
550
551 return response;
552 }
553
554 setVolume(volume) {
555 // Clamp volume between 0.0 and 2.0 (0% to 200%)
556 this.currentVolume = Math.max(0, Math.min(2.0, volume));
557
558 // Set volume for regular audio element (non-streaming)
559 this.audioElement.volume = Math.min(this.currentVolume, 1.0); // HTML audio element max is 1.0
560
561 // Set volume for AudioWorklet (streaming)
562 if (this.audioWorkletNode) {
563 this.audioWorkletNode.port.postMessage({ volume: this.currentVolume });
564 }
565 }
566}