Volcengine tts (#5003) * feat(tts): Add support for Volcengine TTS provider * refactor: Remove the redundant comments in the Volcengine TTS-related code. * fix(volcengine): Fix the audio data processing logic in the voice generation interface * feat(tts): Enhance Volcengine TTS functionality and improve error handling - Return more detailed error information when generating voice fails - Add multiple preset voice options and support custom voice management - Reconstruct the audio stream processing logic to enhance reliability - Improve the UI interface, adding a voice selection dropdown menu and operation buttons * refactor(tts): Optimize the code structure and error handling of the Volcengine TTS provider - Remove the unused "voices" array and "model" parameter - Improve the text processing logic, eliminating unnecessary separators - Standardize the error handling logic, simplifying the status code checks - Fix the DOM operation method, using "createElement" instead of string concatenation - Ensure the existence check of the "customVoices" array * Fix: Change the Content-Type of the audio response to audio/mpeg. * Clean-up --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>

eaa6a00e97f66e70fdeefdc2a1af13b10ade5ddf

Crush0 <54226076+Crush0@users.noreply.github.com>

Signed
Parents: ef25a0365014fe7c298e34b207907f390647d0d8
6 files changed, +462 -0Showing whitespace changes
public/scripts/extensions/tts/index.js+2 -0
@@ -36,6 +36,7 @@ import { PollinationsTtsProvider } from './pollinations.js';
36import { MiniMaxTtsProvider } from './minimax.js';36import { MiniMaxTtsProvider } from './minimax.js';
37import { ElectronHubTtsProvider } from './electronhub.js';37import { ElectronHubTtsProvider } from './electronhub.js';
38import { ChutesTtsProvider } from './chutes.js';38import { ChutesTtsProvider } from './chutes.js';
39import { VolcengineTtsProvider } from './volcengine.js';
3940
40const UPDATE_INTERVAL = 1000;41const UPDATE_INTERVAL = 1000;
41const wrapper = new ModuleWorkerWrapper(moduleWorker);42const wrapper = new ModuleWorkerWrapper(moduleWorker);
@@ -146,6 +147,7 @@ const ttsProviders = {
146 'TTS WebUI': TtsWebuiProvider,147 'TTS WebUI': TtsWebuiProvider,
147 VITS: VITSTtsProvider,148 VITS: VITSTtsProvider,
148 XTTSv2: XTTSTtsProvider,149 XTTSv2: XTTSTtsProvider,
150 Volcengine: VolcengineTtsProvider,
149};151};
150let ttsProvider;152let ttsProvider;
151let ttsProviderName;153let ttsProviderName;
public/scripts/extensions/tts/volcengine.js+316 -0
@@ -0,0 +1,316 @@
1import { event_types, eventSource, getRequestHeaders } from '../../../script.js';
2import { SECRET_KEYS, secret_state } from '../../secrets.js';
3import { saveTtsProviderSettings, initVoiceMap } from './index.js';
4import { Popup } from '../../popup.js';
5export { VolcengineTtsProvider };
6
7class VolcengineTtsProvider {
8 static voices = [
9 {
10 name: 'zh_female_xiaohe_uranus_bigtts',
11 voice_id: 'zh_female_xiaohe_uranus_bigtts',
12 lang: 'cl',
13 },
14 {
15 name: 'zh_female_vv_uranus_bigtts',
16 voice_id: 'zh_female_vv_uranus_bigtts',
17 lang: 'cl',
18 },
19 {
20 name: 'saturn_zh_female_keainvsheng_tob',
21 voice_id: 'saturn_zh_female_keainvsheng_tob',
22 lang: 'cl',
23 },
24 {
25 name: 'saturn_zh_female_tiaopigongzhu_tob',
26 voice_id: 'saturn_zh_female_tiaopigongzhu_tob',
27 lang: 'cl',
28 },
29 {
30 name: 'saturn_zh_female_cancan_tob',
31 voice_id: 'saturn_zh_female_cancan_tob',
32 lang: 'cl',
33 },
34 {
35 name: 'saturn_zh_male_shuanglangshaonian_tob',
36 voice_id: 'saturn_zh_male_shuanglangshaonian_tob',
37 lang: 'cl',
38 },
39 {
40 name: 'saturn_zh_male_tiancaitongzhuo_tob',
41 voice_id: 'saturn_zh_male_tiancaitongzhuo_tob',
42 lang: 'cl',
43 },
44 {
45 name: 'zh_male_taocheng_uranus_bigtts',
46 voice_id: 'zh_male_taocheng_uranus_bigtts',
47 lang: 'cl',
48 },
49 ];
50 settings;
51 audioElement = document.createElement('audio');
52 defaultSettings = {
53 voiceMap: {},
54 customVoices: [],
55 resource_id: '',
56 speed: 0,
57 provider_endpoint: 'https://openspeech.bytedance.com/api/v3/tts/unidirectional',
58 };
59
60 processText(text) {
61 return text.split('...').join('');
62 }
63
64 constructor() {
65 this.handler = async function (/** @type {string} */ key) {
66 if (![SECRET_KEYS.VOLCENGINE_APP_ID, SECRET_KEYS.VOLCENGINE_ACCESS_KEY].includes(key)) return;
67 $('#volcengine-tts-app-id').toggleClass('success', !!secret_state[SECRET_KEYS.VOLCENGINE_APP_ID]);
68 $('#volcengine-tts-access-key').toggleClass('success', !!secret_state[SECRET_KEYS.VOLCENGINE_ACCESS_KEY]);
69 await this.onRefreshClick();
70 }.bind(this);
71 }
72
73 dispose() {
74 [event_types.SECRET_WRITTEN, event_types.SECRET_DELETED, event_types.SECRET_ROTATED].forEach(event => {
75 eventSource.removeListener(event, this.handler);
76 });
77 }
78
79 async previewTtsVoice(voice) {
80 const text = 'Hello! Nice to meet you!';
81 const audio = await this.generateTts(text, voice);
82 const audioElement = new Audio(URL.createObjectURL(await audio.blob()));
83 audioElement.play().catch(e => console.error('Error playing audio:', e));
84 }
85
86 async fetchTtsVoiceObjects() {
87 return this.getAllVoices();
88 }
89
90 get settingsHtml() {
91 let html = `
92 <div>Volcengine (Doubao) TTS Configuration.</div>
93 <small>Hint: Volcengine (Doubao) TTS configuration items.</small>
94 <small>Please refer to the <a href="https://www.volcengine.com/docs/6561/1598757" target="_blank">documentation</a> to obtain the configuration items.</small>
95 <div class="flex-container alignItemsCenter">
96 <div id="volcengine-tts-app-id" class="menu_button menu_button_icon manage-api-keys" data-key="volcengine_app_id">
97 <i class="fa-solid fa-key"></i>
98 <span>App ID</span>
99 </div>
100 <div id="volcengine-tts-access-key" class="menu_button menu_button_icon manage-api-keys" data-key="volcengine_access_key">
101 <i class="fa-solid fa-key"></i>
102 <span>Access Key</span>
103 </div>
104 </div>
105 <div>
106 <label for="volcengine-tts-resource-id">Resource ID:</label>
107 <input type="text" class="text_pole" id="volcengine-tts-resource-id">
108 </div>
109 <label for="volcengine-tts-voice">Custom Voice (Speaker):</label>
110 <div class="tts_custom_voices">
111 <select id="volcengine-tts-voice-select">
112 </select>
113 <i title="Add" id="volcengine-tts-add-voice" class="tts-button fa-solid fa-plus fa-xl success" role="button"></i>
114 <i title="Delete" id="volcengine-tts-delete-voice" class="tts-button fa-solid fa-xmark fa-xl failure" tabindex="0" role="button"></i>
115 </div>
116 <div>
117 <label for="volcengine-tts-speed">Speed:</label>
118 <div class="flex-container">
119 <div class="range-block-range">
120 <input type="range" id="volcengine-tts-speed" min="-50" max="100" step="1">
121 </div>
122 <div class="range-block-counter">
123 <input type="number" min="-50" max="100" step="1" data-for="volcengine-tts-speed" id="volcengine-tts-speed_counter">
124 </div>
125 </div>
126 </div>
127 <div>
128 <label for="volcengine-tts-provider-endpoint">Provider Endpoint:</label>
129 <input type="text" class="text_pole" id="volcengine-tts-provider-endpoint">
130 </div>
131 `;
132 return html;
133 }
134
135 async getVoice(voiceName) {
136 const allVoices = this.getAllVoices();
137 return allVoices.find(voice => voice.name == voiceName);
138 }
139
140 getAllVoices() {
141 const voices = [...VolcengineTtsProvider.voices];
142
143 for (const customVoice of this.settings.customVoices) {
144 voices.push({
145 name: customVoice,
146 voice_id: customVoice,
147 lang: 'cl',
148 });
149 }
150
151 return voices;
152 }
153
154 populateVoices() {
155 const voiceSelect = $('#volcengine-tts-voice-select');
156
157 voiceSelect.empty();
158
159 for (const customVoice of this.settings.customVoices) {
160 const option = document.createElement('option');
161 option.value = customVoice;
162 option.textContent = customVoice;
163 voiceSelect.append(option);
164 }
165 }
166
167 async onRefreshClick() {
168 return await this.checkReady();
169 }
170
171 onSettingsChange() {
172 // Used when provider settings are updated from UI
173 this.settings.resource_id = $('#volcengine-tts-resource-id').val();
174 this.settings.speed = $('#volcengine-tts-speed').val();
175 this.settings.provider_endpoint = $('#volcengine-tts-provider-endpoint').val();
176 saveTtsProviderSettings();
177 this.changeTTSSettings();
178 }
179
180 async changeTTSSettings() {
181 const speed = this.settings.speed;
182 $('#volcengine-tts-speed').val(speed);
183 $('#volcengine-tts-speed_counter').val(speed);
184 }
185
186 async loadSettings(settings) {
187 // Populate Provider UI given input settings
188 if (Object.keys(settings).length == 0) {
189 console.info('Using default TTS Provider settings');
190 }
191
192 // Only accept keys defined in defaultSettings
193 this.settings = { ...this.defaultSettings };
194
195 for (const key in settings) {
196 if (key in this.settings) {
197 this.settings[key] = settings[key];
198 } else {
199 throw `Invalid setting passed to TTS Provider: ${key}`;
200 }
201 }
202
203 // Set initial values from the settings
204 $('#volcengine-tts-resource-id').val(this.settings.resource_id).on('change', this.onSettingsChange.bind(this));
205 $('#volcengine-tts-add-voice').on('click', this.createNewVoice.bind(this));
206 $('#volcengine-tts-delete-voice').on('click', this.deleteSelectedVoice.bind(this));
207
208 // Ensure custom configuration arrays exist
209 if (!this.settings.customVoices) this.settings.customVoices = [];
210
211
212 this.populateVoices();
213
214 // Speed control - range and number inputs
215 const speedInput = $('#volcengine-tts-speed');
216 const speedCounter = $('#volcengine-tts-speed_counter');
217
218 speedInput.val(this.settings.speed).on('input change', (e) => {
219 const value = $(e.target).val();
220 speedCounter.val(value);
221 this.settings.speed = value;
222 saveTtsProviderSettings();
223 this.changeTTSSettings();
224 });
225
226 speedCounter.val(this.settings.speed).on('input change', (e) => {
227 const value = $(e.target).val();
228 speedInput.val(value);
229 this.settings.speed = value;
230 saveTtsProviderSettings();
231 this.changeTTSSettings();
232 });
233
234 $('#volcengine-tts-provider-endpoint').val(this.settings.provider_endpoint).on('change', this.onSettingsChange.bind(this));
235
236 // Initialize secret keys UI
237 $('#volcengine-tts-app-id').toggleClass('success', !!secret_state[SECRET_KEYS.VOLCENGINE_APP_ID]);
238 $('#volcengine-tts-access-key').toggleClass('success', !!secret_state[SECRET_KEYS.VOLCENGINE_ACCESS_KEY]);
239 [event_types.SECRET_WRITTEN, event_types.SECRET_DELETED, event_types.SECRET_ROTATED].forEach(event => {
240 eventSource.on(event, this.handler);
241 });
242
243 await this.checkReady();
244
245 console.info('Volcengine TTS: Settings loaded');
246 }
247
248 async createNewVoice() {
249 const name = await Popup.show.input('Voice name: ', null);
250 if (!name) {
251 return;
252 }
253 if (this.settings.customVoices.includes(name)) {
254 toastr.error('Voice name should be unique.');
255 return;
256 }
257 this.settings.customVoices.push(name);
258 this.populateVoices();
259 initVoiceMap();
260 saveTtsProviderSettings();
261 }
262
263 async deleteSelectedVoice() {
264 const selectedVoiceName = $('#volcengine-tts-voice-select').val();
265
266 if (!selectedVoiceName) {
267 toastr.error('Please select a voice first.');
268 return;
269 }
270
271 const confirm = await Popup.show.confirm(`Are you sure you want to delete the selected voice ${selectedVoiceName}?`);
272 if (!confirm) {
273 return;
274 }
275
276
277 const voiceIndex = this.settings.customVoices.indexOf(selectedVoiceName);
278 if (voiceIndex !== -1) {
279 this.settings.customVoices.splice(voiceIndex, 1);
280 }
281
282 this.populateVoices();
283 initVoiceMap();
284 saveTtsProviderSettings();
285 }
286
287 async checkReady() {
288 await Promise.allSettled([this.changeTTSSettings()]);
289 }
290
291 async generateTts(text, speaker) {
292 const response = await this.fetchTtsGeneration(text, speaker);
293 return response;
294 }
295 async fetchTtsGeneration(text, voice_speaker) {
296 console.info(`Generating new TTS for voice_id ${voice_speaker}`);
297 const response = await fetch('/api/volcengine/generate-voice', {
298 method: 'POST',
299 headers: getRequestHeaders(),
300 body: JSON.stringify({
301 'provider_endpoint': this.settings.provider_endpoint,
302 'resource_id': this.settings.resource_id,
303 'text': text,
304 'voice_speaker': voice_speaker,
305 'speed': this.settings.speed,
306 }),
307 });
308 if (!response.ok) {
309 const errorText = await response.text();
310 console.error(`HTTP ${response.status}: ${errorText}`);
311 toastr.error(errorText);
312 throw new Error(`HTTP ${response.status}: ${errorText}`);
313 }
314 return response;
315 }
316}
public/scripts/secrets.js+4 -0
@@ -74,6 +74,8 @@ export const SECRET_KEYS = {
74 SILICONFLOW: 'api_key_siliconflow',74 SILICONFLOW: 'api_key_siliconflow',
75 ELEVENLABS: 'api_key_elevenlabs',75 ELEVENLABS: 'api_key_elevenlabs',
76 POLLINATIONS: 'api_key_pollinations',76 POLLINATIONS: 'api_key_pollinations',
77 VOLCENGINE_APP_ID: 'volcengine_app_id',
78 VOLCENGINE_ACCESS_KEY: 'volcengine_access_key',
77};79};
7880
79const FRIENDLY_NAMES = {81const FRIENDLY_NAMES = {
@@ -136,6 +138,8 @@ const FRIENDLY_NAMES = {
136 [SECRET_KEYS.SILICONFLOW]: 'SiliconFlow',138 [SECRET_KEYS.SILICONFLOW]: 'SiliconFlow',
137 [SECRET_KEYS.ELEVENLABS]: 'ElevenLabs TTS',139 [SECRET_KEYS.ELEVENLABS]: 'ElevenLabs TTS',
138 [SECRET_KEYS.POLLINATIONS]: 'Pollinations',140 [SECRET_KEYS.POLLINATIONS]: 'Pollinations',
141 [SECRET_KEYS.VOLCENGINE_APP_ID]: 'Volcengine App ID',
142 [SECRET_KEYS.VOLCENGINE_ACCESS_KEY]: 'Volcengine Access Key',
139};143};
140144
141const INPUT_MAP = {145const INPUT_MAP = {
src/endpoints/secrets.js+2 -0
@@ -67,6 +67,8 @@ export const SECRET_KEYS = {
67 SILICONFLOW: 'api_key_siliconflow',67 SILICONFLOW: 'api_key_siliconflow',
68 ELEVENLABS: 'api_key_elevenlabs',68 ELEVENLABS: 'api_key_elevenlabs',
69 POLLINATIONS: 'api_key_pollinations',69 POLLINATIONS: 'api_key_pollinations',
70 VOLCENGINE_APP_ID: 'volcengine_app_id',
71 VOLCENGINE_ACCESS_KEY: 'volcengine_access_key',
70};72};
7173
72/**74/**
src/endpoints/volcengine.js+136 -0
@@ -0,0 +1,136 @@
1import fetch from 'node-fetch';
2import { Router } from 'express';
3
4import { readSecret, SECRET_KEYS } from './secrets.js';
5
6export const router = Router();
7
8
9router.post('/generate-voice', async (req, res) => {
10 try {
11 let provider_endpoint = req.body.provider_endpoint;
12 if (!provider_endpoint) {
13 console.warn('Volcengine endpoint not set, use default endpoint instead');
14 provider_endpoint = 'https://openspeech.bytedance.com/api/v3/tts/unidirectional';
15 }
16
17 const appId = readSecret(req.user.directories, SECRET_KEYS.VOLCENGINE_APP_ID);
18 const accessKey = readSecret(req.user.directories, SECRET_KEYS.VOLCENGINE_ACCESS_KEY);
19
20 if (!appId || !accessKey) {
21 console.warn('Volcengine generate-voice request missing required parameters appId or accessKey');
22 return res.sendStatus(403);
23 }
24
25 const resourceId = req.body.resource_id;
26 const text = req.body.text;
27 const voice_speaker = req.body.voice_speaker;
28
29 if (!resourceId || !text || !voice_speaker) {
30 console.warn('Volcengine generate-voice request missing required parameters resourceId or text or voice_speaker');
31 return res.sendStatus(400);
32 }
33
34 const response = await fetch(provider_endpoint, {
35 method: 'POST',
36 headers: {
37 'X-Api-App-Id': appId || '',
38 'X-Api-Access-Key': accessKey || '',
39 'X-Api-Resource-Id': resourceId || '',
40 'Content-Type': 'application/json',
41 },
42 body: JSON.stringify({
43 'req_params': {
44 'text': text,
45 'speaker': voice_speaker,
46 'audio_params': {
47 'format': 'mp3',
48 'speech_rate': Number.parseInt(req.body.speed || '0'),
49 },
50 'additions': JSON.stringify({
51 'mute_cut_threshold': '400',
52 'mute_cut_remain_ms': '1',
53 'explicit_language': 'crosslingual',
54 'enable_language_detector': true,
55 'disable_markdown_filter': true,
56 'cache_config': {
57 'use_cache': true,
58 'text_type': 1,
59 },
60 }),
61 },
62 }),
63 });
64
65 if (!response.ok) {
66 const logid = response.headers.get('X-Tt-Logid') || '';
67 console.warn('Volcengine Request failed', response.status, response.statusText, logid);
68 return res.header('X-Tt-Logid', logid).status(500).send(`TTS Generation Failed: ${response.statusText}`);
69 }
70 const decoder = new TextDecoder();
71
72 const result = await new Promise((resolve, reject) => {
73 let audioChunks_ = [];
74 let buffer = '';
75 if (!response.body) {
76 reject(new Error('Response body is null'));
77 return;
78 }
79 response.body.on('data', (chunk) => {
80 buffer += decoder.decode(chunk, { stream: true });
81
82 const lines = buffer.split('\n');
83 buffer = lines.pop() || '';
84
85 for (const line of lines) {
86 if (!line.trim()) continue;
87
88 try {
89 const { data, code, message } = JSON.parse(line);
90 if (code !== 0 && code !== 20000000) {
91 reject(`Volcengine TTS stream line code ${code}, ${message}`);
92 return;
93 }
94 if (data) {
95 const audioData = Buffer.from(data, 'base64');
96 audioChunks_.push(audioData);
97 }
98 } catch (e) {
99 console.error('Error parsing Volcengine TTS stream line:', e);
100 }
101 }
102 });
103
104 response.body.on('end', () => {
105 if (buffer.trim()) {
106 try {
107 const { code, data, message } = JSON.parse(buffer);
108 if (code !== 0 && code !== 20000000) {
109 reject(`Volcengine TTS stream line code ${code}, ${message}`);
110 return;
111 }
112 if (data) {
113 const audioData = Buffer.from(data, 'base64');
114 audioChunks_.push(audioData);
115 }
116 } catch (e) {
117 reject(`Error parsing final Volcengine TTS stream line: ${e}`);
118 }
119 }
120 resolve(audioChunks_);
121 });
122
123 response.body.on('error', (error) => {
124 reject(`Error reading Volcengine TTS stream: ${error}`);
125 });
126 });
127
128 const finalAudioData = Buffer.concat(result);
129
130 res.set('Content-Type', 'audio/mpeg');
131 res.status(200).send(finalAudioData);
132 } catch (error) {
133 console.error('Volcengine generate-voice fetch failed', error);
134 res.status(500).send(`TTS Generation Failed: ${error}`);
135 }
136});
src/server-startup.js+2 -0
@@ -48,6 +48,7 @@ import { router as azureRouter } from './endpoints/azure.js';
48import { router as minimaxRouter } from './endpoints/minimax.js';48import { router as minimaxRouter } from './endpoints/minimax.js';
49import { router as dataMaidRouter } from './endpoints/data-maid.js';49import { router as dataMaidRouter } from './endpoints/data-maid.js';
50import { router as backupsRouter } from './endpoints/backups.js';50import { router as backupsRouter } from './endpoints/backups.js';
51import { router as volcengineRouter } from './endpoints/volcengine.js';
5152
52/**53/**
53 * @typedef {object} ServerStartupResult54 * @typedef {object} ServerStartupResult
@@ -174,6 +175,7 @@ export function setupPrivateEndpoints(app) {
174 app.use('/api/backends/chat-completions', chatCompletionsRouter);175 app.use('/api/backends/chat-completions', chatCompletionsRouter);
175 app.use('/api/speech', speechRouter);176 app.use('/api/speech', speechRouter);
176 app.use('/api/azure', azureRouter);177 app.use('/api/azure', azureRouter);
178 app.use('/api/volcengine', volcengineRouter);
177 app.use('/api/minimax', minimaxRouter);179 app.use('/api/minimax', minimaxRouter);
178 app.use('/api/data-maid', dataMaidRouter);180 app.use('/api/data-maid', dataMaidRouter);
179 app.use('/api/backups', backupsRouter);181 app.use('/api/backups', backupsRouter);