Volcengine tts (#5003) * feat(tts): Add support for Volcengine TTS provider * refactor: Remove the redundant comments in the Volcengine TTS-related code. * fix(volcengine): Fix the audio data processing logic in the voice generation interface * feat(tts): Enhance Volcengine TTS functionality and improve error handling - Return more detailed error information when generating voice fails - Add multiple preset voice options and support custom voice management - Reconstruct the audio stream processing logic to enhance reliability - Improve the UI interface, adding a voice selection dropdown menu and operation buttons * refactor(tts): Optimize the code structure and error handling of the Volcengine TTS provider - Remove the unused "voices" array and "model" parameter - Improve the text processing logic, eliminating unnecessary separators - Standardize the error handling logic, simplifying the status code checks - Fix the DOM operation method, using "createElement" instead of string concatenation - Ensure the existence check of the "customVoices" array * Fix: Change the Content-Type of the audio response to audio/mpeg. * Clean-up --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>

eaa6a00e97f66e70fdeefdc2a1af13b10ade5ddf

Crush0 <54226076+Crush0@users.noreply.github.com>

Signed
Parents: ef25a0365014fe7c298e34b207907f390647d0d8
6 files changed, +462 -0Ignore whitespace
public/scripts/extensions/tts/index.js+2 -0
@@ -36,6 +36,7 @@ import { PollinationsTtsProvider } from './pollinations.js';
3636import { MiniMaxTtsProvider } from './minimax.js';
3737import { ElectronHubTtsProvider } from './electronhub.js';
3838import { ChutesTtsProvider } from './chutes.js';
39+import { VolcengineTtsProvider } from './volcengine.js';
3940
4041const UPDATE_INTERVAL = 1000;
4142const wrapper = new ModuleWorkerWrapper(moduleWorker);
@@ -146,6 +147,7 @@ const ttsProviders = {
146147 'TTS WebUI': TtsWebuiProvider,
147148 VITS: VITSTtsProvider,
148149 XTTSv2: XTTSTtsProvider,
150+ Volcengine: VolcengineTtsProvider,
149151};
150152let ttsProvider;
151153let ttsProviderName;
public/scripts/extensions/tts/volcengine.js+316 -0
@@ -0,0 +1,316 @@
1+import { event_types, eventSource, getRequestHeaders } from '../../../script.js';
2+import { SECRET_KEYS, secret_state } from '../../secrets.js';
3+import { saveTtsProviderSettings, initVoiceMap } from './index.js';
4+import { Popup } from '../../popup.js';
5+export { VolcengineTtsProvider };
6+
7+class VolcengineTtsProvider {
8+ static voices = [
9+ {
10+ name: 'zh_female_xiaohe_uranus_bigtts',
11+ voice_id: 'zh_female_xiaohe_uranus_bigtts',
12+ lang: 'cl',
13+ },
14+ {
15+ name: 'zh_female_vv_uranus_bigtts',
16+ voice_id: 'zh_female_vv_uranus_bigtts',
17+ lang: 'cl',
18+ },
19+ {
20+ name: 'saturn_zh_female_keainvsheng_tob',
21+ voice_id: 'saturn_zh_female_keainvsheng_tob',
22+ lang: 'cl',
23+ },
24+ {
25+ name: 'saturn_zh_female_tiaopigongzhu_tob',
26+ voice_id: 'saturn_zh_female_tiaopigongzhu_tob',
27+ lang: 'cl',
28+ },
29+ {
30+ name: 'saturn_zh_female_cancan_tob',
31+ voice_id: 'saturn_zh_female_cancan_tob',
32+ lang: 'cl',
33+ },
34+ {
35+ name: 'saturn_zh_male_shuanglangshaonian_tob',
36+ voice_id: 'saturn_zh_male_shuanglangshaonian_tob',
37+ lang: 'cl',
38+ },
39+ {
40+ name: 'saturn_zh_male_tiancaitongzhuo_tob',
41+ voice_id: 'saturn_zh_male_tiancaitongzhuo_tob',
42+ lang: 'cl',
43+ },
44+ {
45+ name: 'zh_male_taocheng_uranus_bigtts',
46+ voice_id: 'zh_male_taocheng_uranus_bigtts',
47+ lang: 'cl',
48+ },
49+ ];
50+ settings;
51+ audioElement = document.createElement('audio');
52+ defaultSettings = {
53+ voiceMap: {},
54+ customVoices: [],
55+ resource_id: '',
56+ speed: 0,
57+ provider_endpoint: 'https://openspeech.bytedance.com/api/v3/tts/unidirectional',
58+ };
59+
60+ processText(text) {
61+ return text.split('...').join('');
62+ }
63+
64+ constructor() {
65+ this.handler = async function (/** @type {string} */ key) {
66+ if (![SECRET_KEYS.VOLCENGINE_APP_ID, SECRET_KEYS.VOLCENGINE_ACCESS_KEY].includes(key)) return;
67+ $('#volcengine-tts-app-id').toggleClass('success', !!secret_state[SECRET_KEYS.VOLCENGINE_APP_ID]);
68+ $('#volcengine-tts-access-key').toggleClass('success', !!secret_state[SECRET_KEYS.VOLCENGINE_ACCESS_KEY]);
69+ await this.onRefreshClick();
70+ }.bind(this);
71+ }
72+
73+ dispose() {
74+ [event_types.SECRET_WRITTEN, event_types.SECRET_DELETED, event_types.SECRET_ROTATED].forEach(event => {
75+ eventSource.removeListener(event, this.handler);
76+ });
77+ }
78+
79+ async previewTtsVoice(voice) {
80+ const text = 'Hello! Nice to meet you!';
81+ const audio = await this.generateTts(text, voice);
82+ const audioElement = new Audio(URL.createObjectURL(await audio.blob()));
83+ audioElement.play().catch(e => console.error('Error playing audio:', e));
84+ }
85+
86+ async fetchTtsVoiceObjects() {
87+ return this.getAllVoices();
88+ }
89+
90+ get settingsHtml() {
91+ let html = `
92+ <div>Volcengine (Doubao) TTS Configuration.</div>
93+ <small>Hint: Volcengine (Doubao) TTS configuration items.</small>
94+ <small>Please refer to the <a href="https://www.volcengine.com/docs/6561/1598757" target="_blank">documentation</a> to obtain the configuration items.</small>
95+ <div class="flex-container alignItemsCenter">
96+ <div id="volcengine-tts-app-id" class="menu_button menu_button_icon manage-api-keys" data-key="volcengine_app_id">
97+ <i class="fa-solid fa-key"></i>
98+ <span>App ID</span>
99+ </div>
100+ <div id="volcengine-tts-access-key" class="menu_button menu_button_icon manage-api-keys" data-key="volcengine_access_key">
101+ <i class="fa-solid fa-key"></i>
102+ <span>Access Key</span>
103+ </div>
104+ </div>
105+ <div>
106+ <label for="volcengine-tts-resource-id">Resource ID:</label>
107+ <input type="text" class="text_pole" id="volcengine-tts-resource-id">
108+ </div>
109+ <label for="volcengine-tts-voice">Custom Voice (Speaker):</label>
110+ <div class="tts_custom_voices">
111+ <select id="volcengine-tts-voice-select">
112+ </select>
113+ <i title="Add" id="volcengine-tts-add-voice" class="tts-button fa-solid fa-plus fa-xl success" role="button"></i>
114+ <i title="Delete" id="volcengine-tts-delete-voice" class="tts-button fa-solid fa-xmark fa-xl failure" tabindex="0" role="button"></i>
115+ </div>
116+ <div>
117+ <label for="volcengine-tts-speed">Speed:</label>
118+ <div class="flex-container">
119+ <div class="range-block-range">
120+ <input type="range" id="volcengine-tts-speed" min="-50" max="100" step="1">
121+ </div>
122+ <div class="range-block-counter">
123+ <input type="number" min="-50" max="100" step="1" data-for="volcengine-tts-speed" id="volcengine-tts-speed_counter">
124+ </div>
125+ </div>
126+ </div>
127+ <div>
128+ <label for="volcengine-tts-provider-endpoint">Provider Endpoint:</label>
129+ <input type="text" class="text_pole" id="volcengine-tts-provider-endpoint">
130+ </div>
131+ `;
132+ return html;
133+ }
134+
135+ async getVoice(voiceName) {
136+ const allVoices = this.getAllVoices();
137+ return allVoices.find(voice => voice.name == voiceName);
138+ }
139+
140+ getAllVoices() {
141+ const voices = [...VolcengineTtsProvider.voices];
142+
143+ for (const customVoice of this.settings.customVoices) {
144+ voices.push({
145+ name: customVoice,
146+ voice_id: customVoice,
147+ lang: 'cl',
148+ });
149+ }
150+
151+ return voices;
152+ }
153+
154+ populateVoices() {
155+ const voiceSelect = $('#volcengine-tts-voice-select');
156+
157+ voiceSelect.empty();
158+
159+ for (const customVoice of this.settings.customVoices) {
160+ const option = document.createElement('option');
161+ option.value = customVoice;
162+ option.textContent = customVoice;
163+ voiceSelect.append(option);
164+ }
165+ }
166+
167+ async onRefreshClick() {
168+ return await this.checkReady();
169+ }
170+
171+ onSettingsChange() {
172+ // Used when provider settings are updated from UI
173+ this.settings.resource_id = $('#volcengine-tts-resource-id').val();
174+ this.settings.speed = $('#volcengine-tts-speed').val();
175+ this.settings.provider_endpoint = $('#volcengine-tts-provider-endpoint').val();
176+ saveTtsProviderSettings();
177+ this.changeTTSSettings();
178+ }
179+
180+ async changeTTSSettings() {
181+ const speed = this.settings.speed;
182+ $('#volcengine-tts-speed').val(speed);
183+ $('#volcengine-tts-speed_counter').val(speed);
184+ }
185+
186+ async loadSettings(settings) {
187+ // Populate Provider UI given input settings
188+ if (Object.keys(settings).length == 0) {
189+ console.info('Using default TTS Provider settings');
190+ }
191+
192+ // Only accept keys defined in defaultSettings
193+ this.settings = { ...this.defaultSettings };
194+
195+ for (const key in settings) {
196+ if (key in this.settings) {
197+ this.settings[key] = settings[key];
198+ } else {
199+ throw `Invalid setting passed to TTS Provider: ${key}`;
200+ }
201+ }
202+
203+ // Set initial values from the settings
204+ $('#volcengine-tts-resource-id').val(this.settings.resource_id).on('change', this.onSettingsChange.bind(this));
205+ $('#volcengine-tts-add-voice').on('click', this.createNewVoice.bind(this));
206+ $('#volcengine-tts-delete-voice').on('click', this.deleteSelectedVoice.bind(this));
207+
208+ // Ensure custom configuration arrays exist
209+ if (!this.settings.customVoices) this.settings.customVoices = [];
210+
211+
212+ this.populateVoices();
213+
214+ // Speed control - range and number inputs
215+ const speedInput = $('#volcengine-tts-speed');
216+ const speedCounter = $('#volcengine-tts-speed_counter');
217+
218+ speedInput.val(this.settings.speed).on('input change', (e) => {
219+ const value = $(e.target).val();
220+ speedCounter.val(value);
221+ this.settings.speed = value;
222+ saveTtsProviderSettings();
223+ this.changeTTSSettings();
224+ });
225+
226+ speedCounter.val(this.settings.speed).on('input change', (e) => {
227+ const value = $(e.target).val();
228+ speedInput.val(value);
229+ this.settings.speed = value;
230+ saveTtsProviderSettings();
231+ this.changeTTSSettings();
232+ });
233+
234+ $('#volcengine-tts-provider-endpoint').val(this.settings.provider_endpoint).on('change', this.onSettingsChange.bind(this));
235+
236+ // Initialize secret keys UI
237+ $('#volcengine-tts-app-id').toggleClass('success', !!secret_state[SECRET_KEYS.VOLCENGINE_APP_ID]);
238+ $('#volcengine-tts-access-key').toggleClass('success', !!secret_state[SECRET_KEYS.VOLCENGINE_ACCESS_KEY]);
239+ [event_types.SECRET_WRITTEN, event_types.SECRET_DELETED, event_types.SECRET_ROTATED].forEach(event => {
240+ eventSource.on(event, this.handler);
241+ });
242+
243+ await this.checkReady();
244+
245+ console.info('Volcengine TTS: Settings loaded');
246+ }
247+
248+ async createNewVoice() {
249+ const name = await Popup.show.input('Voice name: ', null);
250+ if (!name) {
251+ return;
252+ }
253+ if (this.settings.customVoices.includes(name)) {
254+ toastr.error('Voice name should be unique.');
255+ return;
256+ }
257+ this.settings.customVoices.push(name);
258+ this.populateVoices();
259+ initVoiceMap();
260+ saveTtsProviderSettings();
261+ }
262+
263+ async deleteSelectedVoice() {
264+ const selectedVoiceName = $('#volcengine-tts-voice-select').val();
265+
266+ if (!selectedVoiceName) {
267+ toastr.error('Please select a voice first.');
268+ return;
269+ }
270+
271+ const confirm = await Popup.show.confirm(`Are you sure you want to delete the selected voice ${selectedVoiceName}?`);
272+ if (!confirm) {
273+ return;
274+ }
275+
276+
277+ const voiceIndex = this.settings.customVoices.indexOf(selectedVoiceName);
278+ if (voiceIndex !== -1) {
279+ this.settings.customVoices.splice(voiceIndex, 1);
280+ }
281+
282+ this.populateVoices();
283+ initVoiceMap();
284+ saveTtsProviderSettings();
285+ }
286+
287+ async checkReady() {
288+ await Promise.allSettled([this.changeTTSSettings()]);
289+ }
290+
291+ async generateTts(text, speaker) {
292+ const response = await this.fetchTtsGeneration(text, speaker);
293+ return response;
294+ }
295+ async fetchTtsGeneration(text, voice_speaker) {
296+ console.info(`Generating new TTS for voice_id ${voice_speaker}`);
297+ const response = await fetch('/api/volcengine/generate-voice', {
298+ method: 'POST',
299+ headers: getRequestHeaders(),
300+ body: JSON.stringify({
301+ 'provider_endpoint': this.settings.provider_endpoint,
302+ 'resource_id': this.settings.resource_id,
303+ 'text': text,
304+ 'voice_speaker': voice_speaker,
305+ 'speed': this.settings.speed,
306+ }),
307+ });
308+ if (!response.ok) {
309+ const errorText = await response.text();
310+ console.error(`HTTP ${response.status}: ${errorText}`);
311+ toastr.error(errorText);
312+ throw new Error(`HTTP ${response.status}: ${errorText}`);
313+ }
314+ return response;
315+ }
316+}
public/scripts/secrets.js+4 -0
@@ -74,6 +74,8 @@ export const SECRET_KEYS = {
7474 SILICONFLOW: 'api_key_siliconflow',
7575 ELEVENLABS: 'api_key_elevenlabs',
7676 POLLINATIONS: 'api_key_pollinations',
77+ VOLCENGINE_APP_ID: 'volcengine_app_id',
78+ VOLCENGINE_ACCESS_KEY: 'volcengine_access_key',
7779};
7880
7981const FRIENDLY_NAMES = {
@@ -136,6 +138,8 @@ const FRIENDLY_NAMES = {
136138 [SECRET_KEYS.SILICONFLOW]: 'SiliconFlow',
137139 [SECRET_KEYS.ELEVENLABS]: 'ElevenLabs TTS',
138140 [SECRET_KEYS.POLLINATIONS]: 'Pollinations',
141+ [SECRET_KEYS.VOLCENGINE_APP_ID]: 'Volcengine App ID',
142+ [SECRET_KEYS.VOLCENGINE_ACCESS_KEY]: 'Volcengine Access Key',
139143};
140144
141145const INPUT_MAP = {
src/endpoints/secrets.js+2 -0
@@ -67,6 +67,8 @@ export const SECRET_KEYS = {
6767 SILICONFLOW: 'api_key_siliconflow',
6868 ELEVENLABS: 'api_key_elevenlabs',
6969 POLLINATIONS: 'api_key_pollinations',
70+ VOLCENGINE_APP_ID: 'volcengine_app_id',
71+ VOLCENGINE_ACCESS_KEY: 'volcengine_access_key',
7072};
7173
7274/**
src/endpoints/volcengine.js+136 -0
@@ -0,0 +1,136 @@
1+import fetch from 'node-fetch';
2+import { Router } from 'express';
3+
4+import { readSecret, SECRET_KEYS } from './secrets.js';
5+
6+export const router = Router();
7+
8+
9+router.post('/generate-voice', async (req, res) => {
10+ try {
11+ let provider_endpoint = req.body.provider_endpoint;
12+ if (!provider_endpoint) {
13+ console.warn('Volcengine endpoint not set, use default endpoint instead');
14+ provider_endpoint = 'https://openspeech.bytedance.com/api/v3/tts/unidirectional';
15+ }
16+
17+ const appId = readSecret(req.user.directories, SECRET_KEYS.VOLCENGINE_APP_ID);
18+ const accessKey = readSecret(req.user.directories, SECRET_KEYS.VOLCENGINE_ACCESS_KEY);
19+
20+ if (!appId || !accessKey) {
21+ console.warn('Volcengine generate-voice request missing required parameters appId or accessKey');
22+ return res.sendStatus(403);
23+ }
24+
25+ const resourceId = req.body.resource_id;
26+ const text = req.body.text;
27+ const voice_speaker = req.body.voice_speaker;
28+
29+ if (!resourceId || !text || !voice_speaker) {
30+ console.warn('Volcengine generate-voice request missing required parameters resourceId or text or voice_speaker');
31+ return res.sendStatus(400);
32+ }
33+
34+ const response = await fetch(provider_endpoint, {
35+ method: 'POST',
36+ headers: {
37+ 'X-Api-App-Id': appId || '',
38+ 'X-Api-Access-Key': accessKey || '',
39+ 'X-Api-Resource-Id': resourceId || '',
40+ 'Content-Type': 'application/json',
41+ },
42+ body: JSON.stringify({
43+ 'req_params': {
44+ 'text': text,
45+ 'speaker': voice_speaker,
46+ 'audio_params': {
47+ 'format': 'mp3',
48+ 'speech_rate': Number.parseInt(req.body.speed || '0'),
49+ },
50+ 'additions': JSON.stringify({
51+ 'mute_cut_threshold': '400',
52+ 'mute_cut_remain_ms': '1',
53+ 'explicit_language': 'crosslingual',
54+ 'enable_language_detector': true,
55+ 'disable_markdown_filter': true,
56+ 'cache_config': {
57+ 'use_cache': true,
58+ 'text_type': 1,
59+ },
60+ }),
61+ },
62+ }),
63+ });
64+
65+ if (!response.ok) {
66+ const logid = response.headers.get('X-Tt-Logid') || '';
67+ console.warn('Volcengine Request failed', response.status, response.statusText, logid);
68+ return res.header('X-Tt-Logid', logid).status(500).send(`TTS Generation Failed: ${response.statusText}`);
69+ }
70+ const decoder = new TextDecoder();
71+
72+ const result = await new Promise((resolve, reject) => {
73+ let audioChunks_ = [];
74+ let buffer = '';
75+ if (!response.body) {
76+ reject(new Error('Response body is null'));
77+ return;
78+ }
79+ response.body.on('data', (chunk) => {
80+ buffer += decoder.decode(chunk, { stream: true });
81+
82+ const lines = buffer.split('\n');
83+ buffer = lines.pop() || '';
84+
85+ for (const line of lines) {
86+ if (!line.trim()) continue;
87+
88+ try {
89+ const { data, code, message } = JSON.parse(line);
90+ if (code !== 0 && code !== 20000000) {
91+ reject(`Volcengine TTS stream line code ${code}, ${message}`);
92+ return;
93+ }
94+ if (data) {
95+ const audioData = Buffer.from(data, 'base64');
96+ audioChunks_.push(audioData);
97+ }
98+ } catch (e) {
99+ console.error('Error parsing Volcengine TTS stream line:', e);
100+ }
101+ }
102+ });
103+
104+ response.body.on('end', () => {
105+ if (buffer.trim()) {
106+ try {
107+ const { code, data, message } = JSON.parse(buffer);
108+ if (code !== 0 && code !== 20000000) {
109+ reject(`Volcengine TTS stream line code ${code}, ${message}`);
110+ return;
111+ }
112+ if (data) {
113+ const audioData = Buffer.from(data, 'base64');
114+ audioChunks_.push(audioData);
115+ }
116+ } catch (e) {
117+ reject(`Error parsing final Volcengine TTS stream line: ${e}`);
118+ }
119+ }
120+ resolve(audioChunks_);
121+ });
122+
123+ response.body.on('error', (error) => {
124+ reject(`Error reading Volcengine TTS stream: ${error}`);
125+ });
126+ });
127+
128+ const finalAudioData = Buffer.concat(result);
129+
130+ res.set('Content-Type', 'audio/mpeg');
131+ res.status(200).send(finalAudioData);
132+ } catch (error) {
133+ console.error('Volcengine generate-voice fetch failed', error);
134+ res.status(500).send(`TTS Generation Failed: ${error}`);
135+ }
136+});
src/server-startup.js+2 -0
@@ -48,6 +48,7 @@ import { router as azureRouter } from './endpoints/azure.js';
4848import { router as minimaxRouter } from './endpoints/minimax.js';
4949import { router as dataMaidRouter } from './endpoints/data-maid.js';
5050import { router as backupsRouter } from './endpoints/backups.js';
51+import { router as volcengineRouter } from './endpoints/volcengine.js';
5152
5253/**
5354 * @typedef {object} ServerStartupResult
@@ -174,6 +175,7 @@ export function setupPrivateEndpoints(app) {
174175 app.use('/api/backends/chat-completions', chatCompletionsRouter);
175176 app.use('/api/speech', speechRouter);
176177 app.use('/api/azure', azureRouter);
178+ app.use('/api/volcengine', volcengineRouter);
177179 app.use('/api/minimax', minimaxRouter);
178180 app.use('/api/data-maid', dataMaidRouter);
179181 app.use('/api/backups', backupsRouter);