Feature: Google Native TTS (#4163) * google native tts impl * Refactor getting google auth data * Bring back status json * Native TTS => Gemini TTS --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>

48f18cb74a58b80d214f1cd1c6c93cb571a41853

powercaller <133809462+powercaller@users.noreply.github.com>

Signed
Parents: 8c8e4f032f7e6fd589947a9c2bd56d038efe2dc2
3 files changed, +409 -60Ignore whitespace
public/scripts/extensions/tts/google-native.js+195 -0
@@ -0,0 +1,195 @@
1+import { getRequestHeaders } from '../../../script.js';
2+import { oai_settings } from '../../openai.js';
3+import { isValidUrl } from '../../utils.js';
4+import { getPreviewString, saveTtsProviderSettings } from './index.js';
5+
6+
7+export class GoogleNativeTtsProvider {
8+ settings;
9+ voices = [];
10+ separator = ' . ';
11+ audioElement = document.createElement('audio');
12+
13+ defaultSettings = {
14+ voiceMap: {},
15+ model: 'gemini-2.5-flash-preview-tts',
16+ apiType: 'makersuite',
17+ };
18+
19+ get settingsHtml() {
20+ return `
21+ <small>Hint: Save an API key in the Google AI Studio/Vertex AI connection settings</small>
22+ <div id="google-native-tts-settings">
23+ <div>
24+ <label for="google-tts-api-type">API Type:</label>
25+ <select id="google-tts-api-type">
26+ <option value="makersuite">Google AI Studio (MakerSuite)</option>
27+ <option value="vertexai">Google Vertex AI</option>
28+ </select>
29+ </div>
30+ <div>
31+ <label for="google-tts-model">Model:</label>
32+ <select id="google-tts-model">
33+ <option value="gemini-2.5-flash-preview-tts">Gemini 2.5 Flash Preview TTS</option>
34+ <option value="gemini-2.5-pro-preview-tts">Gemini 2.5 Pro Preview TTS</option>
35+ </select>
36+ </div>
37+ </div>`;
38+ }
39+
40+ async loadSettings(settings) {
41+ if (Object.keys(settings).length === 0) {
42+ console.info('Using default Google TTS Provider settings');
43+ }
44+
45+ this.settings = { ...this.defaultSettings, ...settings };
46+
47+ $('#google-tts-api-type').val(this.settings.apiType);
48+ $('#google-tts-model').val(this.settings.model);
49+
50+ $('#google-tts-api-type, #google-tts-model').on('change', () => this.onSettingsChange());
51+
52+ try {
53+ await this.checkReady();
54+ console.debug('Google TTS: Settings loaded');
55+ } catch (err) {
56+ console.warn('Google TTS: Settings loaded, but not ready.', err.message);
57+ }
58+ }
59+
60+ onSettingsChange() {
61+ this.settings.apiType = $('#google-tts-api-type').val();
62+ this.settings.model = $('#google-tts-model').val();
63+
64+ this.voices = []; // Reset voices cache so it re-fetches
65+ saveTtsProviderSettings();
66+ }
67+
68+ async checkReady() {
69+ await this.fetchTtsVoiceObjects();
70+ }
71+
72+ async onRefreshClick() {
73+ await this.checkReady();
74+ }
75+
76+ async getVoice(voiceName) {
77+ if (this.voices.length === 0) {
78+ this.voices = await this.fetchTtsVoiceObjects();
79+ }
80+
81+ const match = this.voices.find(voice => voice.name === voiceName || voice.voice_id === voiceName);
82+
83+ if (!match) {
84+ throw `TTS Voice name ${voiceName} not found`;
85+ }
86+ return match;
87+ }
88+
89+ async generateTts(text, voiceId) {
90+ return await this.fetchNativeTtsGeneration(text, voiceId);
91+ }
92+
93+ async fetchTtsVoiceObjects() {
94+ try {
95+ const response = await fetch('/api/google/list-native-voices', {
96+ method: 'POST',
97+ headers: getRequestHeaders(),
98+ body: JSON.stringify({}),
99+ });
100+
101+ if (!response.ok) {
102+ let errorMessage = `HTTP ${response.status}: ${response.statusText}`;
103+
104+ try {
105+ const errorJson = await response.json();
106+ if (errorJson.error) {
107+ errorMessage = errorJson.error;
108+ }
109+ } catch (parseError) {
110+ // Response isn't valid JSON, use the HTTP error message
111+ console.debug('Error response is not JSON:', parseError.message);
112+ }
113+
114+ throw new Error(errorMessage);
115+ }
116+
117+ const responseJson = await response.json();
118+
119+ if (!responseJson.voices || !Array.isArray(responseJson.voices)) {
120+ throw new Error('Invalid response format: voices array not found');
121+ }
122+
123+ this.voices = responseJson.voices;
124+ console.info(`Google TTS: Loaded ${this.voices.length} voices`);
125+
126+ return this.voices;
127+
128+ } catch (error) {
129+ console.error('Failed to fetch Google TTS voices:', error);
130+ throw error;
131+ }
132+ }
133+
134+ async previewTtsVoice(id) {
135+ this.audioElement.pause();
136+ this.audioElement.currentTime = 0;
137+
138+ try {
139+ const voice = await this.getVoice(id);
140+ const text = getPreviewString(voice.lang || 'en-US');
141+
142+ const response = await this.fetchNativeTtsGeneration(text, id);
143+
144+ if (!response.ok) {
145+ // Error is handled inside the fetch function, but we still need to stop here
146+ return;
147+ }
148+
149+ const audioBlob = await response.blob();
150+ const url = URL.createObjectURL(audioBlob);
151+ this.audioElement.src = url;
152+ this.audioElement.play();
153+ this.audioElement.onended = () => URL.revokeObjectURL(url);
154+
155+ } catch (error) {
156+ console.error('TTS Preview Error:', error);
157+ toastr.error(`Could not generate preview: ${error.message}`);
158+ }
159+ }
160+
161+ async fetchNativeTtsGeneration(text, voiceId) {
162+ console.info(`Generating native Google TTS for voice_id ${voiceId}`);
163+ const useReverseProxy = oai_settings.reverse_proxy && isValidUrl(oai_settings.reverse_proxy);
164+
165+ const response = await fetch('/api/google/generate-native-tts', {
166+ method: 'POST',
167+ headers: getRequestHeaders(),
168+ body: JSON.stringify({
169+ text: text,
170+ voice: voiceId,
171+ model: this.settings.model,
172+ api: this.settings.apiType,
173+ reverse_proxy: useReverseProxy ? oai_settings.reverse_proxy : '',
174+ proxy_password: useReverseProxy ? oai_settings.proxy_password : '',
175+ vertexai_auth_mode: oai_settings.vertexai_auth_mode,
176+ vertexai_region: oai_settings.vertexai_region,
177+ vertexai_express_project_id: oai_settings.vertexai_express_project_id,
178+ }),
179+ });
180+
181+ if (!response.ok) {
182+ let errorMessage = `HTTP ${response.status}: ${response.statusText}`;
183+ try {
184+ const errorJson = await response.json();
185+ if (errorJson.error) {
186+ errorMessage = errorJson.error;
187+ }
188+ } catch {
189+ // Not a JSON response, do nothing and keep the original http error
190+ }
191+ throw new Error(errorMessage);
192+ }
193+ return response;
194+ }
195+}
public/scripts/extensions/tts/index.js+2 -0
@@ -27,6 +27,7 @@ import { SlashCommandEnumValue, enumTypes } from '../../slash-commands/SlashComm
2727import { enumIcons } from '../../slash-commands/SlashCommandCommonEnumsProvider.js';
2828import { POPUP_TYPE, callGenericPopup } from '../../popup.js';
2929import { GoogleTranslateTtsProvider } from './google-translate.js';
30+import { GoogleNativeTtsProvider } from './google-native.js';
3031import { ChatterboxTtsProvider } from './chatterbox.js';
3132import { KokoroTtsProvider } from './kokoro.js';
3233import { TtsWebuiProvider } from './tts-webui.js';
@@ -96,6 +97,7 @@ const ttsProviders = {
9697 Edge: EdgeTtsProvider,
9798 ElevenLabs: ElevenLabsTtsProvider,
9899 'Google Translate': GoogleTranslateTtsProvider,
100+ 'Google Gemini TTS': GoogleNativeTtsProvider,
99101 GSVI: GSVITtsProvider,
100102 'GPT-SoVITS-V2 (Unofficial)': GptSovitsV2Provider,
101103 Kokoro: KokoroTtsProvider,
src/endpoints/google.js+212 -60
@@ -6,11 +6,34 @@ import crypto from 'node:crypto';
66
77import { readSecret, SECRET_KEYS } from './secrets.js';
88import { GEMINI_SAFETY } from '../constants.js';
99import { getConfigValue, trimTrailingSlash } from '../util.js';
1010
1111const API_MAKERSUITE = 'https://generativelanguage.googleapis.com';
1212const API_VERTEX_AI = 'https://us-central1-aiplatform.googleapis.com';
1313
14+function createWavHeader(dataSize, sampleRate, numChannels = 1, bitsPerSample = 16) {
15+ const header = Buffer.alloc(44);
16+ header.write('RIFF', 0);
17+ header.writeUInt32LE(36 + dataSize, 4);
18+ header.write('WAVE', 8);
19+ header.write('fmt ', 12);
20+ header.writeUInt32LE(16, 16);
21+ header.writeUInt16LE(1, 20);
22+ header.writeUInt16LE(numChannels, 22);
23+ header.writeUInt32LE(sampleRate, 24);
24+ header.writeUInt32LE(sampleRate * numChannels * bitsPerSample / 8, 28);
25+ header.writeUInt16LE(numChannels * bitsPerSample / 8, 32);
26+ header.writeUInt16LE(bitsPerSample, 34);
27+ header.write('data', 36);
28+ header.writeUInt32LE(dataSize, 40);
29+ return header;
30+}
31+
32+function createCompleteWavFile(pcmData, sampleRate) {
33+ const header = createWavHeader(pcmData.length, sampleRate);
34+ return Buffer.concat([header, pcmData]);
35+}
36+
1437// Vertex AI authentication helper functions
1538export async function getVertexAIAuth(request) {
1639 const authMode = request.body.vertexai_auth_mode || 'express';
@@ -128,73 +151,83 @@ export function getProjectIdFromServiceAccount(serviceAccount) {
128151 return projectId;
129152}
130153
154+/**
155+ * Generates Google API URL and headers based on request configuration
156+ * @param {express.Request} request Express request object
157+ * @param {string} model Model name to use
158+ * @param {string} endpoint API endpoint (default: 'generateContent')
159+ * @returns {Promise<{url: string, headers: object, apiName: string}>} URL, headers, and API name
160+ */
161+export async function getGoogleApiConfig(request, model, endpoint = 'generateContent') {
162+ const useVertexAi = request.body.api === 'vertexai';
163+ const region = request.body.vertexai_region || 'us-central1';
164+ const apiName = useVertexAi ? 'Google Vertex AI' : 'Google AI Studio';
165+
166+ let url;
167+ let headers = {
168+ 'Content-Type': 'application/json',
169+ };
170+
171+ if (useVertexAi) {
172+ // Get authentication for Vertex AI
173+ const { authHeader, authType } = await getVertexAIAuth(request);
174+
175+ if (authType === 'express') {
176+ // Express mode: use API key parameter
177+ const keyParam = authHeader.replace('Bearer ', '');
178+ const projectId = request.body.vertexai_express_project_id;
179+ const baseUrl = region === 'global'
180+ ? 'https://aiplatform.googleapis.com'
181+ : `https://${region}-aiplatform.googleapis.com`;
182+ url = projectId
183+ ? `https://aiplatform.googleapis.com/v1/projects/${projectId}/locations/${region}/publishers/google/models/${model}:${endpoint}?key=${keyParam}`
184+ : `${baseUrl}/v1/publishers/google/models/${model}:${endpoint}?key=${keyParam}`;
185+ } else if (authType === 'full') {
186+ // Full mode: use project-specific URL with Authorization header
187+ // Get project ID from Service Account JSON
188+ const serviceAccountJson = readSecret(request.user.directories, SECRET_KEYS.VERTEXAI_SERVICE_ACCOUNT);
189+ if (!serviceAccountJson) {
190+ throw new Error('Vertex AI Service Account JSON is missing.');
191+ }
192+
193+ let projectId;
194+ try {
195+ const serviceAccount = JSON.parse(serviceAccountJson);
196+ projectId = getProjectIdFromServiceAccount(serviceAccount);
197+ } catch (error) {
198+ throw new Error('Failed to extract project ID from Service Account JSON.');
199+ }
200+ // Handle global region differently - no region prefix in hostname
201+ url = region === 'global'
202+ ? `https://aiplatform.googleapis.com/v1/projects/${projectId}/locations/${region}/publishers/google/models/${model}:${endpoint}`
203+ : `https://${region}-aiplatform.googleapis.com/v1/projects/${projectId}/locations/${region}/publishers/google/models/${model}:${endpoint}`;
204+ headers['Authorization'] = authHeader;
205+ } else {
206+ // Proxy mode: use Authorization header
207+ const apiUrl = trimTrailingSlash(request.body.reverse_proxy || API_VERTEX_AI);
208+ url = `${apiUrl}/v1/publishers/google/models/${model}:${endpoint}`;
209+ headers['Authorization'] = authHeader;
210+ }
211+ } else {
212+ // Google AI Studio
213+ const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MAKERSUITE);
214+ const apiUrl = trimTrailingSlash(request.body.reverse_proxy || API_MAKERSUITE);
215+ const apiVersion = getConfigValue('gemini.apiVersion', 'v1beta');
216+ url = `${apiUrl}/${apiVersion}/models/${model}:${endpoint}?key=${apiKey}`;
217+ }
218+
219+ return { url, headers, apiName };
220+}
221+
131222export const router = express.Router();
132223
133224router.post('/caption-image', async (request, response) => {
134225 try {
135226 const mimeType = request.body.image.split(';')[0].split(':')[1];
136227 const base64Data = request.body.image.split(',')[1];
137- const useVertexAi = request.body.api === 'vertexai';
138- const apiName = useVertexAi ? 'Google Vertex AI' : 'Google AI Studio';
139228 const model = request.body.model || 'gemini-2.0-flash';
229+ const { url, headers, apiName } = await getGoogleApiConfig(request, model);
140230
141- let url;
142- let headers = {
143- 'Content-Type': 'application/json',
144- };
145-
146- if (useVertexAi) {
147- // Get authentication for Vertex AI
148- const { authHeader, authType } = await getVertexAIAuth(request);
149-
150- if (authType === 'express') {
151- // Express mode: use API key parameter
152- const keyParam = authHeader.replace('Bearer ', '');
153- const region = request.body.vertexai_region || 'us-central1';
154- const projectId = request.body.vertexai_express_project_id;
155- const baseUrl = region === 'global'
156- ? 'https://aiplatform.googleapis.com'
157- : `https://${region}-aiplatform.googleapis.com`;
158- url = projectId
159- ? `https://aiplatform.googleapis.com/v1/projects/${projectId}/locations/${region}/publishers/google/models/${model}:generateContent?key=${keyParam}`
160- : `${baseUrl}/v1/publishers/google/models/${model}:generateContent?key=${keyParam}`;
161- } else if (authType === 'full') {
162- // Full mode: use project-specific URL with Authorization header
163- // Get project ID from Service Account JSON
164- const serviceAccountJson = readSecret(request.user.directories, SECRET_KEYS.VERTEXAI_SERVICE_ACCOUNT);
165- if (!serviceAccountJson) {
166- console.warn('Vertex AI Service Account JSON is missing.');
167- return response.status(400).send({ error: true });
168- }
169-
170- let projectId;
171- try {
172- const serviceAccount = JSON.parse(serviceAccountJson);
173- projectId = getProjectIdFromServiceAccount(serviceAccount);
174- } catch (error) {
175- console.error('Failed to extract project ID from Service Account JSON:', error);
176- return response.status(400).send({ error: true });
177- }
178- const region = request.body.vertexai_region || 'us-central1';
179- // Handle global region differently - no region prefix in hostname
180- if (region === 'global') {
181- url = `https://aiplatform.googleapis.com/v1/projects/${projectId}/locations/${region}/publishers/google/models/${model}:generateContent`;
182- } else {
183- url = `https://${region}-aiplatform.googleapis.com/v1/projects/${projectId}/locations/${region}/publishers/google/models/${model}:generateContent`;
184- }
185- headers['Authorization'] = authHeader;
186- } else {
187- // Proxy mode: use Authorization header
188- const apiUrl = trimTrailingSlash(request.body.reverse_proxy || API_VERTEX_AI);
189- url = `${apiUrl}/v1/publishers/google/models/${model}:generateContent`;
190- headers['Authorization'] = authHeader;
191- }
192- } else {
193- // Google AI Studio
194- const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MAKERSUITE);
195- const apiUrl = trimTrailingSlash(request.body.reverse_proxy || API_MAKERSUITE);
196- url = `${apiUrl}/v1beta/models/${model}:generateContent?key=${apiKey}`;
197- }
198231 const body = {
199232 contents: [{
200233 role: 'user',
@@ -266,3 +299,122 @@ router.post('/generate-voice', async (request, response) => {
266299 response.status(500).send('Internal server error');
267300 }
268301});
302+
303+router.post('/list-native-voices', async (_, response) => {
304+ try {
305+ // Hardcoded Gemini native TTS voices from official documentation
306+ // Source: https://ai.google.dev/gemini-api/docs/speech-generation#voices
307+ const voices = [
308+ { name: 'Zephyr', voice_id: 'Zephyr', lang: 'en-US', description: 'Bright' },
309+ { name: 'Puck', voice_id: 'Puck', lang: 'en-US', description: 'Upbeat' },
310+ { name: 'Charon', voice_id: 'Charon', lang: 'en-US', description: 'Informative' },
311+ { name: 'Kore', voice_id: 'Kore', lang: 'en-US', description: 'Firm' },
312+ { name: 'Fenrir', voice_id: 'Fenrir', lang: 'en-US', description: 'Excitable' },
313+ { name: 'Leda', voice_id: 'Leda', lang: 'en-US', description: 'Youthful' },
314+ { name: 'Orus', voice_id: 'Orus', lang: 'en-US', description: 'Firm' },
315+ { name: 'Aoede', voice_id: 'Aoede', lang: 'en-US', description: 'Breezy' },
316+ { name: 'Callirhoe', voice_id: 'Callirhoe', lang: 'en-US', description: 'Easy-going' },
317+ { name: 'Autonoe', voice_id: 'Autonoe', lang: 'en-US', description: 'Bright' },
318+ { name: 'Enceladus', voice_id: 'Enceladus', lang: 'en-US', description: 'Breathy' },
319+ { name: 'Iapetus', voice_id: 'Iapetus', lang: 'en-US', description: 'Clear' },
320+ { name: 'Umbriel', voice_id: 'Umbriel', lang: 'en-US', description: 'Easy-going' },
321+ { name: 'Algieba', voice_id: 'Algieba', lang: 'en-US', description: 'Smooth' },
322+ { name: 'Despina', voice_id: 'Despina', lang: 'en-US', description: 'Smooth' },
323+ { name: 'Erinome', voice_id: 'Erinome', lang: 'en-US', description: 'Clear' },
324+ { name: 'Algenib', voice_id: 'Algenib', lang: 'en-US', description: 'Gravelly' },
325+ { name: 'Rasalgethi', voice_id: 'Rasalgethi', lang: 'en-US', description: 'Informative' },
326+ { name: 'Laomedeia', voice_id: 'Laomedeia', lang: 'en-US', description: 'Upbeat' },
327+ { name: 'Achernar', voice_id: 'Achernar', lang: 'en-US', description: 'Soft' },
328+ { name: 'Alnilam', voice_id: 'Alnilam', lang: 'en-US', description: 'Firm' },
329+ { name: 'Schedar', voice_id: 'Schedar', lang: 'en-US', description: 'Even' },
330+ { name: 'Gacrux', voice_id: 'Gacrux', lang: 'en-US', description: 'Mature' },
331+ { name: 'Pulcherrima', voice_id: 'Pulcherrima', lang: 'en-US', description: 'Forward' },
332+ { name: 'Achird', voice_id: 'Achird', lang: 'en-US', description: 'Friendly' },
333+ { name: 'Zubenelgenubi', voice_id: 'Zubenelgenubi', lang: 'en-US', description: 'Casual' },
334+ { name: 'Vindemiatrix', voice_id: 'Vindemiatrix', lang: 'en-US', description: 'Gentle' },
335+ { name: 'Sadachbia', voice_id: 'Sadachbia', lang: 'en-US', description: 'Lively' },
336+ { name: 'Sadaltager', voice_id: 'Sadaltager', lang: 'en-US', description: 'Knowledgeable' },
337+ { name: 'Sulafat', voice_id: 'Sulafat', lang: 'en-US', description: 'Warm' },
338+ ];
339+ return response.json({ voices });
340+ } catch (error) {
341+ console.error('Failed to return Google TTS voices:', error);
342+ response.sendStatus(500);
343+ }
344+});
345+
346+router.post('/generate-native-tts', async (request, response) => {
347+ try {
348+ const { text, voice, model } = request.body;
349+ const { url, headers, apiName } = await getGoogleApiConfig(request, model);
350+
351+ console.debug(`${apiName} TTS request`, { model, text, voice });
352+
353+ const requestBody = {
354+ contents: [{
355+ role: 'user',
356+ parts: [{ text: text }],
357+ }],
358+ generationConfig: {
359+ responseModalities: ['AUDIO'],
360+ speechConfig: {
361+ voiceConfig: {
362+ prebuiltVoiceConfig: {
363+ voiceName: voice,
364+ },
365+ },
366+ },
367+ },
368+ safetySettings: GEMINI_SAFETY,
369+ };
370+
371+ const result = await fetch(url, {
372+ method: 'POST',
373+ headers: headers,
374+ body: JSON.stringify(requestBody),
375+ });
376+
377+ if (!result.ok) {
378+ const errorText = await result.text();
379+ console.error(`${apiName} TTS API error: ${result.status} ${result.statusText}`, errorText);
380+ const errorMessage = JSON.parse(errorText).error?.message || 'TTS generation failed.';
381+ return response.status(result.status).json({ error: errorMessage });
382+ }
383+
384+ /** @type {any} */
385+ const data = await result.json();
386+ const audioPart = data?.candidates?.[0]?.content?.parts?.[0];
387+ const audioData = audioPart?.inlineData?.data;
388+ const mimeType = audioPart?.inlineData?.mimeType;
389+
390+ if (!audioData) {
391+ return response.status(500).json({ error: 'No audio data found in response' });
392+ }
393+
394+ const audioBuffer = Buffer.from(audioData, 'base64');
395+
396+ //If the audio is raw PCM, wrap it in a WAV header and send it.
397+ if (mimeType && mimeType.toLowerCase().includes('audio/l16')) {
398+ const rateMatch = mimeType.match(/rate=(\d+)/);
399+ const sampleRate = rateMatch ? parseInt(rateMatch[1], 10) : 24000;
400+ const pcmData = audioBuffer;
401+
402+ // Create a complete, playable WAV file buffer.
403+ const wavBuffer = createCompleteWavFile(pcmData, sampleRate);
404+
405+ // Send the WAV file directly to the browser. This is much faster.
406+ response.setHeader('Content-Type', 'audio/wav');
407+ return response.send(wavBuffer);
408+ }
409+
410+ // Fallback for any other audio format Google might send in the future.
411+ response.setHeader('Content-Type', mimeType || 'application/octet-stream');
412+ response.send(audioBuffer);
413+ } catch (error) {
414+ console.error('Google TTS generation failed:', error);
415+ if (!response.headersSent) {
416+ return response.status(500).json({ error: 'Internal server error during TTS generation' });
417+ }
418+ return response.end();
419+ }
420+});