Feature: Google Native TTS (#4163) * google native tts impl * Refactor getting google auth data * Bring back status json * Native TTS => Gemini TTS --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>

48f18cb74a58b80d214f1cd1c6c93cb571a41853

powercaller <133809462+powercaller@users.noreply.github.com>

Signed

Parents: 8c8e4f032f7e6fd589947a9c2bd56d038efe2dc2

public/scripts/extensions/tts/google-native.js+195 -0 Viewed

+import { getRequestHeaders } from '../../../script.js';
+import { oai_settings } from '../../openai.js';
+import { isValidUrl } from '../../utils.js';
+import { getPreviewString, saveTtsProviderSettings } from './index.js';
+export class GoogleNativeTtsProvider {
+    settings;
+    voices = [];
+    separator = ' . ';
+    audioElement = document.createElement('audio');
+    defaultSettings = {
+        voiceMap: {},
+        model: 'gemini-2.5-flash-preview-tts',
+        apiType: 'makersuite',
+    };
+    get settingsHtml() {
+        return `
+        <small>Hint: Save an API key in the Google AI Studio/Vertex AI connection settings</small>
+        <div id="google-native-tts-settings">
+            <div>
+                <label for="google-tts-api-type">API Type:</label>
+                <select id="google-tts-api-type">
+                    <option value="makersuite">Google AI Studio (MakerSuite)</option>
+                    <option value="vertexai">Google Vertex AI</option>
+                </select>
+            </div>
+            <div>
+                <label for="google-tts-model">Model:</label>
+                <select id="google-tts-model">
+                    <option value="gemini-2.5-flash-preview-tts">Gemini 2.5 Flash Preview TTS</option>
+                    <option value="gemini-2.5-pro-preview-tts">Gemini 2.5 Pro Preview TTS</option>
+                </select>
+            </div>
+        </div>`;
+    }
+    async loadSettings(settings) {
+        if (Object.keys(settings).length === 0) {
+            console.info('Using default Google TTS Provider settings');
+        }
+        this.settings = { ...this.defaultSettings, ...settings };
+        $('#google-tts-api-type').val(this.settings.apiType);
+        $('#google-tts-model').val(this.settings.model);
+        $('#google-tts-api-type, #google-tts-model').on('change', () => this.onSettingsChange());
+        try {
+            await this.checkReady();
+            console.debug('Google TTS: Settings loaded');
+        } catch (err) {
+            console.warn('Google TTS: Settings loaded, but not ready.', err.message);
+        }
+    }
+    onSettingsChange() {
+        this.settings.apiType = $('#google-tts-api-type').val();
+        this.settings.model = $('#google-tts-model').val();
+        this.voices = []; // Reset voices cache so it re-fetches
+        saveTtsProviderSettings();
+    }
+    async checkReady() {
+        await this.fetchTtsVoiceObjects();
+    }
+    async onRefreshClick() {
+        await this.checkReady();
+    }
+    async getVoice(voiceName) {
+        if (this.voices.length === 0) {
+            this.voices = await this.fetchTtsVoiceObjects();
+        }
+        const match = this.voices.find(voice => voice.name === voiceName || voice.voice_id === voiceName);
+        if (!match) {
+            throw `TTS Voice name ${voiceName} not found`;
+        }
+        return match;
+    }
+    async generateTts(text, voiceId) {
+        return await this.fetchNativeTtsGeneration(text, voiceId);
+    }
+    async fetchTtsVoiceObjects() {
+        try {
+            const response = await fetch('/api/google/list-native-voices', {
+                method: 'POST',
+                headers: getRequestHeaders(),
+                body: JSON.stringify({}),
+            });
+            if (!response.ok) {
+                let errorMessage = `HTTP ${response.status}: ${response.statusText}`;
+                try {
+                    const errorJson = await response.json();
+                    if (errorJson.error) {
+                        errorMessage = errorJson.error;
+                    }
+                } catch (parseError) {
+                    // Response isn't valid JSON, use the HTTP error message
+                    console.debug('Error response is not JSON:', parseError.message);
+                }
+                throw new Error(errorMessage);
+            }
+            const responseJson = await response.json();
+            if (!responseJson.voices || !Array.isArray(responseJson.voices)) {
+                throw new Error('Invalid response format: voices array not found');
+            }
+            this.voices = responseJson.voices;
+            console.info(`Google TTS: Loaded ${this.voices.length} voices`);
+            return this.voices;
+        } catch (error) {
+            console.error('Failed to fetch Google TTS voices:', error);
+            throw error;
+        }
+    }
+    async previewTtsVoice(id) {
+        this.audioElement.pause();
+        this.audioElement.currentTime = 0;
+        try {
+            const voice = await this.getVoice(id);
+            const text = getPreviewString(voice.lang || 'en-US');
+            const response = await this.fetchNativeTtsGeneration(text, id);
+            if (!response.ok) {
+                // Error is handled inside the fetch function, but we still need to stop here
+                return;
+            }
+            const audioBlob = await response.blob();
+            const url = URL.createObjectURL(audioBlob);
+            this.audioElement.src = url;
+            this.audioElement.play();
+            this.audioElement.onended = () => URL.revokeObjectURL(url);
+        } catch (error) {
+            console.error('TTS Preview Error:', error);
+            toastr.error(`Could not generate preview: ${error.message}`);
+        }
+    }
+    async fetchNativeTtsGeneration(text, voiceId) {
+        console.info(`Generating native Google TTS for voice_id ${voiceId}`);
+        const useReverseProxy = oai_settings.reverse_proxy && isValidUrl(oai_settings.reverse_proxy);
+        const response = await fetch('/api/google/generate-native-tts', {
+            method: 'POST',
+            headers: getRequestHeaders(),
+            body: JSON.stringify({
+                text: text,
+                voice: voiceId,
+                model: this.settings.model,
+                api: this.settings.apiType,
+                reverse_proxy: useReverseProxy ? oai_settings.reverse_proxy : '',
+                proxy_password: useReverseProxy ? oai_settings.proxy_password : '',
+                vertexai_auth_mode: oai_settings.vertexai_auth_mode,
+                vertexai_region: oai_settings.vertexai_region,
+                vertexai_express_project_id: oai_settings.vertexai_express_project_id,
+            }),
+        });
+        if (!response.ok) {
+            let errorMessage = `HTTP ${response.status}: ${response.statusText}`;
+            try {
+                const errorJson = await response.json();
+                if (errorJson.error) {
+                    errorMessage = errorJson.error;
+                }
+            } catch {
+                // Not a JSON response, do nothing and keep the original http error
+            }
+            throw new Error(errorMessage);
+        }
+        return response;
+    }
+}

public/scripts/extensions/tts/index.js+2 -0 Viewed

 import { enumIcons } from '../../slash-commands/SlashCommandCommonEnumsProvider.js';
 import { POPUP_TYPE, callGenericPopup } from '../../popup.js';
 import { GoogleTranslateTtsProvider } from './google-translate.js';
+import { GoogleNativeTtsProvider } from './google-native.js';
 import { ChatterboxTtsProvider } from './chatterbox.js';
 import { KokoroTtsProvider } from './kokoro.js';
 import { TtsWebuiProvider } from './tts-webui.js';

     Edge: EdgeTtsProvider,
     ElevenLabs: ElevenLabsTtsProvider,
     'Google Translate': GoogleTranslateTtsProvider,
+    'Google Gemini TTS': GoogleNativeTtsProvider,
     GSVI: GSVITtsProvider,
     'GPT-SoVITS-V2 (Unofficial)': GptSovitsV2Provider,
     Kokoro: KokoroTtsProvider,

src/endpoints/google.js+212 -60 Viewed

 import { readSecret, SECRET_KEYS } from './secrets.js';
 import { GEMINI_SAFETY } from '../constants.js';
-import { trimTrailingSlash } from '../util.js';
+import { getConfigValue, trimTrailingSlash } from '../util.js';
 const API_MAKERSUITE = 'https://generativelanguage.googleapis.com';
 const API_VERTEX_AI = 'https://us-central1-aiplatform.googleapis.com';
+function createWavHeader(dataSize, sampleRate, numChannels = 1, bitsPerSample = 16) {
+    const header = Buffer.alloc(44);
+    header.write('RIFF', 0);
+    header.writeUInt32LE(36 + dataSize, 4);
+    header.write('WAVE', 8);
+    header.write('fmt ', 12);
+    header.writeUInt32LE(16, 16);
+    header.writeUInt16LE(1, 20);
+    header.writeUInt16LE(numChannels, 22);
+    header.writeUInt32LE(sampleRate, 24);
+    header.writeUInt32LE(sampleRate * numChannels * bitsPerSample / 8, 28);
+    header.writeUInt16LE(numChannels * bitsPerSample / 8, 32);
+    header.writeUInt16LE(bitsPerSample, 34);
+    header.write('data', 36);
+    header.writeUInt32LE(dataSize, 40);
+    return header;
+}
+function createCompleteWavFile(pcmData, sampleRate) {
+    const header = createWavHeader(pcmData.length, sampleRate);
+    return Buffer.concat([header, pcmData]);
+}
 // Vertex AI authentication helper functions
 export async function getVertexAIAuth(request) {
     const authMode = request.body.vertexai_auth_mode || 'express';

     return projectId;
+}
+/**
+ * Generates Google API URL and headers based on request configuration
+ * @param {express.Request} request Express request object
+ * @param {string} model Model name to use
+ * @param {string} endpoint API endpoint (default: 'generateContent')
+ * @returns {Promise<{url: string, headers: object, apiName: string}>} URL, headers, and API name
+ */
+export async function getGoogleApiConfig(request, model, endpoint = 'generateContent') {
+    const useVertexAi = request.body.api === 'vertexai';
+    const region = request.body.vertexai_region || 'us-central1';
+    const apiName = useVertexAi ? 'Google Vertex AI' : 'Google AI Studio';
+    let url;
+    let headers = {
+        'Content-Type': 'application/json',
+    };
+    if (useVertexAi) {
+        // Get authentication for Vertex AI
+        const { authHeader, authType } = await getVertexAIAuth(request);
+        if (authType === 'express') {
+            // Express mode: use API key parameter
+            const keyParam = authHeader.replace('Bearer ', '');
+            const projectId = request.body.vertexai_express_project_id;
+            const baseUrl = region === 'global'
+                ? 'https://aiplatform.googleapis.com'
+                : `https://${region}-aiplatform.googleapis.com`;
+            url = projectId
+                ? `https://aiplatform.googleapis.com/v1/projects/${projectId}/locations/${region}/publishers/google/models/${model}:${endpoint}?key=${keyParam}`
+                : `${baseUrl}/v1/publishers/google/models/${model}:${endpoint}?key=${keyParam}`;
+        } else if (authType === 'full') {
+            // Full mode: use project-specific URL with Authorization header
+            // Get project ID from Service Account JSON
+            const serviceAccountJson = readSecret(request.user.directories, SECRET_KEYS.VERTEXAI_SERVICE_ACCOUNT);
+            if (!serviceAccountJson) {
+                throw new Error('Vertex AI Service Account JSON is missing.');
+            }
+            let projectId;
+            try {
+                const serviceAccount = JSON.parse(serviceAccountJson);
+                projectId = getProjectIdFromServiceAccount(serviceAccount);
+            } catch (error) {
+                throw new Error('Failed to extract project ID from Service Account JSON.');
+            }
+            // Handle global region differently - no region prefix in hostname
+            url = region === 'global'
+                ? `https://aiplatform.googleapis.com/v1/projects/${projectId}/locations/${region}/publishers/google/models/${model}:${endpoint}`
+                : `https://${region}-aiplatform.googleapis.com/v1/projects/${projectId}/locations/${region}/publishers/google/models/${model}:${endpoint}`;
+            headers['Authorization'] = authHeader;
+        } else {
+            // Proxy mode: use Authorization header
+            const apiUrl = trimTrailingSlash(request.body.reverse_proxy || API_VERTEX_AI);
+            url = `${apiUrl}/v1/publishers/google/models/${model}:${endpoint}`;
+            headers['Authorization'] = authHeader;
+        }
+    } else {
+        // Google AI Studio
+        const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MAKERSUITE);
+        const apiUrl = trimTrailingSlash(request.body.reverse_proxy || API_MAKERSUITE);
+        const apiVersion = getConfigValue('gemini.apiVersion', 'v1beta');
+        url = `${apiUrl}/${apiVersion}/models/${model}:${endpoint}?key=${apiKey}`;
+    }
+    return { url, headers, apiName };
+}
 export const router = express.Router();
 router.post('/caption-image', async (request, response) => {
     try {
         const mimeType = request.body.image.split(';')[0].split(':')[1];
         const base64Data = request.body.image.split(',')[1];
-        const useVertexAi = request.body.api === 'vertexai';
-        const apiName = useVertexAi ? 'Google Vertex AI' : 'Google AI Studio';
         const model = request.body.model || 'gemini-2.0-flash';
+        const { url, headers, apiName } = await getGoogleApiConfig(request, model);
-        let url;
-        let headers = {
-            'Content-Type': 'application/json',
-        };
-        if (useVertexAi) {
-            // Get authentication for Vertex AI
-            const { authHeader, authType } = await getVertexAIAuth(request);
-            if (authType === 'express') {
-                // Express mode: use API key parameter
-                const keyParam = authHeader.replace('Bearer ', '');
-                const region = request.body.vertexai_region || 'us-central1';
-                const projectId = request.body.vertexai_express_project_id;
-                const baseUrl = region === 'global'
-                    ? 'https://aiplatform.googleapis.com'
-                    : `https://${region}-aiplatform.googleapis.com`;
-                url = projectId
-                    ? `https://aiplatform.googleapis.com/v1/projects/${projectId}/locations/${region}/publishers/google/models/${model}:generateContent?key=${keyParam}`
-                    : `${baseUrl}/v1/publishers/google/models/${model}:generateContent?key=${keyParam}`;
-            } else if (authType === 'full') {
-                // Full mode: use project-specific URL with Authorization header
-                // Get project ID from Service Account JSON
-                const serviceAccountJson = readSecret(request.user.directories, SECRET_KEYS.VERTEXAI_SERVICE_ACCOUNT);
-                if (!serviceAccountJson) {
-                    console.warn('Vertex AI Service Account JSON is missing.');
-                    return response.status(400).send({ error: true });
-                }
-                let projectId;
-                try {
-                    const serviceAccount = JSON.parse(serviceAccountJson);
-                    projectId = getProjectIdFromServiceAccount(serviceAccount);
-                } catch (error) {
-                    console.error('Failed to extract project ID from Service Account JSON:', error);
-                    return response.status(400).send({ error: true });
-                }
-                const region = request.body.vertexai_region || 'us-central1';
-                // Handle global region differently - no region prefix in hostname
-                if (region === 'global') {
-                    url = `https://aiplatform.googleapis.com/v1/projects/${projectId}/locations/${region}/publishers/google/models/${model}:generateContent`;
-                } else {
-                    url = `https://${region}-aiplatform.googleapis.com/v1/projects/${projectId}/locations/${region}/publishers/google/models/${model}:generateContent`;
-                }
-                headers['Authorization'] = authHeader;
-            } else {
-                // Proxy mode: use Authorization header
-                const apiUrl = trimTrailingSlash(request.body.reverse_proxy || API_VERTEX_AI);
-                url = `${apiUrl}/v1/publishers/google/models/${model}:generateContent`;
-                headers['Authorization'] = authHeader;
-            }
-        } else {
-            // Google AI Studio
-            const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MAKERSUITE);
-            const apiUrl = trimTrailingSlash(request.body.reverse_proxy || API_MAKERSUITE);
-            url = `${apiUrl}/v1beta/models/${model}:generateContent?key=${apiKey}`;
-        }
         const body = {
             contents: [{
                 role: 'user',

         response.status(500).send('Internal server error');
+    }
 });
+router.post('/list-native-voices', async (_, response) => {
+    try {
+        // Hardcoded Gemini native TTS voices from official documentation
+        // Source: https://ai.google.dev/gemini-api/docs/speech-generation#voices
+        const voices = [
+            { name: 'Zephyr', voice_id: 'Zephyr', lang: 'en-US', description: 'Bright' },
+            { name: 'Puck', voice_id: 'Puck', lang: 'en-US', description: 'Upbeat' },
+            { name: 'Charon', voice_id: 'Charon', lang: 'en-US', description: 'Informative' },
+            { name: 'Kore', voice_id: 'Kore', lang: 'en-US', description: 'Firm' },
+            { name: 'Fenrir', voice_id: 'Fenrir', lang: 'en-US', description: 'Excitable' },
+            { name: 'Leda', voice_id: 'Leda', lang: 'en-US', description: 'Youthful' },
+            { name: 'Orus', voice_id: 'Orus', lang: 'en-US', description: 'Firm' },
+            { name: 'Aoede', voice_id: 'Aoede', lang: 'en-US', description: 'Breezy' },
+            { name: 'Callirhoe', voice_id: 'Callirhoe', lang: 'en-US', description: 'Easy-going' },
+            { name: 'Autonoe', voice_id: 'Autonoe', lang: 'en-US', description: 'Bright' },
+            { name: 'Enceladus', voice_id: 'Enceladus', lang: 'en-US', description: 'Breathy' },
+            { name: 'Iapetus', voice_id: 'Iapetus', lang: 'en-US', description: 'Clear' },
+            { name: 'Umbriel', voice_id: 'Umbriel', lang: 'en-US', description: 'Easy-going' },
+            { name: 'Algieba', voice_id: 'Algieba', lang: 'en-US', description: 'Smooth' },
+            { name: 'Despina', voice_id: 'Despina', lang: 'en-US', description: 'Smooth' },
+            { name: 'Erinome', voice_id: 'Erinome', lang: 'en-US', description: 'Clear' },
+            { name: 'Algenib', voice_id: 'Algenib', lang: 'en-US', description: 'Gravelly' },
+            { name: 'Rasalgethi', voice_id: 'Rasalgethi', lang: 'en-US', description: 'Informative' },
+            { name: 'Laomedeia', voice_id: 'Laomedeia', lang: 'en-US', description: 'Upbeat' },
+            { name: 'Achernar', voice_id: 'Achernar', lang: 'en-US', description: 'Soft' },
+            { name: 'Alnilam', voice_id: 'Alnilam', lang: 'en-US', description: 'Firm' },
+            { name: 'Schedar', voice_id: 'Schedar', lang: 'en-US', description: 'Even' },
+            { name: 'Gacrux', voice_id: 'Gacrux', lang: 'en-US', description: 'Mature' },
+            { name: 'Pulcherrima', voice_id: 'Pulcherrima', lang: 'en-US', description: 'Forward' },
+            { name: 'Achird', voice_id: 'Achird', lang: 'en-US', description: 'Friendly' },
+            { name: 'Zubenelgenubi', voice_id: 'Zubenelgenubi', lang: 'en-US', description: 'Casual' },
+            { name: 'Vindemiatrix', voice_id: 'Vindemiatrix', lang: 'en-US', description: 'Gentle' },
+            { name: 'Sadachbia', voice_id: 'Sadachbia', lang: 'en-US', description: 'Lively' },
+            { name: 'Sadaltager', voice_id: 'Sadaltager', lang: 'en-US', description: 'Knowledgeable' },
+            { name: 'Sulafat', voice_id: 'Sulafat', lang: 'en-US', description: 'Warm' },
+        ];
+        return response.json({ voices });
+    } catch (error) {
+        console.error('Failed to return Google TTS voices:', error);
+        response.sendStatus(500);
+    }
+});
+router.post('/generate-native-tts', async (request, response) => {
+    try {
+        const { text, voice, model } = request.body;
+        const { url, headers, apiName } = await getGoogleApiConfig(request, model);
+        console.debug(`${apiName} TTS request`, { model, text, voice });
+        const requestBody = {
+            contents: [{
+                role: 'user',
+                parts: [{ text: text }],
+            }],
+            generationConfig: {
+                responseModalities: ['AUDIO'],
+                speechConfig: {
+                    voiceConfig: {
+                        prebuiltVoiceConfig: {
+                            voiceName: voice,
+                        },
+                    },
+                },
+            },
+            safetySettings: GEMINI_SAFETY,
+        };
+        const result = await fetch(url, {
+            method: 'POST',
+            headers: headers,
+            body: JSON.stringify(requestBody),
+        });
+        if (!result.ok) {
+            const errorText = await result.text();
+            console.error(`${apiName} TTS API error: ${result.status} ${result.statusText}`, errorText);
+            const errorMessage = JSON.parse(errorText).error?.message || 'TTS generation failed.';
+            return response.status(result.status).json({ error: errorMessage });
+        }
+        /** @type {any} */
+        const data = await result.json();
+        const audioPart = data?.candidates?.[0]?.content?.parts?.[0];
+        const audioData = audioPart?.inlineData?.data;
+        const mimeType = audioPart?.inlineData?.mimeType;
+        if (!audioData) {
+            return response.status(500).json({ error: 'No audio data found in response' });
+        }
+        const audioBuffer = Buffer.from(audioData, 'base64');
+        //If the audio is raw PCM, wrap it in a WAV header and send it.
+        if (mimeType && mimeType.toLowerCase().includes('audio/l16')) {
+            const rateMatch = mimeType.match(/rate=(\d+)/);
+            const sampleRate = rateMatch ? parseInt(rateMatch[1], 10) : 24000;
+            const pcmData = audioBuffer;
+            // Create a complete, playable WAV file buffer.
+            const wavBuffer = createCompleteWavFile(pcmData, sampleRate);
+            // Send the WAV file directly to the browser. This is much faster.
+            response.setHeader('Content-Type', 'audio/wav');
+            return response.send(wavBuffer);
+        }
+        // Fallback for any other audio format Google might send in the future.
+        response.setHeader('Content-Type', mimeType || 'application/octet-stream');
+        response.send(audioBuffer);
+    } catch (error) {
+        console.error('Google TTS generation failed:', error);
+        if (!response.headersSent) {
+            return response.status(500).json({ error: 'Internal server error during TTS generation' });
+        }
+        return response.end();
+    }
+});