Refactor ElevenLabs TTS API key handling (#4906) * Refactor ElevenLabs TTS API key handling #4483 * Remove unused connection button and related event handler from ElevenLabs TTS provider * Add ElevenLabs STT endpoint

c75df956c598f995d543e383f55917d50ce2eaf5

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

Parents: 0dd32775d60d63880dc6adc63c69921ff9bd207d

public/scripts/extensions/tts/elevenlabs.js+131 -106 Viewed

 import { saveTtsProviderSettings } from './index.js';
+import { event_types, eventSource, getRequestHeaders } from '/script.js';
+import { SECRET_KEYS, secret_state, writeSecret } from '/scripts/secrets.js';
+import { getBase64Async } from '/scripts/utils.js';
 export { ElevenLabsTtsProvider };
 class ElevenLabsTtsProvider {
-    //########//
-    // Config //
-    //########//
     settings;
     voices = [];
     separator = ' ... ... ... ';
     defaultSettings = {
         stability: 0.75,
         similarity_boost: 0.75,
         style_exaggeration: 0.00,
         speaker_boost: true,
         speed: 1.0,
-        apiKey: '',
         model: 'eleven_turbo_v2_5',
         voiceMap: {},
     };

     get settingsHtml() {
         let html = `
         <div class="elevenlabs_tts_settings">
-            <label for="elevenlabs_tts_api_key">API Key</label>
+            <div class="flex-container alignItemsBaseline">
-            <input id="elevenlabs_tts_api_key" type="text" class="text_pole" placeholder="<API Key>"/>
+                <h4 for="elevenlabs_tts_key" class="flex1 margin0">
+                    <a href="https://elevenlabs.io/app/developers/api-keys" target="_blank">ElevenLabs TTS Key</a>
+                </h4>
+                <div id="elevenlabs_tts_key" class="menu_button menu_button_icon manage-api-keys" data-key="api_key_elevenlabs">
+                    <i class="fa-solid fa-key"></i>
+                    <span>Click to set</span>
+                </div>
+            </div>
             <label for="elevenlabs_tts_model">Model</label>
             <select id="elevenlabs_tts_model" class="text_pole">
                 <option value="eleven_v3">Eleven v3</option>

                 <option value="eleven_multilingual_v1">Multilingual v1 (Old)</option>
                 <option value="eleven_turbo_v2">Turbo v2 (Old)</option>
             </select>
-            <input id="eleven_labs_connect" class="menu_button" type="button" value="Connect" />
             <label for="elevenlabs_tts_stability">Stability: <span id="elevenlabs_tts_stability_output"></span></label>
             <input id="elevenlabs_tts_stability" type="range" value="${this.defaultSettings.stability}" min="0" max="1" step="0.01" />
             <label for="elevenlabs_tts_similarity_boost">Similarity Boost: <span id="elevenlabs_tts_similarity_boost_output"></span></label>

         return html;
+    }
+    constructor() {
+        this.handler = async function (/** @type {string} */ key) {
+            if (key !== SECRET_KEYS.ELEVENLABS) return;
+            $('#elevenlabs_tts_key').toggleClass('success', !!secret_state[SECRET_KEYS.ELEVENLABS]);
+            await this.fetchTtsVoiceObjects();
+        }.bind(this);
+    }
+    dispose() {
+        [event_types.SECRET_WRITTEN, event_types.SECRET_DELETED, event_types.SECRET_ROTATED].forEach(event => {
+            eventSource.removeListener(event, this.handler);
+        });
+    }
     shouldInvolveExtendedSettings() {
         // Models that support extended settings (style_exaggeration, speaker_boost)
         const modelsWithExtendedSettings = [

             delete settings['multilingual'];
+        }
+        if (Object.hasOwn(settings, 'apiKey')) {
+            if (settings.apiKey && !secret_state[SECRET_KEYS.ELEVENLABS]){
+                await writeSecret(SECRET_KEYS.ELEVENLABS, settings.apiKey);
+            }
+            delete settings['apiKey'];
+        }
+        $('#elevenlabs_tts_key').toggleClass('success', !!secret_state[SECRET_KEYS.ELEVENLABS]);
+        [event_types.SECRET_WRITTEN, event_types.SECRET_DELETED, event_types.SECRET_ROTATED].forEach(event => {
+            eventSource.on(event, this.handler);
+        });
         for (const key in settings) {
             if (key in this.settings) {
                 this.settings[key] = settings[key];

         $('#elevenlabs_tts_style_exaggeration').val(this.settings.style_exaggeration);
         $('#elevenlabs_tts_speaker_boost').prop('checked', this.settings.speaker_boost);
         $('#elevenlabs_tts_speed').val(this.settings.speed);
-        $('#elevenlabs_tts_api_key').val(this.settings.apiKey);
         $('#elevenlabs_tts_model').val(this.settings.model);
-        $('#eleven_labs_connect').on('click', () => { this.onConnectClick(); });
         $('#elevenlabs_tts_similarity_boost').on('input', this.onSettingsChange.bind(this));
         $('#elevenlabs_tts_stability').on('input', this.onSettingsChange.bind(this));
         $('#elevenlabs_tts_style_exaggeration').on('input', this.onSettingsChange.bind(this));

+    }
     async onRefreshClick() {
-    }
+        await this.fetchTtsVoiceObjects();
-    async onConnectClick() {
-        // Update on Apply click
-        return await this.updateApiKey().catch((error) => {
-            toastr.error(`ElevenLabs: ${error}`);
-        });
+    }
     setupVoiceCloningMenu() {
-        const audioFilesInput = document.getElementById('elevenlabs_tts_audio_files');
+        const audioFilesInput = /** @type {HTMLInputElement} */ (document.getElementById('elevenlabs_tts_audio_files'));
         const selectedFilesListElement = document.getElementById('elevenlabs_tts_selected_files_list');
         const cloneVoiceButton = document.getElementById('elevenlabs_tts_clone_voice_button');
         const uploadAudioFileButton = document.getElementById('upload_audio_file');
-        const voiceCloningNameInput = document.getElementById('elevenlabs_tts_voice_cloning_name');
+        const voiceCloningNameInput = /** @type {HTMLInputElement} */ (document.getElementById('elevenlabs_tts_voice_cloning_name'));
-        const voiceCloningDescriptionInput = document.getElementById('elevenlabs_tts_voice_cloning_description');
+        const voiceCloningDescriptionInput = /** @type {HTMLInputElement} */ (document.getElementById('elevenlabs_tts_voice_cloning_description'));
-        const voiceCloningLabelsInput = document.getElementById('elevenlabs_tts_voice_cloning_labels');
+        const voiceCloningLabelsInput = /** @type {HTMLInputElement} */ (document.getElementById('elevenlabs_tts_voice_cloning_labels'));
         const updateCloneVoiceButtonVisibility = () => {
             cloneVoiceButton.style.display = audioFilesInput.files.length > 0 ? 'inline-block' : 'none';

         updateCloneVoiceButtonVisibility();
+    }
-    async updateApiKey() {
+    /**
-        // Using this call to validate API key
+     * Get voice object by name
-        this.settings.apiKey = $('#elevenlabs_tts_api_key').val();
+     * @param {string} voiceName Voice name to look up
+     * @returns {Promise<Object>} Voice object
-        await this.fetchTtsVoiceObjects().catch(error => {
+     */
-            throw 'TTS API key validation failed';
-        });
-        console.debug(`Saved new API_KEY: ${this.settings.apiKey}`);
-        $('#tts_status').text('');
-        this.onSettingsChange();
-    }
-    //#################//
-    //  TTS Interfaces //
-    //#################//
     async getVoice(voiceName) {
         if (this.voices.length == 0) {
             this.voices = await this.fetchTtsVoiceObjects();

         return match;
+    }
+    /**
+     * Generate TTS audio
+     * @param {string} text Text to synthesize
+     * @param {string} voiceId Voice ID to use for synthesis
+     * @returns {Promise<Response>} Response object containing audio data
+     */
     async generateTts(text, voiceId) {
         const historyId = await this.findTtsGenerationInHistory(text, voiceId);
-        let response;
         if (historyId) {
             console.debug(`Found existing TTS generation with id ${historyId}`);
-            response = await this.fetchTtsFromHistory(historyId);
+            return await this.fetchTtsFromHistory(historyId);
         } else {
             console.debug('No existing TTS generation found, requesting new generation');
-            response = await this.fetchTtsGeneration(text, voiceId);
+            return await this.fetchTtsGeneration(text, voiceId);
+        }
-        return response;
+    }
-    //###################//
+    /**
-    //  Helper Functions //
+     * Find existing TTS generation in history
-    //###################//
+     * @param {string} message Message text used for TTS generation
+     * @param {string} voiceId Voice ID used for TTS generation
+     * @returns {Promise<string>} History item ID if found, empty string otherwise
+     */
     async findTtsGenerationInHistory(message, voiceId) {
         const ttsHistory = await this.fetchTtsHistory();
         for (const history of ttsHistory) {

         return '';
+    }
-    //###########//
-    // API CALLS //
-    //###########//
     async fetchTtsVoiceObjects() {
-        const headers = {
+        const response = await fetch('/api/speech/elevenlabs/voices', {
-            'xi-api-key': this.settings.apiKey,
+            method: 'POST',
-        };
+            headers: getRequestHeaders({ omitContentType: true }),
-        const response = await fetch('https://api.elevenlabs.io/v1/voices', {
-            headers: headers,
         });
         if (!response.ok) {
-            throw new Error(`HTTP ${response.status}: ${await response.text()}`);
+            throw new Error(`HTTP ${response.status}. See server console for details.`);
+        }
         const responseJson = await response.json();
         return responseJson.voices;
+    }
     async fetchTtsVoiceSettings() {
-        const headers = {
+        const response = await fetch('/api/speech/elevenlabs/voice-settings', {
-            'xi-api-key': this.settings.apiKey,
+            method: 'POST',
-        };
+            headers: getRequestHeaders({ omitContentType: true }),
-        const response = await fetch(
+        });
-            'https://api.elevenlabs.io/v1/voices/settings/default',
-            {
-                headers: headers,
-            },
-        );
         if (!response.ok) {
-            throw new Error(`HTTP ${response.status}: ${await response.text()}`);
+            throw new Error(`HTTP ${response.status}. See server console for details.`);
+        }
         return response.json();
+    }
+    /**
+     * Fetch new TTS generation from ElevenLabs API
+     * @param {string} text Text to synthesize
+     * @param {string} voiceId Voice ID to use for synthesis
+     * @returns {Promise<Response>} Response object containing audio data
+     */
     async fetchTtsGeneration(text, voiceId) {
         let model = this.settings.model ?? 'eleven_monolingual_v1';
         console.info(`Generating new TTS for voice_id ${voiceId}, model ${model}`);

             request.voice_settings.style = Number(this.settings.style_exaggeration);
             request.voice_settings.use_speaker_boost = Boolean(this.settings.speaker_boost);
+        }
-        const response = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`, {
+        const response = await fetch('/api/speech/elevenlabs/synthesize', {
             method: 'POST',
-            headers: {
+            headers: getRequestHeaders(),
-                'xi-api-key': this.settings.apiKey,
+            body: JSON.stringify({
-                'Content-Type': 'application/json',
+                voiceId: voiceId,
-            },
+                request: request,
-            body: JSON.stringify(request),
+            }),
         });
         if (!response.ok) {
             toastr.error(response.statusText, 'TTS Generation Failed');
-            throw new Error(`HTTP ${response.status}: ${await response.text()}`);
+            throw new Error(`HTTP ${response.status}. See server console for details.`);
+        }
         return response;
+    }
-    async fetchTtsFromHistory(history_item_id) {
+    /**
-        console.info(`Fetched existing TTS with history_item_id ${history_item_id}`);
+     * Fetch existing TTS audio from history
-        const response = await fetch(
+     * @param {string} historyItemId History item ID to fetch audio for
-            `https://api.elevenlabs.io/v1/history/${history_item_id}/audio`,
+     * @returns {Promise<Response>} Response object containing audio data
-            {
+     */
-                headers: {
+    async fetchTtsFromHistory(historyItemId) {
-                    'xi-api-key': this.settings.apiKey,
+        console.info(`Fetched existing TTS with history_item_id ${historyItemId}`);
-                },
+        const response = await fetch('/api/speech/elevenlabs/history-audio', {
-            },
+            method: 'POST',
-        );
+            headers: getRequestHeaders(),
+            body: JSON.stringify({
+                historyItemId: historyItemId,
+            }),
+        });
         if (!response.ok) {
-            throw new Error(`HTTP ${response.status}: ${await response.text()}`);
+            throw new Error(`HTTP ${response.status}. See server console for details.`);
+        }
         return response;
+    }
+    /**
+     * Fetch TTS generation history
+     * @returns {Promise<Array>} Array of TTS history items
+     */
     async fetchTtsHistory() {
-        const headers = {
+        const response = await fetch('/api/speech/elevenlabs/history', {
-            'xi-api-key': this.settings.apiKey,
+            method: 'POST',
-        };
+            headers: getRequestHeaders({ omitContentType: true }),
-        const response = await fetch('https://api.elevenlabs.io/v1/history', {
-            headers: headers,
         });
         if (!response.ok) {
-            throw new Error(`HTTP ${response.status}: ${await response.text()}`);
+            throw new Error(`HTTP ${response.status}. See server console for details.`);
+        }
         const responseJson = await response.json();
         return responseJson.history;
+    }
+    /**
+     * Add a new voice via ElevenLabs API
+     * @param {string} name Voice name
+     * @param {string} description Voice description
+     * @param {string} labels Voice labels
+     * @returns {Promise<Object>} Newly created voice object
+     */
     async addVoice(name, description, labels) {
-        const selected_files = document.querySelectorAll('input[type="file"][name="audio_files"]');
+        const audioFilesInput = /** @type {HTMLInputElement} */ (document.getElementById('elevenlabs_tts_audio_files'));
-        const formData = new FormData();
+        if (!(audioFilesInput instanceof HTMLInputElement) || audioFilesInput.files.length === 0) {
+            throw new Error('No audio files selected for voice cloning.');
+        }
-        formData.append('name', name);
+        const data = {
-        formData.append('description', description);
+            name: name,
-        formData.append('labels', labels);
+            description: description,
+            labels: labels,
+            files: [],
+        };
-        for (const file of selected_files) {
+        for (const file of audioFilesInput.files) {
-            if (file.files.length > 0) {
+            const base64Data = await getBase64Async(file);
-                formData.append('files', file.files[0]);
+            data.files.push(base64Data);
-            }
+        }
-        const response = await fetch('https://api.elevenlabs.io/v1/voices/add', {
+        const response = await fetch('/api/speech/elevenlabs/voices/add', {
             method: 'POST',
-            headers: {
+            headers: getRequestHeaders(),
-                'xi-api-key': this.settings.apiKey,
+            body: JSON.stringify(data),
-            },
-            body: formData,
         });
         if (!response.ok) {
-            throw new Error(`HTTP ${response.status}: ${await response.text()}`);
+            throw new Error(`HTTP ${response.status}. See server console for details.`);
+        }
         return await response.json();

public/scripts/secrets.js+2 -0 Viewed

     COMETAPI: 'api_key_cometapi',
     ZAI: 'api_key_zai',
     SILICONFLOW: 'api_key_siliconflow',
+    ELEVENLABS: 'api_key_elevenlabs',
 };
 const FRIENDLY_NAMES = {

     [SECRET_KEYS.AZURE_OPENAI]: 'Azure OpenAI',
     [SECRET_KEYS.ZAI]: 'Z.AI',
     [SECRET_KEYS.SILICONFLOW]: 'SiliconFlow',
+    [SECRET_KEYS.ELEVENLABS]: 'ElevenLabs TTS',
 };
 const INPUT_MAP = {

src/endpoints/secrets.js+1 -0 Viewed

     AZURE_OPENAI: 'api_key_azure_openai',
     ZAI: 'api_key_zai',
     SILICONFLOW: 'api_key_siliconflow',
+    ELEVENLABS: 'api_key_elevenlabs',
 };
 /**

src/endpoints/speech.js+261 -0 Viewed

 import { Buffer } from 'node:buffer';
+import fs from 'node:fs';
 import express from 'express';
 import wavefile from 'wavefile';
 import fetch from 'node-fetch';
+import FormData from 'form-data';
+import mime from 'mime-types';
 import { getPipeline } from '../transformers.js';
 import { forwardFetchResponse } from '../util.js';
+import { readSecret, SECRET_KEYS } from './secrets.js';
 export const router = express.Router();

 });
 router.use('/pollinations', pollinations);
+const elevenlabs = express.Router();
+elevenlabs.post('/voices', async (req, res) => {
+    try {
+        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);
+        if (!apiKey) {
+            console.warn('ElevenLabs API key not found');
+            return res.sendStatus(400);
+        }
+        const response = await fetch('https://api.elevenlabs.io/v1/voices', {
+            headers: {
+                'xi-api-key': apiKey,
+            },
+        });
+        if (!response.ok) {
+            const text = await response.text();
+            console.warn(`ElevenLabs voices fetch failed: HTTP ${response.status} - ${text}`);
+            return res.sendStatus(500);
+        }
+        const responseJson = await response.json();
+        return res.json(responseJson);
+    } catch (error) {
+        console.error(error);
+        return res.sendStatus(500);
+    }
+});
+elevenlabs.post('/voice-settings', async (req, res) => {
+    try {
+        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);
+        if (!apiKey) {
+            console.warn('ElevenLabs API key not found');
+            return res.sendStatus(400);
+        }
+        const response = await fetch('https://api.elevenlabs.io/v1/voices/settings/default', {
+            headers: {
+                'xi-api-key': apiKey,
+            },
+        });
+        if (!response.ok) {
+            const text = await response.text();
+            console.warn(`ElevenLabs voice settings fetch failed: HTTP ${response.status} - ${text}`);
+            return res.sendStatus(500);
+        }
+        const responseJson = await response.json();
+        return res.json(responseJson);
+    } catch (error) {
+        console.error(error);
+        return res.sendStatus(500);
+    }
+});
+elevenlabs.post('/synthesize', async (req, res) => {
+    try {
+        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);
+        if (!apiKey) {
+            console.warn('ElevenLabs API key not found');
+            return res.sendStatus(400);
+        }
+        const { voiceId, request } = req.body;
+        if (!voiceId || !request) {
+            console.warn('ElevenLabs synthesis request missing voiceId or request body');
+            return res.sendStatus(400);
+        }
+        console.debug('ElevenLabs TTS request:', request);
+        const response = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`, {
+            method: 'POST',
+            headers: {
+                'xi-api-key': apiKey,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify(request),
+        });
+        if (!response.ok) {
+            const text = await response.text();
+            console.warn(`ElevenLabs synthesis failed: HTTP ${response.status} - ${text}`);
+            return res.sendStatus(500);
+        }
+        res.set('Content-Type', 'audio/mpeg');
+        forwardFetchResponse(response, res);
+    } catch (error) {
+        console.error(error);
+        return res.sendStatus(500);
+    }
+});
+elevenlabs.post('/history', async (req, res) => {
+    try {
+        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);
+        if (!apiKey) {
+            console.warn('ElevenLabs API key not found');
+            return res.sendStatus(400);
+        }
+        const response = await fetch('https://api.elevenlabs.io/v1/history', {
+            headers: {
+                'xi-api-key': apiKey,
+            },
+        });
+        if (!response.ok) {
+            const text = await response.text();
+            console.warn(`ElevenLabs history fetch failed: HTTP ${response.status} - ${text}`);
+            return res.sendStatus(500);
+        }
+        const responseJson = await response.json();
+        return res.json(responseJson);
+    } catch (error) {
+        console.error(error);
+        return res.sendStatus(500);
+    }
+});
+elevenlabs.post('/history-audio', async (req, res) => {
+    try {
+        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);
+        if (!apiKey) {
+            console.warn('ElevenLabs API key not found');
+            return res.sendStatus(400);
+        }
+        const { historyItemId } = req.body;
+        if (!historyItemId) {
+            console.warn('ElevenLabs history audio request missing historyItemId');
+            return res.sendStatus(400);
+        }
+        console.debug('ElevenLabs history audio request for ID:', historyItemId);
+        const response = await fetch(`https://api.elevenlabs.io/v1/history/${historyItemId}/audio`, {
+            headers: {
+                'xi-api-key': apiKey,
+            },
+        });
+        if (!response.ok) {
+            const text = await response.text();
+            console.warn(`ElevenLabs history audio fetch failed: HTTP ${response.status} - ${text}`);
+            return res.sendStatus(500);
+        }
+        res.set('Content-Type', 'audio/mpeg');
+        forwardFetchResponse(response, res);
+    } catch (error) {
+        console.error(error);
+        return res.sendStatus(500);
+    }
+});
+elevenlabs.post('/voices/add', async (req, res) => {
+    try {
+        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);
+        if (!apiKey) {
+            console.warn('ElevenLabs API key not found');
+            return res.sendStatus(400);
+        }
+        const { name, description, labels, files } = req.body;
+        const formData = new FormData();
+        formData.append('name', name || 'Custom Voice');
+        formData.append('description', description || 'Uploaded via SillyTavern');
+        formData.append('labels', labels || '');
+        for (const fileData of (files || [])) {
+            const [mimeType, base64Data] = /^data:(.+);base64,(.+)$/.exec(fileData)?.slice(1) || [];
+            if (!mimeType || !base64Data) {
+                console.warn('Invalid audio file data provided for ElevenLabs voice upload');
+                continue;
+            }
+            const buffer = Buffer.from(base64Data, 'base64');
+            formData.append('files', buffer, {
+                filename: `audio.${mime.extension(mimeType) || 'wav'}`,
+                contentType: mimeType,
+            });
+        }
+        console.debug('ElevenLabs voice upload request:', { name, description, labels, files: files?.length || 0 });
+        const response = await fetch('https://api.elevenlabs.io/v1/voices/add', {
+            method: 'POST',
+            headers: {
+                'xi-api-key': apiKey,
+            },
+            body: formData,
+        });
+        if (!response.ok) {
+            const text = await response.text();
+            console.warn(`ElevenLabs voice upload failed: HTTP ${response.status} - ${text}`);
+            return res.sendStatus(500);
+        }
+        const responseJson = await response.json();
+        return res.json(responseJson);
+    } catch (error) {
+        console.error(error);
+        return res.sendStatus(500);
+    }
+});
+elevenlabs.post('/recognize', async (req, res) => {
+    try {
+        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);
+        if (!apiKey) {
+            console.warn('ElevenLabs API key not found');
+            return res.sendStatus(400);
+        }
+        if (!req.file) {
+            console.warn('No audio file found');
+            return res.sendStatus(400);
+        }
+        console.info('Processing audio file with ElevenLabs', req.file.path);
+        const formData = new FormData();
+        formData.append('file', fs.createReadStream(req.file.path), { filename: 'audio.wav', contentType: 'audio/wav' });
+        formData.append('model_id', req.body.model);
+        const response = await fetch('https://api.elevenlabs.io/v1/speech-to-text', {
+            method: 'POST',
+            headers: {
+                'xi-api-key': apiKey,
+            },
+            body: formData,
+        });
+        if (!response.ok) {
+            const text = await response.text();
+            console.warn(`ElevenLabs speech recognition failed: HTTP ${response.status} - ${text}`);
+            return res.sendStatus(500);
+        }
+        fs.unlinkSync(req.file.path);
+        const responseJson = await response.json();
+        console.debug('ElevenLabs speech recognition response:', responseJson);
+        return res.json(responseJson);
+    } catch (error) {
+        console.error(error);
+        return res.sendStatus(500);
+    }
+});
+router.use('/elevenlabs', elevenlabs);