Refactor ElevenLabs TTS API key handling (#4906) * Refactor ElevenLabs TTS API key handling #4483 * Remove unused connection button and related event handler from ElevenLabs TTS provider * Add ElevenLabs STT endpoint

c75df956c598f995d543e383f55917d50ce2eaf5

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

Parents: 0dd32775d60d63880dc6adc63c69921ff9bd207d

public/scripts/extensions/tts/elevenlabs.js+131 -106 Viewed

 import { saveTtsProviderSettings } from './index.js';
 +import { event_types, eventSource, getRequestHeaders } from '/script.js';
 +import { SECRET_KEYS, secret_state, writeSecret } from '/scripts/secrets.js';
 +import { getBase64Async } from '/scripts/utils.js';
 export { ElevenLabsTtsProvider };
 class ElevenLabsTtsProvider {
 -    //########//
 -    // Config //
 -    //########//
+-
     settings;
     voices = [];
     separator = ' ... ... ... ';
+-
     defaultSettings = {
         stability: 0.75,
         similarity_boost: 0.75,
         style_exaggeration: 0.00,
         speaker_boost: true,
         speed: 1.0,
 -        apiKey: '',
         model: 'eleven_turbo_v2_5',
         voiceMap: {},
     };

     get settingsHtml() {
         let html = `
         <div class="elevenlabs_tts_settings">
 -            <label for="elevenlabs_tts_api_key">API Key</label>
 +            <div class="flex-container alignItemsBaseline">
 -            <input id="elevenlabs_tts_api_key" type="text" class="text_pole" placeholder="<API Key>"/>
 +                <h4 for="elevenlabs_tts_key" class="flex1 margin0">
 +                    <a href="https://elevenlabs.io/app/developers/api-keys" target="_blank">ElevenLabs TTS Key</a>
 +                </h4>
 +                <div id="elevenlabs_tts_key" class="menu_button menu_button_icon manage-api-keys" data-key="api_key_elevenlabs">
 +                    <i class="fa-solid fa-key"></i>
 +                    <span>Click to set</span>
 +                </div>
 +            </div>
             <label for="elevenlabs_tts_model">Model</label>
             <select id="elevenlabs_tts_model" class="text_pole">
                 <option value="eleven_v3">Eleven v3</option>

                 <option value="eleven_multilingual_v1">Multilingual v1 (Old)</option>
                 <option value="eleven_turbo_v2">Turbo v2 (Old)</option>
             </select>
 -            <input id="eleven_labs_connect" class="menu_button" type="button" value="Connect" />
             <label for="elevenlabs_tts_stability">Stability: <span id="elevenlabs_tts_stability_output"></span></label>
             <input id="elevenlabs_tts_stability" type="range" value="${this.defaultSettings.stability}" min="0" max="1" step="0.01" />
             <label for="elevenlabs_tts_similarity_boost">Similarity Boost: <span id="elevenlabs_tts_similarity_boost_output"></span></label>

         return html;
+    }
 +    constructor() {
 +        this.handler = async function (/** @type {string} */ key) {
 +            if (key !== SECRET_KEYS.ELEVENLABS) return;
 +            $('#elevenlabs_tts_key').toggleClass('success', !!secret_state[SECRET_KEYS.ELEVENLABS]);
 +            await this.fetchTtsVoiceObjects();
 +        }.bind(this);
 +    }
++
 +    dispose() {
 +        [event_types.SECRET_WRITTEN, event_types.SECRET_DELETED, event_types.SECRET_ROTATED].forEach(event => {
 +            eventSource.removeListener(event, this.handler);
 +        });
 +    }
++
     shouldInvolveExtendedSettings() {
         // Models that support extended settings (style_exaggeration, speaker_boost)
         const modelsWithExtendedSettings = [

             delete settings['multilingual'];
+        }
 +        if (Object.hasOwn(settings, 'apiKey')) {
 +            if (settings.apiKey && !secret_state[SECRET_KEYS.ELEVENLABS]){
 +                await writeSecret(SECRET_KEYS.ELEVENLABS, settings.apiKey);
 +            }
 +            delete settings['apiKey'];
 +        }
++
 +        $('#elevenlabs_tts_key').toggleClass('success', !!secret_state[SECRET_KEYS.ELEVENLABS]);
 +        [event_types.SECRET_WRITTEN, event_types.SECRET_DELETED, event_types.SECRET_ROTATED].forEach(event => {
 +            eventSource.on(event, this.handler);
 +        });
++
         for (const key in settings) {
             if (key in this.settings) {
                 this.settings[key] = settings[key];

         $('#elevenlabs_tts_style_exaggeration').val(this.settings.style_exaggeration);
         $('#elevenlabs_tts_speaker_boost').prop('checked', this.settings.speaker_boost);
         $('#elevenlabs_tts_speed').val(this.settings.speed);
 -        $('#elevenlabs_tts_api_key').val(this.settings.apiKey);
         $('#elevenlabs_tts_model').val(this.settings.model);
 -        $('#eleven_labs_connect').on('click', () => { this.onConnectClick(); });
         $('#elevenlabs_tts_similarity_boost').on('input', this.onSettingsChange.bind(this));
         $('#elevenlabs_tts_stability').on('input', this.onSettingsChange.bind(this));
         $('#elevenlabs_tts_style_exaggeration').on('input', this.onSettingsChange.bind(this));

+    }
     async onRefreshClick() {
 -    }
 +        await this.fetchTtsVoiceObjects();
+-
 -    async onConnectClick() {
 -        // Update on Apply click
 -        return await this.updateApiKey().catch((error) => {
 -            toastr.error(`ElevenLabs: ${error}`);
 -        });
+    }
     setupVoiceCloningMenu() {
         const audioFilesInput = /** @type {HTMLInputElement} */ (document.getElementById('elevenlabs_tts_audio_files'));
         const selectedFilesListElement = document.getElementById('elevenlabs_tts_selected_files_list');
         const cloneVoiceButton = document.getElementById('elevenlabs_tts_clone_voice_button');
         const uploadAudioFileButton = document.getElementById('upload_audio_file');
         const voiceCloningNameInput = /** @type {HTMLInputElement} */ (document.getElementById('elevenlabs_tts_voice_cloning_name'));
         const voiceCloningDescriptionInput = /** @type {HTMLInputElement} */ (document.getElementById('elevenlabs_tts_voice_cloning_description'));
         const voiceCloningLabelsInput = /** @type {HTMLInputElement} */ (document.getElementById('elevenlabs_tts_voice_cloning_labels'));
         const updateCloneVoiceButtonVisibility = () => {
             cloneVoiceButton.style.display = audioFilesInput.files.length > 0 ? 'inline-block' : 'none';

         updateCloneVoiceButtonVisibility();
+    }
 -    async updateApiKey() {
 +    /**
 -        // Using this call to validate API key
 +     * Get voice object by name
 -        this.settings.apiKey = $('#elevenlabs_tts_api_key').val();
 +     * @param {string} voiceName Voice name to look up
+-
 +     * @returns {Promise<Object>} Voice object
 -        await this.fetchTtsVoiceObjects().catch(error => {
 +     */
 -            throw 'TTS API key validation failed';
 -        });
 -        console.debug(`Saved new API_KEY: ${this.settings.apiKey}`);
 -        $('#tts_status').text('');
 -        this.onSettingsChange();
 -    }
+-
 -    //#################//
 -    //  TTS Interfaces //
 -    //#################//
+-
     async getVoice(voiceName) {
         if (this.voices.length == 0) {
             this.voices = await this.fetchTtsVoiceObjects();

         return match;
+    }
+-
 +    /**
 +     * Generate TTS audio
 +     * @param {string} text Text to synthesize
 +     * @param {string} voiceId Voice ID to use for synthesis
 +     * @returns {Promise<Response>} Response object containing audio data
 +     */
     async generateTts(text, voiceId) {
         const historyId = await this.findTtsGenerationInHistory(text, voiceId);
 -        let response;
         if (historyId) {
             console.debug(`Found existing TTS generation with id ${historyId}`);
             response =return await this.fetchTtsFromHistory(historyId);
         } else {
             console.debug('No existing TTS generation found, requesting new generation');
             response =return await this.fetchTtsGeneration(text, voiceId);
+        }
 -        return response;
+    }
 -    //###################//
 +    /**
 -    //  Helper Functions //
 +     * Find existing TTS generation in history
 -    //###################//
 +     * @param {string} message Message text used for TTS generation
+-
 +     * @param {string} voiceId Voice ID used for TTS generation
 +     * @returns {Promise<string>} History item ID if found, empty string otherwise
 +     */
     async findTtsGenerationInHistory(message, voiceId) {
         const ttsHistory = await this.fetchTtsHistory();
         for (const history of ttsHistory) {

         return '';
+    }
+-
 -    //###########//
 -    // API CALLS //
 -    //###########//
     async fetchTtsVoiceObjects() {
 -        const headers = {
 +        const response = await fetch('/api/speech/elevenlabs/voices', {
 -            'xi-api-key': this.settings.apiKey,
 +            method: 'POST',
 -        };
 +            headers: getRequestHeaders({ omitContentType: true }),
 -        const response = await fetch('https://api.elevenlabs.io/v1/voices', {
 -            headers: headers,
         });
         if (!response.ok) {
             throw new Error(`HTTP ${response.status}:. ${awaitSee responseserver console for details.text()}`);
+        }
         const responseJson = await response.json();
         return responseJson.voices;
+    }
     async fetchTtsVoiceSettings() {
 -        const headers = {
 +        const response = await fetch('/api/speech/elevenlabs/voice-settings', {
 -            'xi-api-key': this.settings.apiKey,
 +            method: 'POST',
 -        };
 +            headers: getRequestHeaders({ omitContentType: true }),
 -        const response = await fetch(
 +        });
 -            'https://api.elevenlabs.io/v1/voices/settings/default',
 -            {
 -                headers: headers,
 -            },
 -        );
         if (!response.ok) {
             throw new Error(`HTTP ${response.status}:. ${awaitSee responseserver console for details.text()}`);
+        }
         return response.json();
+    }
 +    /**
 +     * Fetch new TTS generation from ElevenLabs API
 +     * @param {string} text Text to synthesize
 +     * @param {string} voiceId Voice ID to use for synthesis
 +     * @returns {Promise<Response>} Response object containing audio data
 +     */
     async fetchTtsGeneration(text, voiceId) {
         let model = this.settings.model ?? 'eleven_monolingual_v1';
         console.info(`Generating new TTS for voice_id ${voiceId}, model ${model}`);

             request.voice_settings.style = Number(this.settings.style_exaggeration);
             request.voice_settings.use_speaker_boost = Boolean(this.settings.speaker_boost);
+        }
         const response = await fetch(`https:/'/api.elevenlabs.io/v1/text-to-speech/${voiceId}`elevenlabs/synthesize', {
             method: 'POST',
             headers: {getRequestHeaders(),
 -                'xi-api-key': this.settings.apiKey,
 +            body: JSON.stringify({
 -                'Content-Type': 'application/json',
 +                voiceId: voiceId,
 -            },
 +                request: request,
 -            body: JSON.stringify(request),
 +            }),
         });
         if (!response.ok) {
             toastr.error(response.statusText, 'TTS Generation Failed');
             throw new Error(`HTTP ${response.status}:. ${awaitSee responseserver console for details.text()}`);
+        }
         return response;
+    }
 -    async fetchTtsFromHistory(history_item_id) {
 +    /**
 -        console.info(`Fetched existing TTS with history_item_id ${history_item_id}`);
 +     * Fetch existing TTS audio from history
 -        const response = await fetch(
 +     * @param {string} historyItemId History item ID to fetch audio for
 -            `https://api.elevenlabs.io/v1/history/${history_item_id}/audio`,
 +     * @returns {Promise<Response>} Response object containing audio data
 -            {
 +     */
 -                headers: {
 +    async fetchTtsFromHistory(historyItemId) {
 -                    'xi-api-key': this.settings.apiKey,
 +        console.info(`Fetched existing TTS with history_item_id ${historyItemId}`);
 -                },
 +        const response = await fetch('/api/speech/elevenlabs/history-audio', {
 -            },
 +            method: 'POST',
 -        );
 +            headers: getRequestHeaders(),
 +            body: JSON.stringify({
 +                historyItemId: historyItemId,
 +            }),
 +        });
         if (!response.ok) {
             throw new Error(`HTTP ${response.status}:. ${awaitSee responseserver console for details.text()}`);
+        }
         return response;
+    }
 +    /**
 +     * Fetch TTS generation history
 +     * @returns {Promise<Array>} Array of TTS history items
 +     */
     async fetchTtsHistory() {
 -        const headers = {
 +        const response = await fetch('/api/speech/elevenlabs/history', {
 -            'xi-api-key': this.settings.apiKey,
 +            method: 'POST',
 -        };
 +            headers: getRequestHeaders({ omitContentType: true }),
 -        const response = await fetch('https://api.elevenlabs.io/v1/history', {
 -            headers: headers,
         });
         if (!response.ok) {
             throw new Error(`HTTP ${response.status}:. ${awaitSee responseserver console for details.text()}`);
+        }
         const responseJson = await response.json();
         return responseJson.history;
+    }
 +    /**
 +     * Add a new voice via ElevenLabs API
 +     * @param {string} name Voice name
 +     * @param {string} description Voice description
 +     * @param {string} labels Voice labels
 +     * @returns {Promise<Object>} Newly created voice object
 +     */
     async addVoice(name, description, labels) {
 -        const selected_files = document.querySelectorAll('input[type="file"][name="audio_files"]');
 +        const audioFilesInput = /** @type {HTMLInputElement} */ (document.getElementById('elevenlabs_tts_audio_files'));
 -        const formData = new FormData();
 +        if (!(audioFilesInput instanceof HTMLInputElement) || audioFilesInput.files.length === 0) {
 +            throw new Error('No audio files selected for voice cloning.');
 +        }
 -        formData.append('name', name);
 +        const data = {
 -        formData.append('description', description);
 +            name: name,
 -        formData.append('labels', labels);
 +            description: description,
 +            labels: labels,
 +            files: [],
 +        };
         for (const file of selected_filesaudioFilesInput.files) {
 -            if (file.files.length > 0) {
 +            const base64Data = await getBase64Async(file);
 -                formData.append('files', file.files[0]);
 +            data.files.push(base64Data);
 -            }
+        }
         const response = await fetch('https://api.elevenlabs.io/v1speech/elevenlabs/voices/add', {
             method: 'POST',
             headers: {getRequestHeaders(),
 -                'xi-api-key': this.settings.apiKey,
 +            body: JSON.stringify(data),
 -            },
 -            body: formData,
         });
         if (!response.ok) {
             throw new Error(`HTTP ${response.status}:. ${awaitSee responseserver console for details.text()}`);
+        }
         return await response.json();

public/scripts/secrets.js+2 -0 Viewed

     COMETAPI: 'api_key_cometapi',
     ZAI: 'api_key_zai',
     SILICONFLOW: 'api_key_siliconflow',
 +    ELEVENLABS: 'api_key_elevenlabs',
 };
 const FRIENDLY_NAMES = {

     [SECRET_KEYS.AZURE_OPENAI]: 'Azure OpenAI',
     [SECRET_KEYS.ZAI]: 'Z.AI',
     [SECRET_KEYS.SILICONFLOW]: 'SiliconFlow',
 +    [SECRET_KEYS.ELEVENLABS]: 'ElevenLabs TTS',
 };
 const INPUT_MAP = {

src/endpoints/secrets.js+1 -0 Viewed

     AZURE_OPENAI: 'api_key_azure_openai',
     ZAI: 'api_key_zai',
     SILICONFLOW: 'api_key_siliconflow',
 +    ELEVENLABS: 'api_key_elevenlabs',
 };
 /**

src/endpoints/speech.js+261 -0 Viewed

 import { Buffer } from 'node:buffer';
 +import fs from 'node:fs';
 import express from 'express';
 import wavefile from 'wavefile';
 import fetch from 'node-fetch';
 +import FormData from 'form-data';
 +import mime from 'mime-types';
 import { getPipeline } from '../transformers.js';
 import { forwardFetchResponse } from '../util.js';
 +import { readSecret, SECRET_KEYS } from './secrets.js';
 export const router = express.Router();

 });
 router.use('/pollinations', pollinations);
++
 +const elevenlabs = express.Router();
++
 +elevenlabs.post('/voices', async (req, res) => {
 +    try {
 +        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);
 +        if (!apiKey) {
 +            console.warn('ElevenLabs API key not found');
 +            return res.sendStatus(400);
 +        }
++
 +        const response = await fetch('https://api.elevenlabs.io/v1/voices', {
 +            headers: {
 +                'xi-api-key': apiKey,
 +            },
 +        });
++
 +        if (!response.ok) {
 +            const text = await response.text();
 +            console.warn(`ElevenLabs voices fetch failed: HTTP ${response.status} - ${text}`);
 +            return res.sendStatus(500);
 +        }
++
 +        const responseJson = await response.json();
 +        return res.json(responseJson);
 +    } catch (error) {
 +        console.error(error);
 +        return res.sendStatus(500);
 +    }
 +});
++
 +elevenlabs.post('/voice-settings', async (req, res) => {
 +    try {
 +        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);
 +        if (!apiKey) {
 +            console.warn('ElevenLabs API key not found');
 +            return res.sendStatus(400);
 +        }
++
 +        const response = await fetch('https://api.elevenlabs.io/v1/voices/settings/default', {
 +            headers: {
 +                'xi-api-key': apiKey,
 +            },
 +        });
++
 +        if (!response.ok) {
 +            const text = await response.text();
 +            console.warn(`ElevenLabs voice settings fetch failed: HTTP ${response.status} - ${text}`);
 +            return res.sendStatus(500);
 +        }
 +        const responseJson = await response.json();
 +        return res.json(responseJson);
 +    } catch (error) {
 +        console.error(error);
 +        return res.sendStatus(500);
 +    }
 +});
++
 +elevenlabs.post('/synthesize', async (req, res) => {
 +    try {
 +        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);
 +        if (!apiKey) {
 +            console.warn('ElevenLabs API key not found');
 +            return res.sendStatus(400);
 +        }
++
 +        const { voiceId, request } = req.body;
++
 +        if (!voiceId || !request) {
 +            console.warn('ElevenLabs synthesis request missing voiceId or request body');
 +            return res.sendStatus(400);
 +        }
++
 +        console.debug('ElevenLabs TTS request:', request);
++
 +        const response = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`, {
 +            method: 'POST',
 +            headers: {
 +                'xi-api-key': apiKey,
 +                'Content-Type': 'application/json',
 +            },
 +            body: JSON.stringify(request),
 +        });
++
 +        if (!response.ok) {
 +            const text = await response.text();
 +            console.warn(`ElevenLabs synthesis failed: HTTP ${response.status} - ${text}`);
 +            return res.sendStatus(500);
 +        }
++
 +        res.set('Content-Type', 'audio/mpeg');
 +        forwardFetchResponse(response, res);
 +    } catch (error) {
 +        console.error(error);
 +        return res.sendStatus(500);
 +    }
 +});
++
 +elevenlabs.post('/history', async (req, res) => {
 +    try {
 +        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);
 +        if (!apiKey) {
 +            console.warn('ElevenLabs API key not found');
 +            return res.sendStatus(400);
 +        }
++
 +        const response = await fetch('https://api.elevenlabs.io/v1/history', {
 +            headers: {
 +                'xi-api-key': apiKey,
 +            },
 +        });
++
 +        if (!response.ok) {
 +            const text = await response.text();
 +            console.warn(`ElevenLabs history fetch failed: HTTP ${response.status} - ${text}`);
 +            return res.sendStatus(500);
 +        }
++
 +        const responseJson = await response.json();
 +        return res.json(responseJson);
 +    } catch (error) {
 +        console.error(error);
 +        return res.sendStatus(500);
 +    }
 +});
++
 +elevenlabs.post('/history-audio', async (req, res) => {
 +    try {
 +        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);
 +        if (!apiKey) {
 +            console.warn('ElevenLabs API key not found');
 +            return res.sendStatus(400);
 +        }
++
 +        const { historyItemId } = req.body;
 +        if (!historyItemId) {
 +            console.warn('ElevenLabs history audio request missing historyItemId');
 +            return res.sendStatus(400);
 +        }
++
 +        console.debug('ElevenLabs history audio request for ID:', historyItemId);
++
 +        const response = await fetch(`https://api.elevenlabs.io/v1/history/${historyItemId}/audio`, {
 +            headers: {
 +                'xi-api-key': apiKey,
 +            },
 +        });
++
 +        if (!response.ok) {
 +            const text = await response.text();
 +            console.warn(`ElevenLabs history audio fetch failed: HTTP ${response.status} - ${text}`);
 +            return res.sendStatus(500);
 +        }
++
 +        res.set('Content-Type', 'audio/mpeg');
 +        forwardFetchResponse(response, res);
 +    } catch (error) {
 +        console.error(error);
 +        return res.sendStatus(500);
 +    }
 +});
++
 +elevenlabs.post('/voices/add', async (req, res) => {
 +    try {
 +        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);
 +        if (!apiKey) {
 +            console.warn('ElevenLabs API key not found');
 +            return res.sendStatus(400);
 +        }
++
 +        const { name, description, labels, files } = req.body;
++
 +        const formData = new FormData();
 +        formData.append('name', name || 'Custom Voice');
 +        formData.append('description', description || 'Uploaded via SillyTavern');
 +        formData.append('labels', labels || '');
++
 +        for (const fileData of (files || [])) {
 +            const [mimeType, base64Data] = /^data:(.+);base64,(.+)$/.exec(fileData)?.slice(1) || [];
 +            if (!mimeType || !base64Data) {
 +                console.warn('Invalid audio file data provided for ElevenLabs voice upload');
 +                continue;
 +            }
 +            const buffer = Buffer.from(base64Data, 'base64');
 +            formData.append('files', buffer, {
 +                filename: `audio.${mime.extension(mimeType) || 'wav'}`,
 +                contentType: mimeType,
 +            });
 +        }
++
 +        console.debug('ElevenLabs voice upload request:', { name, description, labels, files: files?.length || 0 });
++
 +        const response = await fetch('https://api.elevenlabs.io/v1/voices/add', {
 +            method: 'POST',
 +            headers: {
 +                'xi-api-key': apiKey,
 +            },
 +            body: formData,
 +        });
++
 +        if (!response.ok) {
 +            const text = await response.text();
 +            console.warn(`ElevenLabs voice upload failed: HTTP ${response.status} - ${text}`);
 +            return res.sendStatus(500);
 +        }
++
 +        const responseJson = await response.json();
 +        return res.json(responseJson);
 +    } catch (error) {
 +        console.error(error);
 +        return res.sendStatus(500);
 +    }
 +});
++
 +elevenlabs.post('/recognize', async (req, res) => {
 +    try {
 +        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);
 +        if (!apiKey) {
 +            console.warn('ElevenLabs API key not found');
 +            return res.sendStatus(400);
 +        }
++
 +        if (!req.file) {
 +            console.warn('No audio file found');
 +            return res.sendStatus(400);
 +        }
++
 +        console.info('Processing audio file with ElevenLabs', req.file.path);
 +        const formData = new FormData();
 +        formData.append('file', fs.createReadStream(req.file.path), { filename: 'audio.wav', contentType: 'audio/wav' });
 +        formData.append('model_id', req.body.model);
++
 +        const response = await fetch('https://api.elevenlabs.io/v1/speech-to-text', {
 +            method: 'POST',
 +            headers: {
 +                'xi-api-key': apiKey,
 +            },
 +            body: formData,
 +        });
++
 +        if (!response.ok) {
 +            const text = await response.text();
 +            console.warn(`ElevenLabs speech recognition failed: HTTP ${response.status} - ${text}`);
 +            return res.sendStatus(500);
 +        }
++
 +        fs.unlinkSync(req.file.path);
 +        const responseJson = await response.json();
 +        console.debug('ElevenLabs speech recognition response:', responseJson);
 +        return res.json(responseJson);
 +    } catch (error) {
 +        console.error(error);
 +        return res.sendStatus(500);
 +    }
 +});
++
 +router.use('/elevenlabs', elevenlabs);