1
import { saveTtsProviderSettings } from './index.js';
2
import { event_types, eventSource, getRequestHeaders } from '/script.js';
3
import { SECRET_KEYS, secret_state, writeSecret } from '/scripts/secrets.js';
4
import { getBase64Async } from '/scripts/utils.js';
5
export { ElevenLabsTtsProvider };
6

7
class ElevenLabsTtsProvider {
8
    settings;
9
    voices = [];
10
    separator = ' ... ... ... ';
11

12
    defaultSettings = {
13
        stability: 0.75,
14
        similarity_boost: 0.75,
15
        style_exaggeration: 0.00,
16
        speaker_boost: true,
17
        speed: 1.0,
18
        model: 'eleven_turbo_v2_5',
19
        voiceMap: {},
20
    };
21

22
    get settingsHtml() {
23
        let html = `
24
        <div class="elevenlabs_tts_settings">
25
            <div class="flex-container alignItemsBaseline">
26
                <h4 for="elevenlabs_tts_key" class="flex1 margin0">
27
                    <a href="https://elevenlabs.io/app/developers/api-keys" target="_blank">ElevenLabs TTS Key</a>
28
                </h4>
29
                <div id="elevenlabs_tts_key" class="menu_button menu_button_icon manage-api-keys" data-key="api_key_elevenlabs">
30
                    <i class="fa-solid fa-key"></i>
31
                    <span>Click to set</span>
32
                </div>
33
            </div>
34
            <label for="elevenlabs_tts_model">Model</label>
35
            <select id="elevenlabs_tts_model" class="text_pole">
36
                <option value="eleven_v3">Eleven v3</option>
37
                <option value="eleven_ttv_v3">Eleven ttv v3</option>
38
                <option value="eleven_multilingual_v2">Multilingual v2</option>
39
                <option value="eleven_flash_v2_5">Eleven Flash v2.5</option>
40
                <option value="eleven_turbo_v2_5">Turbo v2.5</option>
41
                <option value="eleven_multilingual_ttv_v2">Multilingual ttv v2</option>
42
                <option value="eleven_monolingual_v1">English v1 (Old)</option>
43
                <option value="eleven_multilingual_v1">Multilingual v1 (Old)</option>
44
                <option value="eleven_turbo_v2">Turbo v2 (Old)</option>
45
            </select>
46
            <label for="elevenlabs_tts_stability">Stability: <span id="elevenlabs_tts_stability_output"></span></label>
47
            <input id="elevenlabs_tts_stability" type="range" value="${this.defaultSettings.stability}" min="0" max="1" step="0.01" />
48
            <label for="elevenlabs_tts_similarity_boost">Similarity Boost: <span id="elevenlabs_tts_similarity_boost_output"></span></label>
49
            <input id="elevenlabs_tts_similarity_boost" type="range" value="${this.defaultSettings.similarity_boost}" min="0" max="1" step="0.01" />
50
            <label for="elevenlabs_tts_speed">Speed: <span id="elevenlabs_tts_speed_output"></span></label>
51
            <input id="elevenlabs_tts_speed" type="range" value="${this.defaultSettings.speed}" min="0.7" max="1.2" step="0.01" />
52
            <div id="elevenlabs_tts_v2_options" style="display: none;">
53
                <label for="elevenlabs_tts_style_exaggeration">Style Exaggeration: <span id="elevenlabs_tts_style_exaggeration_output"></span></label>
54
                <input id="elevenlabs_tts_style_exaggeration" type="range" value="${this.defaultSettings.style_exaggeration}" min="0" max="1" step="0.01" />
55
                <label for="elevenlabs_tts_speaker_boost">Speaker Boost:</label>
56
                <input id="elevenlabs_tts_speaker_boost" style="display: inline-grid" type="checkbox" />
57
            </div>
58
            <hr>
59
            <div id="elevenlabs_tts_voice_cloning">
60
                <span>Instant Voice Cloning</span><br>
61
                <input id="elevenlabs_tts_voice_cloning_name" type="text" class="text_pole" placeholder="Voice Name"/>
62
                <input id="elevenlabs_tts_voice_cloning_description" type="text" class="text_pole" placeholder="Voice Description"/>
63
                <input id="elevenlabs_tts_voice_cloning_labels" type="text" class="text_pole" placeholder="Labels"/>
64
                <div class="menu_button menu_button_icon" id="upload_audio_file">
65
                    <i class="fa-solid fa-file-import"></i>
66
                    <span>Upload Audio Files</span>
67
                </div>
68
                <input id="elevenlabs_tts_audio_files" type="file" name="audio_files" accept="audio/*" style="display: none;" multiple>
69
                <div id="elevenlabs_tts_selected_files_list"></div>
70
                <input id="elevenlabs_tts_clone_voice_button" class="menu_button menu_button_icon" type="button" value="Clone Voice">
71
            </div>
72
            <hr>
73
        </div>
74
        `;
75
        return html;
76
    }
77

78
    constructor() {
79
        this.handler = async function (/** @type {string} */ key) {
80
            if (key !== SECRET_KEYS.ELEVENLABS) return;
81
            $('#elevenlabs_tts_key').toggleClass('success', !!secret_state[SECRET_KEYS.ELEVENLABS]);
82
            await this.fetchTtsVoiceObjects();
83
        }.bind(this);
84
    }
85

86
    dispose() {
87
        [event_types.SECRET_WRITTEN, event_types.SECRET_DELETED, event_types.SECRET_ROTATED].forEach(event => {
88
            eventSource.removeListener(event, this.handler);
89
        });
90
    }
91

92
    shouldInvolveExtendedSettings() {
93
        // Models that support extended settings (style_exaggeration, speaker_boost)
94
        const modelsWithExtendedSettings = [
95
            'eleven_v3',
96
            'eleven_ttv_v3',
97
            'eleven_multilingual_v2',
98
            'eleven_multilingual_ttv_v2',
99
        ];
100
        return modelsWithExtendedSettings.includes(this.settings.model);
101
    }
102

103
    onSettingsChange() {
104
        // Update dynamically
105
        this.settings.stability = $('#elevenlabs_tts_stability').val();
106
        this.settings.similarity_boost = $('#elevenlabs_tts_similarity_boost').val();
107
        this.settings.style_exaggeration = $('#elevenlabs_tts_style_exaggeration').val();
108
        this.settings.speaker_boost = $('#elevenlabs_tts_speaker_boost').is(':checked');
109
        this.settings.speed = $('#elevenlabs_tts_speed').val();
110
        this.settings.model = $('#elevenlabs_tts_model').find(':selected').val();
111
        $('#elevenlabs_tts_stability_output').text(Math.round(this.settings.stability * 100) + '%');
112
        $('#elevenlabs_tts_similarity_boost_output').text(Math.round(this.settings.similarity_boost * 100) + '%');
113
        $('#elevenlabs_tts_style_exaggeration_output').text(Math.round(this.settings.style_exaggeration * 100) + '%');
114
        $('#elevenlabs_tts_speed_output').text(this.settings.speed + 'x');
115
        $('#elevenlabs_tts_v2_options').toggle(this.shouldInvolveExtendedSettings());
116
        saveTtsProviderSettings();
117
    }
118

119
    async loadSettings(settings) {
120
        // Pupulate Provider UI given input settings
121
        if (Object.keys(settings).length == 0) {
122
            console.info('Using default TTS Provider settings');
123
        }
124

125
        // Only accept keys defined in defaultSettings
126
        this.settings = this.defaultSettings;
127

128
        // Migrate old settings
129
        if (settings.multilingual !== undefined) {
130
            settings.model = settings.multilingual ? 'eleven_multilingual_v1' : 'eleven_monolingual_v1';
131
            delete settings.multilingual;
132
        }
133

134
        if (Object.hasOwn(settings, 'apiKey')) {
135
            if (settings.apiKey && !secret_state[SECRET_KEYS.ELEVENLABS]) {
136
                await writeSecret(SECRET_KEYS.ELEVENLABS, settings.apiKey);
137
            }
138
            delete settings.apiKey;
139
        }
140

141
        $('#elevenlabs_tts_key').toggleClass('success', !!secret_state[SECRET_KEYS.ELEVENLABS]);
142
        [event_types.SECRET_WRITTEN, event_types.SECRET_DELETED, event_types.SECRET_ROTATED].forEach(event => {
143
            eventSource.on(event, this.handler);
144
        });
145

146
        for (const key in settings) {
147
            if (key in this.settings) {
148
                this.settings[key] = settings[key];
149
            } else {
150
                throw `Invalid setting passed to TTS Provider: ${key}`;
151
            }
152
        }
153

154
        $('#elevenlabs_tts_stability').val(this.settings.stability);
155
        $('#elevenlabs_tts_similarity_boost').val(this.settings.similarity_boost);
156
        $('#elevenlabs_tts_style_exaggeration').val(this.settings.style_exaggeration);
157
        $('#elevenlabs_tts_speaker_boost').prop('checked', this.settings.speaker_boost);
158
        $('#elevenlabs_tts_speed').val(this.settings.speed);
159
        $('#elevenlabs_tts_model').val(this.settings.model);
160
        $('#elevenlabs_tts_similarity_boost').on('input', this.onSettingsChange.bind(this));
161
        $('#elevenlabs_tts_stability').on('input', this.onSettingsChange.bind(this));
162
        $('#elevenlabs_tts_style_exaggeration').on('input', this.onSettingsChange.bind(this));
163
        $('#elevenlabs_tts_speaker_boost').on('change', this.onSettingsChange.bind(this));
164
        $('#elevenlabs_tts_speed').on('input', this.onSettingsChange.bind(this));
165
        $('#elevenlabs_tts_model').on('change', this.onSettingsChange.bind(this));
166
        $('#elevenlabs_tts_stability_output').text(Math.round(this.settings.stability * 100) + '%');
167
        $('#elevenlabs_tts_similarity_boost_output').text(Math.round(this.settings.similarity_boost * 100) + '%');
168
        $('#elevenlabs_tts_style_exaggeration_output').text(Math.round(this.settings.style_exaggeration * 100) + '%');
169
        $('#elevenlabs_tts_speed_output').text(this.settings.speed + 'x');
170
        $('#elevenlabs_tts_v2_options').toggle(this.shouldInvolveExtendedSettings());
171
        try {
172
            await this.checkReady();
173
            console.debug('ElevenLabs: Settings loaded');
174
        } catch {
175
            console.debug('ElevenLabs: Settings loaded, but not ready');
176
        }
177

178
        this.setupVoiceCloningMenu();
179
    }
180

181
    // Perform a simple readiness check by trying to fetch voiceIds
182
    async checkReady() {
183
        await this.fetchTtsVoiceObjects();
184
    }
185

186
    async onRefreshClick() {
187
        await this.fetchTtsVoiceObjects();
188
    }
189

190
    setupVoiceCloningMenu() {
191
        const audioFilesInput = /** @type {HTMLInputElement} */ (document.getElementById('elevenlabs_tts_audio_files'));
192
        const selectedFilesListElement = document.getElementById('elevenlabs_tts_selected_files_list');
193
        const cloneVoiceButton = document.getElementById('elevenlabs_tts_clone_voice_button');
194
        const uploadAudioFileButton = document.getElementById('upload_audio_file');
195
        const voiceCloningNameInput = /** @type {HTMLInputElement} */ (document.getElementById('elevenlabs_tts_voice_cloning_name'));
196
        const voiceCloningDescriptionInput = /** @type {HTMLInputElement} */ (document.getElementById('elevenlabs_tts_voice_cloning_description'));
197
        const voiceCloningLabelsInput = /** @type {HTMLInputElement} */ (document.getElementById('elevenlabs_tts_voice_cloning_labels'));
198

199
        const updateCloneVoiceButtonVisibility = () => {
200
            cloneVoiceButton.style.display = audioFilesInput.files.length > 0 ? 'inline-block' : 'none';
201
        };
202

203
        const clearSelectedFiles = () => {
204
            audioFilesInput.value = '';
205
            selectedFilesListElement.innerHTML = '';
206
            updateCloneVoiceButtonVisibility();
207
        };
208

209
        uploadAudioFileButton.addEventListener('click', () => {
210
            audioFilesInput.click();
211
        });
212

213
        audioFilesInput.addEventListener('change', () => {
214
            selectedFilesListElement.innerHTML = '';
215
            for (const file of audioFilesInput.files) {
216
                const listItem = document.createElement('div');
217
                listItem.textContent = file.name;
218
                selectedFilesListElement.appendChild(listItem);
219
            }
220
            updateCloneVoiceButtonVisibility();
221
        });
222

223
        cloneVoiceButton.addEventListener('click', async () => {
224
            const voiceName = voiceCloningNameInput.value.trim();
225
            const voiceDescription = voiceCloningDescriptionInput.value.trim();
226
            const voiceLabels = voiceCloningLabelsInput.value.trim();
227

228
            if (!voiceName) {
229
                toastr.error('Please provide a name for the cloned voice.');
230
                return;
231
            }
232

233
            try {
234
                await this.addVoice(voiceName, voiceDescription, voiceLabels);
235
                toastr.success('Voice cloned successfully. Hit reload to see the new voice in the voice listing.');
236
                clearSelectedFiles();
237
                voiceCloningNameInput.value = '';
238
                voiceCloningDescriptionInput.value = '';
239
                voiceCloningLabelsInput.value = '';
240
            } catch (error) {
241
                toastr.error(`Failed to clone voice: ${error.message}`);
242
            }
243
        });
244

245
        updateCloneVoiceButtonVisibility();
246
    }
247

248
    /**
249
     * Get voice object by name
250
     * @param {string} voiceName Voice name to look up
251
     * @returns {Promise<Object>} Voice object
252
     */
253
    async getVoice(voiceName) {
254
        if (this.voices.length == 0) {
255
            this.voices = await this.fetchTtsVoiceObjects();
256
        }
257
        const match = this.voices.filter(
258
            elevenVoice => elevenVoice.name == voiceName,
259
        )[0];
260
        if (!match) {
261
            throw `TTS Voice name ${voiceName} not found in ElevenLabs account`;
262
        }
263
        return match;
264
    }
265

266
    /**
267
     * Generate TTS audio
268
     * @param {string} text Text to synthesize
269
     * @param {string} voiceId Voice ID to use for synthesis
270
     * @returns {Promise<Response>} Response object containing audio data
271
     */
272
    async generateTts(text, voiceId) {
273
        const historyId = await this.findTtsGenerationInHistory(text, voiceId);
274

275
        if (historyId) {
276
            console.debug(`Found existing TTS generation with id ${historyId}`);
277
            return await this.fetchTtsFromHistory(historyId);
278
        } else {
279
            console.debug('No existing TTS generation found, requesting new generation');
280
            return await this.fetchTtsGeneration(text, voiceId);
281
        }
282
    }
283

284
    /**
285
     * Find existing TTS generation in history
286
     * @param {string} message Message text used for TTS generation
287
     * @param {string} voiceId Voice ID used for TTS generation
288
     * @returns {Promise<string>} History item ID if found, empty string otherwise
289
     */
290
    async findTtsGenerationInHistory(message, voiceId) {
291
        const ttsHistory = await this.fetchTtsHistory();
292
        for (const history of ttsHistory) {
293
            const text = history.text;
294
            const itemId = history.history_item_id;
295
            if (message === text && history.voice_id == voiceId) {
296
                console.info(`Existing TTS history item ${itemId} found: ${text} `);
297
                return itemId;
298
            }
299
        }
300
        return '';
301
    }
302

303
    async fetchTtsVoiceObjects() {
304
        const response = await fetch('/api/speech/elevenlabs/voices', {
305
            method: 'POST',
306
            headers: getRequestHeaders({ omitContentType: true }),
307
        });
308
        if (!response.ok) {
309
            throw new Error(`HTTP ${response.status}. See server console for details.`);
310
        }
311
        const responseJson = await response.json();
312
        return responseJson.voices;
313
    }
314

315
    async fetchTtsVoiceSettings() {
316
        const response = await fetch('/api/speech/elevenlabs/voice-settings', {
317
            method: 'POST',
318
            headers: getRequestHeaders({ omitContentType: true }),
319
        });
320
        if (!response.ok) {
321
            throw new Error(`HTTP ${response.status}. See server console for details.`);
322
        }
323
        return response.json();
324
    }
325

326
    /**
327
     * Fetch new TTS generation from ElevenLabs API
328
     * @param {string} text Text to synthesize
329
     * @param {string} voiceId Voice ID to use for synthesis
330
     * @returns {Promise<Response>} Response object containing audio data
331
     */
332
    async fetchTtsGeneration(text, voiceId) {
333
        let model = this.settings.model ?? 'eleven_monolingual_v1';
334
        console.info(`Generating new TTS for voice_id ${voiceId}, model ${model}`);
335
        const request = {
336
            model_id: model,
337
            text: text,
338
            voice_settings: {
339
                stability: Number(this.settings.stability),
340
                similarity_boost: Number(this.settings.similarity_boost),
341
                speed: Number(this.settings.speed),
342
            },
343
        };
344
        if (this.shouldInvolveExtendedSettings()) {
345
            request.voice_settings.style = Number(this.settings.style_exaggeration);
346
            request.voice_settings.use_speaker_boost = Boolean(this.settings.speaker_boost);
347
        }
348
        const response = await fetch('/api/speech/elevenlabs/synthesize', {
349
            method: 'POST',
350
            headers: getRequestHeaders(),
351
            body: JSON.stringify({
352
                voiceId: voiceId,
353
                request: request,
354
            }),
355
        });
356
        if (!response.ok) {
357
            toastr.error(response.statusText, 'TTS Generation Failed');
358
            throw new Error(`HTTP ${response.status}. See server console for details.`);
359
        }
360
        return response;
361
    }
362

363
    /**
364
     * Fetch existing TTS audio from history
365
     * @param {string} historyItemId History item ID to fetch audio for
366
     * @returns {Promise<Response>} Response object containing audio data
367
     */
368
    async fetchTtsFromHistory(historyItemId) {
369
        console.info(`Fetched existing TTS with history_item_id ${historyItemId}`);
370
        const response = await fetch('/api/speech/elevenlabs/history-audio', {
371
            method: 'POST',
372
            headers: getRequestHeaders(),
373
            body: JSON.stringify({
374
                historyItemId: historyItemId,
375
            }),
376
        });
377
        if (!response.ok) {
378
            throw new Error(`HTTP ${response.status}. See server console for details.`);
379
        }
380
        return response;
381
    }
382

383
    /**
384
     * Fetch TTS generation history
385
     * @returns {Promise<Array>} Array of TTS history items
386
     */
387
    async fetchTtsHistory() {
388
        const response = await fetch('/api/speech/elevenlabs/history', {
389
            method: 'POST',
390
            headers: getRequestHeaders({ omitContentType: true }),
391
        });
392
        if (!response.ok) {
393
            throw new Error(`HTTP ${response.status}. See server console for details.`);
394
        }
395
        const responseJson = await response.json();
396
        return responseJson.history;
397
    }
398

399
    /**
400
     * Add a new voice via ElevenLabs API
401
     * @param {string} name Voice name
402
     * @param {string} description Voice description
403
     * @param {string} labels Voice labels
404
     * @returns {Promise<Object>} Newly created voice object
405
     */
406
    async addVoice(name, description, labels) {
407
        const audioFilesInput = /** @type {HTMLInputElement} */ (document.getElementById('elevenlabs_tts_audio_files'));
408
        if (!(audioFilesInput instanceof HTMLInputElement) || audioFilesInput.files.length === 0) {
409
            throw new Error('No audio files selected for voice cloning.');
410
        }
411

412
        const data = {
413
            name: name,
414
            description: description,
415
            labels: labels,
416
            files: [],
417
        };
418

419
        for (const file of audioFilesInput.files) {
420
            const base64Data = await getBase64Async(file);
421
            data.files.push(base64Data);
422
        }
423

424
        const response = await fetch('/api/speech/elevenlabs/voices/add', {
425
            method: 'POST',
426
            headers: getRequestHeaders(),
427
            body: JSON.stringify(data),
428
        });
429

430
        if (!response.ok) {
431
            throw new Error(`HTTP ${response.status}. See server console for details.`);
432
        }
433

434
        return await response.json();
435
    }
436
}