Add gpt-4o-mini-tts model with character voice instructions #3879 (#4352) * Add gpt-4o-mini-tts model with character voice instructions * Restyle instructions block * Add debug log for tts request --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>

894f3e986ebc1b52d76cf2e69041f5f19e148535

Ni-co-la-s <96898279+Ni-co-la-s@users.noreply.github.com>

Signed

Parents: c8612e2ab6326ce528578db4f63b8119a701a896

public/scripts/extensions/tts/css/openai-tts.css+11 -0 Viewed

+#openai-character-instructions {
+    display: flex;
+    flex-direction: column;
+    gap: 10px;
+}
+#openai-character-instructions .character-instructions {
+    display: flex;
+    flex-direction: column;
+    gap: 5px;
+}

public/scripts/extensions/tts/index.js+2 -2 Viewed

         await addAudioJob(response, char);
+    }
-    let response = await ttsProvider.generateTts(text, voiceId);
+    let response = await ttsProvider.generateTts(text, voiceId, char);
     // If async generator, process every chunk as it comes in
     if (typeof response[Symbol.asyncIterator] === 'function') {

     return characters.filter(onlyUnique);
+}
-function sanitizeId(input) {
+export function sanitizeId(input) {
     // Remove any non-alphanumeric characters except underscore (_) and hyphen (-)
     let sanitized = encodeURIComponent(input).replace(/[^a-zA-Z0-9-_]/g, '');

public/scripts/extensions/tts/openai.js+115 -10 Viewed

 import { getRequestHeaders } from '../../../script.js';
-import { saveTtsProviderSettings } from './index.js';
+import { saveTtsProviderSettings, sanitizeId } from './index.js';
 export { OpenAITtsProvider };
 class OpenAITtsProvider {
     static voices = [
         { name: 'Alloy', voice_id: 'alloy', lang: 'en-US', preview_url: 'https://cdn.openai.com/API/docs/audio/alloy.wav' },
+        { name: 'Ash', voice_id: 'ash', lang: 'en-US', preview_url: 'https://cdn.openai.com/API/docs/audio/ash.wav' },
+        { name: 'Coral', voice_id: 'coral', lang: 'en-US', preview_url: 'https://cdn.openai.com/API/docs/audio/coral.wav' },
         { name: 'Echo', voice_id: 'echo', lang: 'en-US', preview_url: 'https://cdn.openai.com/API/docs/audio/echo.wav' },
         { name: 'Fable', voice_id: 'fable', lang: 'en-US', preview_url: 'https://cdn.openai.com/API/docs/audio/fable.wav' },
         { name: 'Onyx', voice_id: 'onyx', lang: 'en-US', preview_url: 'https://cdn.openai.com/API/docs/audio/onyx.wav' },
         { name: 'Nova', voice_id: 'nova', lang: 'en-US', preview_url: 'https://cdn.openai.com/API/docs/audio/nova.wav' },
+        { name: 'Sage', voice_id: 'sage', lang: 'en-US', preview_url: 'https://cdn.openai.com/API/docs/audio/sage.wav' },
         { name: 'Shimmer', voice_id: 'shimmer', lang: 'en-US', preview_url: 'https://cdn.openai.com/API/docs/audio/shimmer.wav' },
     ];

         customVoices: [],
         model: 'tts-1',
         speed: 1,
+        characterInstructions: {},
     };
     get settingsHtml() {

                 <optgroup label="Latest">
                     <option value="tts-1">tts-1</option>
                     <option value="tts-1-hd">tts-1-hd</option>
+                    <option value="gpt-4o-mini-tts">gpt-4o-mini-tts</option>
                 </optgroup>
                 <optgroup label="Snapshots">
                     <option value="tts-1-1106">tts-1-1106</option>

         $('#openai-tts-speed-output').text(this.settings.speed);
         await this.checkReady();
+        // Initialize UI state based on current model (gpt-4o-mini-tts or other)
+        this.updateInstructionsUI();
+        // Look for voice map changes
+        this.setupVoiceMapObserver();
         console.debug('OpenAI TTS: Settings loaded');
+    }
+    setupVoiceMapObserver() {
+        if (this.voiceMapObserver) {
+            this.voiceMapObserver.disconnect();
+            this.voiceMapObserver = null;
+        }
+        const targetNode = document.getElementById('tts_voicemap_block');
+        if (!targetNode) return;
+        const observer = new MutationObserver(() => {
+            if (this.settings.model === 'gpt-4o-mini-tts') {
+                this.populateCharacterInstructions();
+            }
+        });
+        observer.observe(targetNode, { childList: true, subtree: true });
+        this.voiceMapObserver = observer;
+    }
     onSettingsChange() {
         // Update dynamically
         this.settings.model = String($('#openai-tts-model').find(':selected').val());
         this.settings.speed = Number($('#openai-tts-speed').val());
         $('#openai-tts-speed-output').text(this.settings.speed);
+        this.updateInstructionsUI();
+        saveTtsProviderSettings();
+    }
+    updateInstructionsUI() {
+        if (this.settings.model === 'gpt-4o-mini-tts') {
+            this.createInstructionsContainer();
+            $('#openai-instructions-container').show();
+            this.populateCharacterInstructions();
+        } else {
+            $('#openai-instructions-container').hide();
+            this.voiceMapObserver?.disconnect();
+            this.voiceMapObserver = null;
+        }
+    }
+    createInstructionsContainer() {
+        if ($('#openai-instructions-container').length === 0) {
+            const containerHtml = `
+                <div id="openai-instructions-container" style="display: none;">
+                    <span>Voice Instructions (GPT-4o Mini TTS)</span><br>
+                    <small>Customize how each character speaks</small>
+                    <div id="openai-character-instructions"></div>
+                </div>
+            `;
+            $('#openai-tts-speed').parent().after(containerHtml);
+        }
+    }
+    populateCharacterInstructions() {
+        const currentCharacters = $('.tts_voicemap_block_char span').map((i, el) => $(el).text()).get();
+        $('#openai-character-instructions').empty();
+        for (const char of currentCharacters) {
+            if (char === 'SillyTavern System' || char === '[Default Voice]') continue;
+            const sanitizedName = sanitizeId(char);
+            const savedInstructions = this.settings.characterInstructions?.[char] || '';
+            const instructionBlock = document.createElement('div');
+            const label = document.createElement('label');
+            const textArea = document.createElement('textarea');
+            instructionBlock.appendChild(label);
+            instructionBlock.appendChild(textArea);
+            instructionBlock.className = 'character-instructions';
+            label.setAttribute('for', `openai_char_${sanitizedName}`);
+            label.innerText = `${char}:`;
+            textArea.id = `openai_char_${sanitizedName}`;
+            textArea.placeholder = 'e.g., "Speak cheerfully and energetically"';
+            textArea.className = 'textarea_compact autoSetHeight';
+            textArea.value = savedInstructions;
+            textArea.addEventListener('input', () => {
+                this.saveCharacterInstructions(char, textArea.value);
+            });
+            $('#openai-character-instructions').append(instructionBlock);
+        }
+    }
+    saveCharacterInstructions(characterName, instructions) {
+        if (!this.settings.characterInstructions) {
+            this.settings.characterInstructions = {};
+        }
+        this.settings.characterInstructions[characterName] = instructions;
         saveTtsProviderSettings();
+    }

@@ -112,8 +207,8 @@ class OpenAITtsProvider {
112	return voice;	207	return voice;
113	}	208	}
114		209
115	async generateTts(text, voiceId) {	210	async generateTts(text, voiceId, characterName = null) {
116	const response = await this.fetchTtsGeneration(text, voiceId);	211	const response = await this.fetchTtsGeneration(text, voiceId, characterName);
117	return response;	212	return response;
118	}	213	}
119		214

         return;
+    }
-    async fetchTtsGeneration(inputText, voiceId) {
+    async fetchTtsGeneration(inputText, voiceId, characterName = null) {
         console.info(`Generating new TTS for voice_id ${voiceId}`);
+        const requestBody = {
+            'text': inputText,
+            'voice': voiceId,
+            'model': this.settings.model,
+            'speed': this.settings.speed,
+        };
+        if (this.settings.model === 'gpt-4o-mini-tts' && characterName) {
+            const instructions = this.settings.characterInstructions?.[characterName];
+            if (instructions && instructions.trim()) {
+                requestBody.instructions = instructions;
+            }
+        }
         const response = await fetch('/api/openai/generate-voice', {
             method: 'POST',
             headers: getRequestHeaders(),
-            body: JSON.stringify({
+            body: JSON.stringify(requestBody),
-                'text': inputText,
-                'voice': voiceId,
-                'model': this.settings.model,
-                'speed': this.settings.speed,
-            }),
         });
         if (!response.ok) {

public/scripts/extensions/tts/style.css+3 -2 Viewed

+@import './css/minimax-tts.css';
+@import './css/openai-tts.css';
 .voice_preview {
     margin: 0.25rem 0.5rem;
     display: flex;

@@ -125,5 +128,3 @@
125	#at-status_info {	128	#at-status_info {
126	color: lightgreen;	129	color: lightgreen;
127	}	130	}
128
129	@import './css/minimax-tts.css';

src/endpoints/openai.js+15 -7 Viewed

             return response.sendStatus(400);
+        }
+        const requestBody = {
+            input: request.body.text,
+            response_format: 'mp3',
+            voice: request.body.voice ?? 'alloy',
+            speed: request.body.speed ?? 1,
+            model: request.body.model ?? 'tts-1',
+        };
+        if (request.body.instructions) {
+            requestBody.instructions = request.body.instructions;
+        }
+        console.debug('OpenAI TTS request', requestBody);
         const result = await fetch('https://api.openai.com/v1/audio/speech', {
             method: 'POST',
             headers: {
                 'Content-Type': 'application/json',
                 Authorization: `Bearer ${key}`,
             },
-            body: JSON.stringify({
+            body: JSON.stringify(requestBody),
-                input: request.body.text,
-                response_format: 'mp3',
-                voice: request.body.voice ?? 'alloy',
-                speed: request.body.speed ?? 1,
-                model: request.body.model ?? 'tts-1',
-            }),
         });
         if (!result.ok) {