Add gpt-4o-mini-tts model with character voice instructions #3879 (#4352) * Add gpt-4o-mini-tts model with character voice instructions * Restyle instructions block * Add debug log for tts request --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>

894f3e986ebc1b52d76cf2e69041f5f19e148535

Ni-co-la-s <96898279+Ni-co-la-s@users.noreply.github.com>

Signed
Parents: c8612e2ab6326ce528578db4f63b8119a701a896
5 files changed, +146 -21Ignore whitespace
public/scripts/extensions/tts/css/openai-tts.css+11 -0
@@ -0,0 +1,11 @@
1+#openai-character-instructions {
2+ display: flex;
3+ flex-direction: column;
4+ gap: 10px;
5+}
6+
7+#openai-character-instructions .character-instructions {
8+ display: flex;
9+ flex-direction: column;
10+ gap: 5px;
11+}
public/scripts/extensions/tts/index.js+2 -2
@@ -476,7 +476,7 @@ async function tts(text, voiceId, char) {
476476 await addAudioJob(response, char);
477477 }
478478
479479 let response = await ttsProvider.generateTts(text, voiceId, char);
480480
481481 // If async generator, process every chunk as it comes in
482482 if (typeof response[Symbol.asyncIterator] === 'function') {
@@ -1007,7 +1007,7 @@ function getCharacters(unrestricted) {
10071007 return characters.filter(onlyUnique);
10081008}
10091009
10101010export function sanitizeId(input) {
10111011 // Remove any non-alphanumeric characters except underscore (_) and hyphen (-)
10121012 let sanitized = encodeURIComponent(input).replace(/[^a-zA-Z0-9-_]/g, '');
10131013
public/scripts/extensions/tts/openai.js+115 -10
@@ -1,15 +1,18 @@
11import { getRequestHeaders } from '../../../script.js';
22import { saveTtsProviderSettings, sanitizeId } from './index.js';
33
44export { OpenAITtsProvider };
55
66class OpenAITtsProvider {
77 static voices = [
88 { name: 'Alloy', voice_id: 'alloy', lang: 'en-US', preview_url: 'https://cdn.openai.com/API/docs/audio/alloy.wav' },
9+ { name: 'Ash', voice_id: 'ash', lang: 'en-US', preview_url: 'https://cdn.openai.com/API/docs/audio/ash.wav' },
10+ { name: 'Coral', voice_id: 'coral', lang: 'en-US', preview_url: 'https://cdn.openai.com/API/docs/audio/coral.wav' },
911 { name: 'Echo', voice_id: 'echo', lang: 'en-US', preview_url: 'https://cdn.openai.com/API/docs/audio/echo.wav' },
1012 { name: 'Fable', voice_id: 'fable', lang: 'en-US', preview_url: 'https://cdn.openai.com/API/docs/audio/fable.wav' },
1113 { name: 'Onyx', voice_id: 'onyx', lang: 'en-US', preview_url: 'https://cdn.openai.com/API/docs/audio/onyx.wav' },
1214 { name: 'Nova', voice_id: 'nova', lang: 'en-US', preview_url: 'https://cdn.openai.com/API/docs/audio/nova.wav' },
15+ { name: 'Sage', voice_id: 'sage', lang: 'en-US', preview_url: 'https://cdn.openai.com/API/docs/audio/sage.wav' },
1316 { name: 'Shimmer', voice_id: 'shimmer', lang: 'en-US', preview_url: 'https://cdn.openai.com/API/docs/audio/shimmer.wav' },
1417 ];
1518
@@ -23,6 +26,7 @@ class OpenAITtsProvider {
2326 customVoices: [],
2427 model: 'tts-1',
2528 speed: 1,
29+ characterInstructions: {},
2630 };
2731
2832 get settingsHtml() {
@@ -35,6 +39,7 @@ class OpenAITtsProvider {
3539 <optgroup label="Latest">
3640 <option value="tts-1">tts-1</option>
3741 <option value="tts-1-hd">tts-1-hd</option>
42+ <option value="gpt-4o-mini-tts">gpt-4o-mini-tts</option>
3843 </optgroup>
3944 <optgroup label="Snapshots">
4045 <option value="tts-1-1106">tts-1-1106</option>
@@ -79,14 +84,104 @@ class OpenAITtsProvider {
7984 $('#openai-tts-speed-output').text(this.settings.speed);
8085
8186 await this.checkReady();
87+ // Initialize UI state based on current model (gpt-4o-mini-tts or other)
88+ this.updateInstructionsUI();
89+ // Look for voice map changes
90+ this.setupVoiceMapObserver();
91+
8292 console.debug('OpenAI TTS: Settings loaded');
8393 }
8494
95+ setupVoiceMapObserver() {
96+ if (this.voiceMapObserver) {
97+ this.voiceMapObserver.disconnect();
98+ this.voiceMapObserver = null;
99+ }
100+
101+ const targetNode = document.getElementById('tts_voicemap_block');
102+ if (!targetNode) return;
103+
104+ const observer = new MutationObserver(() => {
105+ if (this.settings.model === 'gpt-4o-mini-tts') {
106+ this.populateCharacterInstructions();
107+ }
108+ });
109+
110+ observer.observe(targetNode, { childList: true, subtree: true });
111+ this.voiceMapObserver = observer;
112+ }
113+
85114 onSettingsChange() {
86115 // Update dynamically
87116 this.settings.model = String($('#openai-tts-model').find(':selected').val());
88117 this.settings.speed = Number($('#openai-tts-speed').val());
89118 $('#openai-tts-speed-output').text(this.settings.speed);
119+ this.updateInstructionsUI();
120+ saveTtsProviderSettings();
121+ }
122+
123+ updateInstructionsUI() {
124+ if (this.settings.model === 'gpt-4o-mini-tts') {
125+ this.createInstructionsContainer();
126+ $('#openai-instructions-container').show();
127+ this.populateCharacterInstructions();
128+ } else {
129+ $('#openai-instructions-container').hide();
130+ this.voiceMapObserver?.disconnect();
131+ this.voiceMapObserver = null;
132+ }
133+ }
134+
135+ createInstructionsContainer() {
136+ if ($('#openai-instructions-container').length === 0) {
137+ const containerHtml = `
138+ <div id="openai-instructions-container" style="display: none;">
139+ <span>Voice Instructions (GPT-4o Mini TTS)</span><br>
140+ <small>Customize how each character speaks</small>
141+ <div id="openai-character-instructions"></div>
142+ </div>
143+ `;
144+ $('#openai-tts-speed').parent().after(containerHtml);
145+ }
146+ }
147+
148+ populateCharacterInstructions() {
149+
150+ const currentCharacters = $('.tts_voicemap_block_char span').map((i, el) => $(el).text()).get();
151+
152+ $('#openai-character-instructions').empty();
153+
154+ for (const char of currentCharacters) {
155+ if (char === 'SillyTavern System' || char === '[Default Voice]') continue;
156+
157+ const sanitizedName = sanitizeId(char);
158+ const savedInstructions = this.settings.characterInstructions?.[char] || '';
159+
160+ const instructionBlock = document.createElement('div');
161+ const label = document.createElement('label');
162+ const textArea = document.createElement('textarea');
163+ instructionBlock.appendChild(label);
164+ instructionBlock.appendChild(textArea);
165+ instructionBlock.className = 'character-instructions';
166+ label.setAttribute('for', `openai_char_${sanitizedName}`);
167+ label.innerText = `${char}:`;
168+ textArea.id = `openai_char_${sanitizedName}`;
169+ textArea.placeholder = 'e.g., "Speak cheerfully and energetically"';
170+ textArea.className = 'textarea_compact autoSetHeight';
171+ textArea.value = savedInstructions;
172+ textArea.addEventListener('input', () => {
173+ this.saveCharacterInstructions(char, textArea.value);
174+ });
175+
176+ $('#openai-character-instructions').append(instructionBlock);
177+ }
178+ }
179+
180+ saveCharacterInstructions(characterName, instructions) {
181+ if (!this.settings.characterInstructions) {
182+ this.settings.characterInstructions = {};
183+ }
184+ this.settings.characterInstructions[characterName] = instructions;
90185 saveTtsProviderSettings();
91186 }
92187
@@ -112,8 +207,8 @@ class OpenAITtsProvider {
112207 return voice;
113208 }
114209
115210 async generateTts(text, voiceId, characterName = null) {
116211 const response = await this.fetchTtsGeneration(text, voiceId, characterName);
117212 return response;
118213 }
119214
@@ -125,17 +220,27 @@ class OpenAITtsProvider {
125220 return;
126221 }
127222
128223 async fetchTtsGeneration(inputText, voiceId, characterName = null) {
129224 console.info(`Generating new TTS for voice_id ${voiceId}`);
225+
226+ const requestBody = {
227+ 'text': inputText,
228+ 'voice': voiceId,
229+ 'model': this.settings.model,
230+ 'speed': this.settings.speed,
231+ };
232+
233+ if (this.settings.model === 'gpt-4o-mini-tts' && characterName) {
234+ const instructions = this.settings.characterInstructions?.[characterName];
235+ if (instructions && instructions.trim()) {
236+ requestBody.instructions = instructions;
237+ }
238+ }
239+
130240 const response = await fetch('/api/openai/generate-voice', {
131241 method: 'POST',
132242 headers: getRequestHeaders(),
133243 body: JSON.stringify({requestBody),
134- 'text': inputText,
135- 'voice': voiceId,
136- 'model': this.settings.model,
137- 'speed': this.settings.speed,
138- }),
139244 });
140245
141246 if (!response.ok) {
public/scripts/extensions/tts/style.css+3 -2
@@ -1,3 +1,6 @@
1+@import './css/minimax-tts.css';
2+@import './css/openai-tts.css';
3+
14.voice_preview {
25 margin: 0.25rem 0.5rem;
36 display: flex;
@@ -125,5 +128,3 @@
125128#at-status_info {
126129 color: lightgreen;
127130}
128-
129-@import './css/minimax-tts.css';
src/endpoints/openai.js+15 -7
@@ -271,19 +271,27 @@ router.post('/generate-voice', async (request, response) => {
271271 return response.sendStatus(400);
272272 }
273273
274+ const requestBody = {
275+ input: request.body.text,
276+ response_format: 'mp3',
277+ voice: request.body.voice ?? 'alloy',
278+ speed: request.body.speed ?? 1,
279+ model: request.body.model ?? 'tts-1',
280+ };
281+
282+ if (request.body.instructions) {
283+ requestBody.instructions = request.body.instructions;
284+ }
285+
286+ console.debug('OpenAI TTS request', requestBody);
287+
274288 const result = await fetch('https://api.openai.com/v1/audio/speech', {
275289 method: 'POST',
276290 headers: {
277291 'Content-Type': 'application/json',
278292 Authorization: `Bearer ${key}`,
279293 },
280294 body: JSON.stringify({requestBody),
281- input: request.body.text,
282- response_format: 'mp3',
283- voice: request.body.voice ?? 'alloy',
284- speed: request.body.speed ?? 1,
285- model: request.body.model ?? 'tts-1',
286- }),
287295 });
288296
289297 if (!result.ok) {