Add multiple voice per character in TTS extension (#4367) * Add multiple voice per character in TTS extension * Reformat settings template --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>

e71cfee4b85e382df13381e31a75ea15be20b292

Ni-co-la-s <96898279+Ni-co-la-s@users.noreply.github.com>

Signed

Parents: a88afde22ed63c1d4ad02e837845cbdf01bd4ae9

public/scripts/extensions/tts/index.js+170 -12 Viewed

     currentTtsJob = null;
+}
-async function tts(text, voiceId, char) {
+async function tts(text, voiceId, char, voiceMapKey = null) {
     async function processResponse(response) {
         // RVC injection
         if (typeof window['rvcVoiceConversion'] === 'function' && extension_settings.rvc.enabled)

         await addAudioJob(response, char);
+    }
-    let response = await ttsProvider.generateTts(text, voiceId, char);
+    // voiceMapKey can also include segment qualifiers, e.g. '{char} ("Quotes")'
+    let response = await ttsProvider.generateTts(text, voiceId, voiceMapKey);
     // If async generator, process every chunk as it comes in
     if (typeof response[Symbol.asyncIterator] === 'function') {

     completeTtsJob();
+}
+function parseMessageSegments(text) {
+    if (!extension_settings.tts.multi_voice_enabled) {
+        return [{ type: 'other', text: text }];
+    }
+    const segments = [];
+    const segmentRegex = /(\*[^*]*?\*)|(".*?")|(\u201C.*?\u201D)|(\u00AB.*?\u00BB)|(\u300C.*?\u300D)|(\u300E.*?\u300F)|(\uFF02.*?\uFF02)/gim;
+    let lastIndex = 0;
+    let match;
+    segmentRegex.lastIndex = 0;
+    while ((match = segmentRegex.exec(text)) !== null) {
+        // Add other text before this match
+        if (match.index > lastIndex) {
+            const otherText = text.substring(lastIndex, match.index).trim();
+            if (otherText && otherText.length > 0) {
+                segments.push({ type: 'other', text: otherText });
+            }
+        }
+        const matchedText = match[0];
+        let segmentType = 'other';
+        let content = '';
+        if (match[1]) {
+            // Asterisk content (*action*)
+            segmentType = 'action';
+            content = matchedText.slice(1, -1);
+        } else if (match[2] || match[3] || match[4] || match[5] || match[6] || match[7]) {
+            // Various quote types ("dialogue")
+            segmentType = 'dialogue';
+            content = matchedText.slice(1, -1);
+        }
+        // Trim and check for actual content
+        content = content.trim();
+        if (content.length > 0) {
+            segments.push({
+                type: segmentType,
+                text: content,
+            });
+        }
+        lastIndex = match.index + matchedText.length;
+    }
+    // Add remaining other text after last match
+    if (lastIndex < text.length) {
+        const otherText = text.substring(lastIndex).trim();
+        if (otherText.length > 0) {
+            segments.push({ type: 'other', text: otherText });
+        }
+    }
+    // If no segments found and not empty, treat whole text as other text
+    if (segments.length === 0 && text.trim().length > 0) {
+        segments.push({ type: 'other', text: text.trim() });
+    }
+    return segments;
+}
 async function processTtsQueue() {
     // Called each moduleWorker iteration to pull chat messages from queue
     if (currentTtsJob || ttsJobQueue.length <= 0 || audioPaused) {

     console.debug('New message found, running TTS');
     currentTtsJob = ttsJobQueue.shift();
+    // Handle segmented jobs that already have processed text
+    if (currentTtsJob.segmentType && currentTtsJob.segmentText) {
+        const char = currentTtsJob.name;
+        const segmentText = currentTtsJob.segmentText;
+        const segmentType = currentTtsJob.segmentType;
+        console.log(`TTS (${segmentType}): ${segmentText}`);
+        try {
+            let voiceMapKey = char;
+            // If multi-voice is enabled, modify the voice map key based on segment type
+            if (extension_settings.tts.multi_voice_enabled && char !== DEFAULT_VOICE_MARKER) {
+                switch (segmentType) {
+                    case 'dialogue':
+                        voiceMapKey = `${char} ("Quotes")`;
+                        break;
+                    case 'action':
+                        voiceMapKey = `${char} (*Text inside asterisks*)`;
+                        break;
+                    case 'other':
+                    default:
+                        voiceMapKey = `${char} (Other text)`;
+                        break;
+                }
+            }
+            const voiceMapEntry = voiceMap[voiceMapKey] === DEFAULT_VOICE_MARKER ? voiceMap[DEFAULT_VOICE_MARKER] : voiceMap[voiceMapKey];
+            if (!voiceMapEntry || voiceMapEntry === DISABLED_VOICE_MARKER) {
+                throw `${char} not in voicemap. Configure character in extension settings voice map`;
+            }
+            const voice = await ttsProvider.getVoice(voiceMapEntry);
+            const voiceId = voice.voice_id;
+            if (voiceId == null) {
+                toastr.error(`Specified voice for ${char} was not found. Check the TTS extension settings.`);
+                throw `Unable to attain voiceId for ${char}`;
+            }
+            // Pass the full voiceMapKey (e.g., "User ("Quotes")") as well with character name
+            await tts(segmentText, voiceId, char, voiceMapKey);
+        } catch (error) {
+            toastr.error(error.toString());
+            console.error(error);
+            currentTtsJob = null;
+        }
+        return;
+    }
+    // Process unsegmented job (first time processing)
     let text = extension_settings.tts.narrate_translated_only ? (currentTtsJob?.extra?.display_text || currentTtsJob.mes) : currentTtsJob.mes;
     // Substitute macros

             return;
+        }
-        const voiceMapEntry = voiceMap[char] === DEFAULT_VOICE_MARKER ? voiceMap[DEFAULT_VOICE_MARKER] : voiceMap[char];
+        // Parse message into segments if multi-voice is enabled
+        const segments = parseMessageSegments(text);
-        if (!voiceMapEntry || voiceMapEntry === DISABLED_VOICE_MARKER) {
+        if (segments.length === 0) {
-            throw `${char} not in voicemap. Configure character in extension settings voice map`;
+            console.warn('No valid segments found in text.');
+            completeTtsJob();
+            return;
+        }
-        const voice = await ttsProvider.getVoice(voiceMapEntry);
-        const voiceId = voice.voice_id;
+        // Add all segments to the queue as separate jobs (in reverse order so they process in correct order)
-        if (voiceId == null) {
+        for (let i = segments.length - 1; i >= 0; i--) {
-            toastr.error(`Specified voice for ${char} was not found. Check the TTS extension settings.`);
+            const segmentJob = {
-            throw `Unable to attain voiceId for ${char}`;
+                name: char,
+                segmentType: segments[i].type,
+                segmentText: segments[i].text,
+                is_user: currentTtsJob.is_user,
+                mes: currentTtsJob.mes,
+                extra: currentTtsJob.extra,
+            };
+            ttsJobQueue.unshift(segmentJob);
+        }
-        await tts(text, voiceId, char);
+        // Clear current job so the segmented jobs can be processed
+        currentTtsJob = null;
     } catch (error) {
         toastr.error(error.toString());
         console.error(error);

     $('#tts_pass_asterisks').prop('checked', extension_settings.tts.pass_asterisks);
     $('#tts_skip_codeblocks').prop('checked', extension_settings.tts.skip_codeblocks);
     $('#tts_skip_tags').prop('checked', extension_settings.tts.skip_tags);
+    $('#tts_multi_voice_enabled').prop('checked', extension_settings.tts.multi_voice_enabled);
     $('#playback_rate').val(extension_settings.tts.playback_rate);
     $('#playback_rate_counter').val(Number(extension_settings.tts.playback_rate).toFixed(2));
     $('#playback_rate_block').toggle(extension_settings.tts.currentProvider !== 'System');

     auto_generation: true,
     narrate_user: false,
     playback_rate: 1,
+    multi_voice_enabled: false,
 };
 function setTtsStatus(status, success) {

     console.log('setting pass asterisks', extension_settings.tts.pass_asterisks);
+}
+function onMultiVoiceClick() {
+    extension_settings.tts.multi_voice_enabled = !!$('#tts_multi_voice_enabled').prop('checked');
+    saveSettingsDebounced();
+    // Reinitialize voice map to show/hide voices
+    initVoiceMap();
+}
 //##############//
 // TTS Provider //
 //##############//

+            }
+        }
+    }
-    return characters.filter(onlyUnique);
+    characters = characters.filter(onlyUnique);
+    // If multi-voice is enabled, expand characters to include segment types
+    if (extension_settings.tts.multi_voice_enabled) {
+        const expandedCharacters = [];
+        for (const char of characters) {
+            if (char === DEFAULT_VOICE_MARKER || char === 'SillyTavern System') {
+                expandedCharacters.push(char);
+            } else {
+                expandedCharacters.push(`${char} ("Quotes")`);
+                expandedCharacters.push(`${char} (*Text inside asterisks*)`);
+                expandedCharacters.push(`${char} (Other text)`);
+            }
+        }
+        return expandedCharacters;
+    }
+    return characters;
+}
 export function sanitizeId(input) {

         $('#tts_periodic_auto_generation').on('click', onPeriodicAutoGenerationClick);
         $('#tts_narrate_by_paragraphs').on('click', onNarrateByParagraphsClick);
         $('#tts_narrate_user').on('click', onNarrateUserClick);
+        $('#tts_multi_voice_enabled').on('click', onMultiVoiceClick);
         $('#playback_rate').on('input', function () {
             const value = $(this).val();

public/scripts/extensions/tts/settings.html+10 -2 Viewed

                     <small data-i18n="Skip tagged blocks">Skip &lt;tagged&gt; blocks</small>
                 </label>
                 <label class="checkbox_label" for="tts_pass_asterisks">
-                <input type="checkbox" id="tts_pass_asterisks">
+                    <input type="checkbox" id="tts_pass_asterisks">
-                <small data-i18n="Pass Asterisks to TTS Engine">Pass Asterisks to TTS Engine</small>
+                    <small data-i18n="Pass Asterisks to TTS Engine">Pass Asterisks to TTS Engine</small>
+                </label>
+                <label class="checkbox_label" for="tts_multi_voice_enabled"
+                data-i18n="[title]Works best when: Pass Asterisks to TTS Engine is enabled, and both Only narrate quotes and Ignore *text, even 'quotes', inside asterisks* are disabled."
+                title="Works best when: Pass Asterisks to TTS Engine is enabled, and both Only narrate quotes and Ignore *text, even 'quotes', inside asterisks* are disabled.">
+                    <input type="checkbox" id="tts_multi_voice_enabled">
+                    <small data-i18n="Different voices for quotes and text inside asterisks">
+                        Different voices for "quotes", *text inside asterisks* and other text
+                    </small>
                 </label>
             </div>
             <div id="playback_rate_block" class="range-block">