Add multiple voice per character in TTS extension (#4367) * Add multiple voice per character in TTS extension * Reformat settings template --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>

e71cfee4b85e382df13381e31a75ea15be20b292

Ni-co-la-s <96898279+Ni-co-la-s@users.noreply.github.com>

Signed

Parents: a88afde22ed63c1d4ad02e837845cbdf01bd4ae9

public/scripts/extensions/tts/index.js+170 -12 Viewed

     currentTtsJob = null;
+}
 async function tts(text, voiceId, char, voiceMapKey = null) {
     async function processResponse(response) {
         // RVC injection
         if (typeof window['rvcVoiceConversion'] === 'function' && extension_settings.rvc.enabled)

         await addAudioJob(response, char);
+    }
 -    let response = await ttsProvider.generateTts(text, voiceId, char);
 +    // voiceMapKey can also include segment qualifiers, e.g. '{char} ("Quotes")'
 +    let response = await ttsProvider.generateTts(text, voiceId, voiceMapKey);
     // If async generator, process every chunk as it comes in
     if (typeof response[Symbol.asyncIterator] === 'function') {

     completeTtsJob();
+}
 +function parseMessageSegments(text) {
 +    if (!extension_settings.tts.multi_voice_enabled) {
 +        return [{ type: 'other', text: text }];
 +    }
++
 +    const segments = [];
 +    const segmentRegex = /(\*[^*]*?\*)|(".*?")|(\u201C.*?\u201D)|(\u00AB.*?\u00BB)|(\u300C.*?\u300D)|(\u300E.*?\u300F)|(\uFF02.*?\uFF02)/gim;
 +    let lastIndex = 0;
 +    let match;
++
 +    segmentRegex.lastIndex = 0;
++
 +    while ((match = segmentRegex.exec(text)) !== null) {
 +        // Add other text before this match
 +        if (match.index > lastIndex) {
 +            const otherText = text.substring(lastIndex, match.index).trim();
 +            if (otherText && otherText.length > 0) {
 +                segments.push({ type: 'other', text: otherText });
 +            }
 +        }
++
 +        const matchedText = match[0];
 +        let segmentType = 'other';
 +        let content = '';
++
 +        if (match[1]) {
 +            // Asterisk content (*action*)
 +            segmentType = 'action';
 +            content = matchedText.slice(1, -1);
 +        } else if (match[2] || match[3] || match[4] || match[5] || match[6] || match[7]) {
 +            // Various quote types ("dialogue")
 +            segmentType = 'dialogue';
 +            content = matchedText.slice(1, -1);
 +        }
++
 +        // Trim and check for actual content
 +        content = content.trim();
 +        if (content.length > 0) {
 +            segments.push({
 +                type: segmentType,
 +                text: content,
 +            });
 +        }
++
 +        lastIndex = match.index + matchedText.length;
 +    }
++
 +    // Add remaining other text after last match
 +    if (lastIndex < text.length) {
 +        const otherText = text.substring(lastIndex).trim();
 +        if (otherText.length > 0) {
 +            segments.push({ type: 'other', text: otherText });
 +        }
 +    }
++
 +    // If no segments found and not empty, treat whole text as other text
 +    if (segments.length === 0 && text.trim().length > 0) {
 +        segments.push({ type: 'other', text: text.trim() });
 +    }
++
 +    return segments;
 +}
++
 async function processTtsQueue() {
     // Called each moduleWorker iteration to pull chat messages from queue
     if (currentTtsJob || ttsJobQueue.length <= 0 || audioPaused) {

     console.debug('New message found, running TTS');
     currentTtsJob = ttsJobQueue.shift();
++
 +    // Handle segmented jobs that already have processed text
 +    if (currentTtsJob.segmentType && currentTtsJob.segmentText) {
 +        const char = currentTtsJob.name;
 +        const segmentText = currentTtsJob.segmentText;
 +        const segmentType = currentTtsJob.segmentType;
++
 +        console.log(`TTS (${segmentType}): ${segmentText}`);
++
 +        try {
 +            let voiceMapKey = char;
++
 +            // If multi-voice is enabled, modify the voice map key based on segment type
 +            if (extension_settings.tts.multi_voice_enabled && char !== DEFAULT_VOICE_MARKER) {
 +                switch (segmentType) {
 +                    case 'dialogue':
 +                        voiceMapKey = `${char} ("Quotes")`;
 +                        break;
 +                    case 'action':
 +                        voiceMapKey = `${char} (*Text inside asterisks*)`;
 +                        break;
 +                    case 'other':
 +                    default:
 +                        voiceMapKey = `${char} (Other text)`;
 +                        break;
 +                }
 +            }
++
 +            const voiceMapEntry = voiceMap[voiceMapKey] === DEFAULT_VOICE_MARKER ? voiceMap[DEFAULT_VOICE_MARKER] : voiceMap[voiceMapKey];
++
 +            if (!voiceMapEntry || voiceMapEntry === DISABLED_VOICE_MARKER) {
 +                throw `${char} not in voicemap. Configure character in extension settings voice map`;
 +            }
++
 +            const voice = await ttsProvider.getVoice(voiceMapEntry);
 +            const voiceId = voice.voice_id;
 +            if (voiceId == null) {
 +                toastr.error(`Specified voice for ${char} was not found. Check the TTS extension settings.`);
 +                throw `Unable to attain voiceId for ${char}`;
 +            }
++
 +            // Pass the full voiceMapKey (e.g., "User ("Quotes")") as well with character name
 +            await tts(segmentText, voiceId, char, voiceMapKey);
++
 +        } catch (error) {
 +            toastr.error(error.toString());
 +            console.error(error);
 +            currentTtsJob = null;
 +        }
 +        return;
 +    }
++
 +    // Process unsegmented job (first time processing)
     let text = extension_settings.tts.narrate_translated_only ? (currentTtsJob?.extra?.display_text || currentTtsJob.mes) : currentTtsJob.mes;
     // Substitute macros

             return;
+        }
 -        const voiceMapEntry = voiceMap[char] === DEFAULT_VOICE_MARKER ? voiceMap[DEFAULT_VOICE_MARKER] : voiceMap[char];
 +        // Parse message into segments if multi-voice is enabled
 +        const segments = parseMessageSegments(text);
         if (!voiceMapEntry || voiceMapEntrysegments.length === DISABLED_VOICE_MARKER0) {
 -            throw `${char} not in voicemap. Configure character in extension settings voice map`;
 +            console.warn('No valid segments found in text.');
 +            completeTtsJob();
 +            return;
+        }
 -        const voice = await ttsProvider.getVoice(voiceMapEntry);
++
 -        const voiceId = voice.voice_id;
 +        // Add all segments to the queue as separate jobs (in reverse order so they process in correct order)
 -        if (voiceId == null) {
 +        for (let i = segments.length - 1; i >= 0; i--) {
 -            toastr.error(`Specified voice for ${char} was not found. Check the TTS extension settings.`);
 +            const segmentJob = {
 -            throw `Unable to attain voiceId for ${char}`;
 +                name: char,
 +                segmentType: segments[i].type,
 +                segmentText: segments[i].text,
 +                is_user: currentTtsJob.is_user,
 +                mes: currentTtsJob.mes,
 +                extra: currentTtsJob.extra,
 +            };
 +            ttsJobQueue.unshift(segmentJob);
+        }
 -        await tts(text, voiceId, char);
++
 +        // Clear current job so the segmented jobs can be processed
 +        currentTtsJob = null;
++
     } catch (error) {
         toastr.error(error.toString());
         console.error(error);

     $('#tts_pass_asterisks').prop('checked', extension_settings.tts.pass_asterisks);
     $('#tts_skip_codeblocks').prop('checked', extension_settings.tts.skip_codeblocks);
     $('#tts_skip_tags').prop('checked', extension_settings.tts.skip_tags);
 +    $('#tts_multi_voice_enabled').prop('checked', extension_settings.tts.multi_voice_enabled);
     $('#playback_rate').val(extension_settings.tts.playback_rate);
     $('#playback_rate_counter').val(Number(extension_settings.tts.playback_rate).toFixed(2));
     $('#playback_rate_block').toggle(extension_settings.tts.currentProvider !== 'System');

     auto_generation: true,
     narrate_user: false,
     playback_rate: 1,
 +    multi_voice_enabled: false,
 };
 function setTtsStatus(status, success) {

     console.log('setting pass asterisks', extension_settings.tts.pass_asterisks);
+}
 +function onMultiVoiceClick() {
 +    extension_settings.tts.multi_voice_enabled = !!$('#tts_multi_voice_enabled').prop('checked');
 +    saveSettingsDebounced();
 +    // Reinitialize voice map to show/hide voices
 +    initVoiceMap();
 +}
++
 //##############//
 // TTS Provider //
 //##############//

+            }
+        }
+    }
     returncharacters = characters.filter(onlyUnique);
++
 +    // If multi-voice is enabled, expand characters to include segment types
 +    if (extension_settings.tts.multi_voice_enabled) {
 +        const expandedCharacters = [];
 +        for (const char of characters) {
 +            if (char === DEFAULT_VOICE_MARKER || char === 'SillyTavern System') {
 +                expandedCharacters.push(char);
 +            } else {
 +                expandedCharacters.push(`${char} ("Quotes")`);
 +                expandedCharacters.push(`${char} (*Text inside asterisks*)`);
 +                expandedCharacters.push(`${char} (Other text)`);
 +            }
 +        }
 +        return expandedCharacters;
 +    }
++
 +    return characters;
++
+}
 export function sanitizeId(input) {

         $('#tts_periodic_auto_generation').on('click', onPeriodicAutoGenerationClick);
         $('#tts_narrate_by_paragraphs').on('click', onNarrateByParagraphsClick);
         $('#tts_narrate_user').on('click', onNarrateUserClick);
 +        $('#tts_multi_voice_enabled').on('click', onMultiVoiceClick);
         $('#playback_rate').on('input', function () {
             const value = $(this).val();

public/scripts/extensions/tts/settings.html+10 -2 Viewed

                     <small data-i18n="Skip tagged blocks">Skip &lt;tagged&gt; blocks</small>
                 </label>
                 <label class="checkbox_label" for="tts_pass_asterisks">
                                     <input type="checkbox" id="tts_pass_asterisks">
                                     <small data-i18n="Pass Asterisks to TTS Engine">Pass Asterisks to TTS Engine</small>
 +                </label>
 +                <label class="checkbox_label" for="tts_multi_voice_enabled"
 +                data-i18n="[title]Works best when: Pass Asterisks to TTS Engine is enabled, and both Only narrate quotes and Ignore *text, even 'quotes', inside asterisks* are disabled."
 +                title="Works best when: Pass Asterisks to TTS Engine is enabled, and both Only narrate quotes and Ignore *text, even 'quotes', inside asterisks* are disabled.">
 +                    <input type="checkbox" id="tts_multi_voice_enabled">
 +                    <small data-i18n="Different voices for quotes and text inside asterisks">
 +                        Different voices for "quotes", *text inside asterisks* and other text
 +                    </small>
                 </label>
             </div>
             <div id="playback_rate_block" class="range-block">