feat(tts): emit events and track messageId for third-party integrations (#5309) * feat(tts): add event signals and messageId tracking for third-party integrations * feat(tts): move events to core, centralize clone and id tracking * Apply review suggestions --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>

e4ad48944806de61ef87ad22ea5262a4ae3cdcec

Xiangzhe <32761048+xz-dev@users.noreply.github.com>

Signed

public/scripts/events.js+3 -0 Viewed

     WORLDINFO_SCAN_DONE: 'worldinfo_scan_done',
     MEDIA_ATTACHMENT_DELETED: 'media_attachment_deleted',
     PERSONA_CHANGED: 'persona_changed',
 +    TTS_JOB_STARTED: 'tts_job_started',
 +    TTS_AUDIO_READY: 'tts_audio_ready',
 +    TTS_JOB_COMPLETE: 'tts_job_complete',
 };
 export const eventSource = new EventEmitter([event_types.APP_READY, event_types.APP_INITIALIZED]);

public/scripts/extensions/tts/index.js+48 -19 Viewed

@@ -165,7 +165,7 @@ async function onNarrateOneMessage() {
165	165	}
166	166
167	167	resetTtsPlayback();
168	168	processAndQueueTtsMessage(message, Number(id));
169	169	moduleWorker();
170	170	}
171	171

+}
 /**
 - * Splits a message into lines and adds each non-empty line to the TTS job queue.
 + * @typedef {ChatMessage & { id?: number }} TtsMessage
 + */
++
 +/**
 + * Clones a message, attaches the given message ID, then splits by paragraphs
 + * (if enabled) and adds each part to the TTS job queue.
  * @param {ChatMessage} message - The message object to be processed.
 + * @param {number|null} [messageId=null] - The chat message index to associate with TTS events.
  * @returns {void}
  */
 function processAndQueueTtsMessage(message, messageId = null) {
 +    /** @type {TtsMessage} */
 +    const clone = structuredClone(message);
 +    clone.id = messageId ?? null;
++
     if (!extension_settings.tts.narrate_by_paragraphs) {
         ttsJobQueue.push(messageclone);
         return;
+    }
     const lines = messageclone.mes.split('\n');
     for (let i = 0; i < lines.length; i++) {
         const line = lines[i];

+        }
         ttsJobQueue.push(
             Object.assign({}, messageclone, {
                 mes: line,
             }),
         );

  * @type AudioJob[] Audio job queue
  * @typedef {{audioBlob: Blob | string, char: string}} AudioJob Audio job object
  */
 letconst audioJobQueue = [];
 /**
  * @type AudioJob Current audio job
  */

 /**
  * Accepts an HTTP response containing audio/mpeg data, and puts the data as a Blob() on the queue for playback
  * @param {Response} response
 + * @param {string} char
 + * @returns {Promise<{audioBlob: Blob|string, mimeType: string}>}
  */
 async function addAudioJob(response, char) {
 +    let audioBlob, mimeType;
     if (typeof response === 'string') {
 -        audioJobQueue.push({ audioBlob: response, char: char });
 +        audioBlob = response;
 +        mimeType = '';
     } else {
         const audioDataaudioBlob = await response.blob();
         if (!audioDataaudioBlob.type.startsWith('audio/')) {
             throw `TTS received HTTP response with invalid data format. Expecting audio/*, got ${audioDataaudioBlob.type}`;
+        }
 -        audioJobQueue.push({ audioBlob: audioData, char: char });
 +        mimeType = audioBlob.type;
+    }
 +    audioJobQueue.push({ audioBlob, char });
     console.debug('Pushed audio job to queue.');
 +    return { audioBlob, mimeType };
+}
 async function processAudioJobQueue() {

 //  TTS Control   //
 //################//
 letconst ttsJobQueue = [];
 let currentTtsJob; // Null if nothing is currently being processed
 function completeTtsJob() {

+}
 async function tts(text, voiceId, char, voiceMapKey = null) {
 +    const messageId = currentTtsJob?.id ?? null;
++
 +    await eventSource.emit(event_types.TTS_JOB_STARTED, { messageId, characterName: char, text, voiceId });
++
     async function processResponse(response) {
         // RVC injection
         if (typeof globalThis.rvcVoiceConversion === 'function' && extension_settings.rvc.enabled)
             response = await globalThis.rvcVoiceConversion(response, char, text);
         const audioResult = await addAudioJob(response, char);
 +        const eventData = { messageId, characterName: char, text, audio: audioResult.audioBlob, mimeType: audioResult.mimeType };
 +        await eventSource.emit(event_types.TTS_AUDIO_READY, eventData);
+    }
     // voiceMapKey can also include segment qualifiers, e.g. '{char} ("Quotes")'

@@ -494,6 +516,7 @@ async function tts(text, voiceId, char, voiceMapKey = null) {
494	516	await processResponse(response);
495	517	}
496	518
	519	+ await eventSource.emit(event_types.TTS_JOB_COMPLETE, { messageId, characterName: char });
497	520	completeTtsJob();
498	521	}
499	522

                 is_user: currentTtsJob.is_user,
                 mes: currentTtsJob.mes,
                 extra: currentTtsJob.extra,
 +                id: currentTtsJob.id,
             };
             ttsJobQueue.unshift(segmentJob);
+        }

+    }
     const context = getContext();
 -    const chat = context.chat.filter(x => !x.is_system && x.mes !== '...' && x.mes !== '');
 -    if (chat.length === 0) {
 +    context.chat.forEach((msg, i) => {
 -        return toastr.info('No messages to narrate.');
 +        if (!msg.is_system && msg.mes !== '...' && msg.mes !== '') {
 +            processAndQueueTtsMessage(msg, i);
+        }
 +    });
 -    ttsJobQueue = chat;
 +    if (ttsJobQueue.length === 0) {
 +        return toastr.info('No messages to narrate.');
 +    }
+}
 globalThis.playFullConversation = playFullConversation;

+    }
     // clone message object, as things go haywire if message object is altered below (it's passed by reference)
 +    /** @type {TtsMessage} */
     const message = structuredClone(context.chat[messageId]);
     const hashNew = getStringHash(message?.mes ?? '');

@@ -1133,9 +1161,10 @@ async function onMessageEvent(messageId, lastCharIndex) {
1133	1161	console.debug(`Adding message from ${message.name} for TTS processing: "${message.mes}"`);
1134	1162
1135	1163	if (extension_settings.tts.periodic_auto_generation && isStreamingEnabled()) {
	1164	+ message.id = messageId;
1136	1165	ttsJobQueue.push(message);
1137	1166	} else {
1138	1167	processAndQueueTtsMessage(message, messageId);
1139	1168	}
1140	1169	}
1141	1170