feat(tts): emit events and track messageId for third-party integrations (#5309) * feat(tts): add event signals and messageId tracking for third-party integrations * feat(tts): move events to core, centralize clone and id tracking * Apply review suggestions --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>

e4ad48944806de61ef87ad22ea5262a4ae3cdcec

Xiangzhe <32761048+xz-dev@users.noreply.github.com>

Signed

public/scripts/events.js+3 -0 Viewed

     WORLDINFO_SCAN_DONE: 'worldinfo_scan_done',
     MEDIA_ATTACHMENT_DELETED: 'media_attachment_deleted',
     PERSONA_CHANGED: 'persona_changed',
+    TTS_JOB_STARTED: 'tts_job_started',
+    TTS_AUDIO_READY: 'tts_audio_ready',
+    TTS_JOB_COMPLETE: 'tts_job_complete',
 };
 export const eventSource = new EventEmitter([event_types.APP_READY, event_types.APP_INITIALIZED]);

public/scripts/extensions/tts/index.js+48 -19 Viewed

@@ -165,7 +165,7 @@ async function onNarrateOneMessage() {
165	}	165	}
166		166
167	resetTtsPlayback();	167	resetTtsPlayback();
168	processAndQueueTtsMessage(message);	168	processAndQueueTtsMessage(message, Number(id));
169	moduleWorker();	169	moduleWorker();
170	}	170	}
171		171

+}
 /**
- * Splits a message into lines and adds each non-empty line to the TTS job queue.
+ * @typedef {ChatMessage & { id?: number }} TtsMessage
+ */
+/**
+ * Clones a message, attaches the given message ID, then splits by paragraphs
+ * (if enabled) and adds each part to the TTS job queue.
  * @param {ChatMessage} message - The message object to be processed.
+ * @param {number|null} [messageId=null] - The chat message index to associate with TTS events.
  * @returns {void}
  */
-function processAndQueueTtsMessage(message) {
+function processAndQueueTtsMessage(message, messageId = null) {
+    /** @type {TtsMessage} */
+    const clone = structuredClone(message);
+    clone.id = messageId ?? null;
     if (!extension_settings.tts.narrate_by_paragraphs) {
-        ttsJobQueue.push(message);
+        ttsJobQueue.push(clone);
         return;
+    }
-    const lines = message.mes.split('\n');
+    const lines = clone.mes.split('\n');
     for (let i = 0; i < lines.length; i++) {
         const line = lines[i];

+        }
         ttsJobQueue.push(
-            Object.assign({}, message, {
+            Object.assign({}, clone, {
                 mes: line,
             }),
         );

  * @type AudioJob[] Audio job queue
  * @typedef {{audioBlob: Blob | string, char: string}} AudioJob Audio job object
  */
-let audioJobQueue = [];
+const audioJobQueue = [];
 /**
  * @type AudioJob Current audio job
  */

 /**
  * Accepts an HTTP response containing audio/mpeg data, and puts the data as a Blob() on the queue for playback
  * @param {Response} response
+ * @param {string} char
+ * @returns {Promise<{audioBlob: Blob|string, mimeType: string}>}
  */
 async function addAudioJob(response, char) {
+    let audioBlob, mimeType;
     if (typeof response === 'string') {
-        audioJobQueue.push({ audioBlob: response, char: char });
+        audioBlob = response;
+        mimeType = '';
     } else {
-        const audioData = await response.blob();
+        audioBlob = await response.blob();
-        if (!audioData.type.startsWith('audio/')) {
+        if (!audioBlob.type.startsWith('audio/')) {
-            throw `TTS received HTTP response with invalid data format. Expecting audio/*, got ${audioData.type}`;
+            throw `TTS received HTTP response with invalid data format. Expecting audio/*, got ${audioBlob.type}`;
+        }
-        audioJobQueue.push({ audioBlob: audioData, char: char });
+        mimeType = audioBlob.type;
+    }
+    audioJobQueue.push({ audioBlob, char });
     console.debug('Pushed audio job to queue.');
+    return { audioBlob, mimeType };
+}
 async function processAudioJobQueue() {

 //  TTS Control   //
 //################//
-let ttsJobQueue = [];
+const ttsJobQueue = [];
 let currentTtsJob; // Null if nothing is currently being processed
 function completeTtsJob() {

+}
 async function tts(text, voiceId, char, voiceMapKey = null) {
+    const messageId = currentTtsJob?.id ?? null;
+    await eventSource.emit(event_types.TTS_JOB_STARTED, { messageId, characterName: char, text, voiceId });
     async function processResponse(response) {
         // RVC injection
         if (typeof globalThis.rvcVoiceConversion === 'function' && extension_settings.rvc.enabled)
             response = await globalThis.rvcVoiceConversion(response, char, text);
-        await addAudioJob(response, char);
+        const audioResult = await addAudioJob(response, char);
+        const eventData = { messageId, characterName: char, text, audio: audioResult.audioBlob, mimeType: audioResult.mimeType };
+        await eventSource.emit(event_types.TTS_AUDIO_READY, eventData);
+    }
     // voiceMapKey can also include segment qualifiers, e.g. '{char} ("Quotes")'

@@ -494,6 +516,7 @@ async function tts(text, voiceId, char, voiceMapKey = null) {
494	await processResponse(response);	516	await processResponse(response);
495	}	517	}
496		518
		519	await eventSource.emit(event_types.TTS_JOB_COMPLETE, { messageId, characterName: char });
497	completeTtsJob();	520	completeTtsJob();
498	}	521	}
499		522

                 is_user: currentTtsJob.is_user,
                 mes: currentTtsJob.mes,
                 extra: currentTtsJob.extra,
+                id: currentTtsJob.id,
             };
             ttsJobQueue.unshift(segmentJob);
+        }

@@ -797,13 +821,16 @@ async function playFullConversation() {
797	}	821	}
798		822
799	const context = getContext();	823	const context = getContext();
800	const chat = context.chat.filter(x => !x.is_system && x.mes !== '...' && x.mes !== '');
801		824
802	if (chat.length === 0) {	825	context.chat.forEach((msg, i) => {
		826	if (!msg.is_system && msg.mes !== '...' && msg.mes !== '') {
		827	processAndQueueTtsMessage(msg, i);
		828	}
		829	});
		830
		831	if (ttsJobQueue.length === 0) {
803	return toastr.info('No messages to narrate.');	832	return toastr.info('No messages to narrate.');
804	}	833	}
805
806	ttsJobQueue = chat;
807	}	834	}
808		835
809	globalThis.playFullConversation = playFullConversation;	836	globalThis.playFullConversation = playFullConversation;

+    }
     // clone message object, as things go haywire if message object is altered below (it's passed by reference)
+    /** @type {TtsMessage} */
     const message = structuredClone(context.chat[messageId]);
     const hashNew = getStringHash(message?.mes ?? '');

@@ -1133,9 +1161,10 @@ async function onMessageEvent(messageId, lastCharIndex) {
1133	console.debug(`Adding message from ${message.name} for TTS processing: "${message.mes}"`);	1161	console.debug(`Adding message from ${message.name} for TTS processing: "${message.mes}"`);
1134		1162
1135	if (extension_settings.tts.periodic_auto_generation && isStreamingEnabled()) {	1163	if (extension_settings.tts.periodic_auto_generation && isStreamingEnabled()) {
		1164	message.id = messageId;
1136	ttsJobQueue.push(message);	1165	ttsJobQueue.push(message);
1137	} else {	1166	} else {
1138	processAndQueueTtsMessage(message);	1167	processAndQueueTtsMessage(message, messageId);
1139	}	1168	}
1140	}	1169	}
1141		1170