Added time to first token

d14b5e51a9715a119100a9f943c480cf3f044d56

bmen25124 <bmen25124@gmail.com>

Parents: 5d255d758ebb2516e54342edae04d2e18a3ae443
1 files changed, +10 -3Showing whitespace changes
public/script.js+10 -3
@@ -2477,7 +2477,7 @@ export function addOneMessage(mes, { type = 'normal', insertAfter = null, scroll
2477 timestamp: timestamp,2477 timestamp: timestamp,
2478 extra: mes.extra,2478 extra: mes.extra,
2479 tokenCount: mes.extra?.token_count ?? 0,2479 tokenCount: mes.extra?.token_count ?? 0,
2480 ...formatGenerationTimer(mes.gen_started, mes.gen_finished, mes.extra?.token_count, mes.extra?.reasoning_duration),2480 ...formatGenerationTimer(mes.gen_started, mes.gen_finished, mes.extra?.token_count, mes.time_to_first_token, mes.extra?.reasoning_duration),
2481 };2481 };
24822482
2483 const renderedMessage = getMessageFromTemplate(params);2483 const renderedMessage = getMessageFromTemplate(params);
@@ -2597,6 +2597,7 @@ export function formatCharacterAvatar(characterAvatar) {
2597 * @param {Date} gen_started Date when generation was started2597 * @param {Date} gen_started Date when generation was started
2598 * @param {Date} gen_finished Date when generation was finished2598 * @param {Date} gen_finished Date when generation was finished
2599 * @param {number} tokenCount Number of tokens generated (0 if not available)2599 * @param {number} tokenCount Number of tokens generated (0 if not available)
2600 * @param {number} timeToFirstToken Time to first token
2600 * @param {number?} [reasoningDuration=null] Reasoning duration (null if no reasoning was done)2601 * @param {number?} [reasoningDuration=null] Reasoning duration (null if no reasoning was done)
2601 * @returns {Object} Object containing the formatted timer value and title2602 * @returns {Object} Object containing the formatted timer value and title
2602 * @example2603 * @example
@@ -2604,7 +2605,7 @@ export function formatCharacterAvatar(characterAvatar) {
2604 * console.log(timerValue); // 1.2s2605 * console.log(timerValue); // 1.2s
2605 * console.log(timerTitle); // Generation queued: 12:34:56 7 Jan 2021\nReply received: 12:34:57 7 Jan 2021\nTime to generate: 1.2 seconds\nToken rate: 5 t/s2606 * console.log(timerTitle); // Generation queued: 12:34:56 7 Jan 2021\nReply received: 12:34:57 7 Jan 2021\nTime to generate: 1.2 seconds\nToken rate: 5 t/s
2606 */2607 */
2607function formatGenerationTimer(gen_started, gen_finished, tokenCount, reasoningDuration = null) {2608function formatGenerationTimer(gen_started, gen_finished, tokenCount, timeToFirstToken, reasoningDuration = null) {
2608 if (!gen_started || !gen_finished) {2609 if (!gen_started || !gen_finished) {
2609 return {};2610 return {};
2610 }2611 }
@@ -2618,6 +2619,7 @@ function formatGenerationTimer(gen_started, gen_finished, tokenCount, reasoningD
2618 `Generation queued: ${start.format(dateFormat)}`,2619 `Generation queued: ${start.format(dateFormat)}`,
2619 `Reply received: ${finish.format(dateFormat)}`,2620 `Reply received: ${finish.format(dateFormat)}`,
2620 `Time to generate: ${seconds} seconds`,2621 `Time to generate: ${seconds} seconds`,
2622 timeToFirstToken > 0 ? `Time to first token: ${(timeToFirstToken / 1000).toFixed(1)} seconds` : '',
2621 reasoningDuration > 0 ? `Time to think: ${reasoningDuration / 1000} seconds` : '',2623 reasoningDuration > 0 ? `Time to think: ${reasoningDuration / 1000} seconds` : '',
2622 tokenCount > 0 ? `Token rate: ${Number(tokenCount / seconds).toFixed(1)} t/s` : '',2624 tokenCount > 0 ? `Token rate: ${Number(tokenCount / seconds).toFixed(1)} t/s` : '',
2623 ].filter(x => x).join('\n').trim();2625 ].filter(x => x).join('\n').trim();
@@ -3158,6 +3160,8 @@ class StreamingProcessor {
3158 this.abortController = new AbortController();3160 this.abortController = new AbortController();
3159 this.firstMessageText = '...';3161 this.firstMessageText = '...';
3160 this.timeStarted = timeStarted;3162 this.timeStarted = timeStarted;
3163 this.timeToFirstToken = -1;
3164 this.createdAt = new Date();
3161 this.continueMessage = type === 'continue' ? continueMessage : '';3165 this.continueMessage = type === 'continue' ? continueMessage : '';
3162 this.swipes = [];3166 this.swipes = [];
3163 /** @type {import('./scripts/logprobs.js').TokenLogprobs[]} */3167 /** @type {import('./scripts/logprobs.js').TokenLogprobs[]} */
@@ -3195,6 +3199,8 @@ class StreamingProcessor {
3195 }3199 }
31963200
3197 async onStartStreaming(text) {3201 async onStartStreaming(text) {
3202 this.timeToFirstToken = Date.now() - this.createdAt.getTime();
3203
3198 if (this.type === 'continue' && this.promptReasoning.prefixReasoning) {3204 if (this.type === 'continue' && this.promptReasoning.prefixReasoning) {
3199 this.reasoningHandler.initContinue(this.promptReasoning);3205 this.reasoningHandler.initContinue(this.promptReasoning);
3200 }3206 }
@@ -3245,6 +3251,7 @@ class StreamingProcessor {
3245 const currentTime = new Date();3251 const currentTime = new Date();
3246 chat[messageId]['mes'] = processedText;3252 chat[messageId]['mes'] = processedText;
3247 chat[messageId]['gen_started'] = this.timeStarted;3253 chat[messageId]['gen_started'] = this.timeStarted;
3254 chat[messageId]['time_to_first_token'] = this.timeToFirstToken;
3248 chat[messageId]['gen_finished'] = currentTime;3255 chat[messageId]['gen_finished'] = currentTime;
3249 if (!chat[messageId]['extra']) {3256 if (!chat[messageId]['extra']) {
3250 chat[messageId]['extra'] = {};3257 chat[messageId]['extra'] = {};
@@ -3282,7 +3289,7 @@ class StreamingProcessor {
3282 this.messageTextDom.innerHTML = formattedText;3289 this.messageTextDom.innerHTML = formattedText;
3283 }3290 }
32843291
3285 const timePassed = formatGenerationTimer(this.timeStarted, currentTime, currentTokenCount, this.reasoningHandler.getDuration());3292 const timePassed = formatGenerationTimer(this.timeStarted, currentTime, currentTokenCount, this.timeToFirstToken, this.reasoningHandler.getDuration());
3286 if (this.messageTimerDom instanceof HTMLElement) {3293 if (this.messageTimerDom instanceof HTMLElement) {
3287 this.messageTimerDom.textContent = timePassed.timerValue;3294 this.messageTimerDom.textContent = timePassed.timerValue;
3288 this.messageTimerDom.title = timePassed.timerTitle;3295 this.messageTimerDom.title = timePassed.timerTitle;