Merge pull request #3610 from bmen25124/time_to_first_token Added time to first token

c36607be6f72074030c72e9e22eb7e451e549f06

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: 3b9fcfae60ddd54612801a9065538d28ac49ee1a fb5f3e0f97602682c4aec498a318b4c589f30a5f
1 files changed, +33 -8Showing whitespace changes
public/script.js+33 -8
@@ -2477,7 +2477,7 @@ export function addOneMessage(mes, { type = 'normal', insertAfter = null, scroll
2477 timestamp: timestamp,2477 timestamp: timestamp,
2478 extra: mes.extra,2478 extra: mes.extra,
2479 tokenCount: mes.extra?.token_count ?? 0,2479 tokenCount: mes.extra?.token_count ?? 0,
2480 ...formatGenerationTimer(mes.gen_started, mes.gen_finished, mes.extra?.token_count, mes.extra?.reasoning_duration),2480 ...formatGenerationTimer(mes.gen_started, mes.gen_finished, mes.extra?.token_count, mes.extra?.reasoning_duration, mes.extra?.time_to_first_token),
2481 };2481 };
24822482
2483 const renderedMessage = getMessageFromTemplate(params);2483 const renderedMessage = getMessageFromTemplate(params);
@@ -2598,13 +2598,14 @@ export function formatCharacterAvatar(characterAvatar) {
2598 * @param {Date} gen_finished Date when generation was finished2598 * @param {Date} gen_finished Date when generation was finished
2599 * @param {number} tokenCount Number of tokens generated (0 if not available)2599 * @param {number} tokenCount Number of tokens generated (0 if not available)
2600 * @param {number?} [reasoningDuration=null] Reasoning duration (null if no reasoning was done)2600 * @param {number?} [reasoningDuration=null] Reasoning duration (null if no reasoning was done)
2601 * @param {number?} [timeToFirstToken=null] Time to first token
2601 * @returns {Object} Object containing the formatted timer value and title2602 * @returns {Object} Object containing the formatted timer value and title
2602 * @example2603 * @example
2603 * const { timerValue, timerTitle } = formatGenerationTimer(gen_started, gen_finished, tokenCount);2604 * const { timerValue, timerTitle } = formatGenerationTimer(gen_started, gen_finished, tokenCount);
2604 * console.log(timerValue); // 1.2s2605 * console.log(timerValue); // 1.2s
2605 * console.log(timerTitle); // Generation queued: 12:34:56 7 Jan 2021\nReply received: 12:34:57 7 Jan 2021\nTime to generate: 1.2 seconds\nToken rate: 5 t/s2606 * console.log(timerTitle); // Generation queued: 12:34:56 7 Jan 2021\nReply received: 12:34:57 7 Jan 2021\nTime to generate: 1.2 seconds\nToken rate: 5 t/s
2606 */2607 */
2607function formatGenerationTimer(gen_started, gen_finished, tokenCount, reasoningDuration = null) {2608function formatGenerationTimer(gen_started, gen_finished, tokenCount, reasoningDuration = null, timeToFirstToken = null) {
2608 if (!gen_started || !gen_finished) {2609 if (!gen_started || !gen_finished) {
2609 return {};2610 return {};
2610 }2611 }
@@ -2618,8 +2619,9 @@ function formatGenerationTimer(gen_started, gen_finished, tokenCount, reasoningD
2618 `Generation queued: ${start.format(dateFormat)}`,2619 `Generation queued: ${start.format(dateFormat)}`,
2619 `Reply received: ${finish.format(dateFormat)}`,2620 `Reply received: ${finish.format(dateFormat)}`,
2620 `Time to generate: ${seconds} seconds`,2621 `Time to generate: ${seconds} seconds`,
2622 timeToFirstToken ? `Time to first token: ${timeToFirstToken / 1000} seconds` : '',
2621 reasoningDuration > 0 ? `Time to think: ${reasoningDuration / 1000} seconds` : '',2623 reasoningDuration > 0 ? `Time to think: ${reasoningDuration / 1000} seconds` : '',
2622 tokenCount > 0 ? `Token rate: ${Number(tokenCount / seconds).toFixed(1)} t/s` : '',2624 tokenCount > 0 ? `Token rate: ${Number(tokenCount / seconds).toFixed(3)} t/s` : '',
2623 ].filter(x => x).join('\n').trim();2625 ].filter(x => x).join('\n').trim();
26242626
2625 if (isNaN(seconds) || seconds < 0) {2627 if (isNaN(seconds) || seconds < 0) {
@@ -3158,6 +3160,9 @@ class StreamingProcessor {
3158 this.abortController = new AbortController();3160 this.abortController = new AbortController();
3159 this.firstMessageText = '...';3161 this.firstMessageText = '...';
3160 this.timeStarted = timeStarted;3162 this.timeStarted = timeStarted;
3163 /** @type {number?} */
3164 this.timeToFirstToken = null;
3165 this.createdAt = new Date();
3161 this.continueMessage = type === 'continue' ? continueMessage : '';3166 this.continueMessage = type === 'continue' ? continueMessage : '';
3162 this.swipes = [];3167 this.swipes = [];
3163 /** @type {import('./scripts/logprobs.js').TokenLogprobs[]} */3168 /** @type {import('./scripts/logprobs.js').TokenLogprobs[]} */
@@ -3249,6 +3254,7 @@ class StreamingProcessor {
3249 if (!chat[messageId]['extra']) {3254 if (!chat[messageId]['extra']) {
3250 chat[messageId]['extra'] = {};3255 chat[messageId]['extra'] = {};
3251 }3256 }
3257 chat[messageId]['extra']['time_to_first_token'] = this.timeToFirstToken;
32523258
3253 // Update reasoning3259 // Update reasoning
3254 await this.reasoningHandler.process(messageId, mesChanged);3260 await this.reasoningHandler.process(messageId, mesChanged);
@@ -3266,7 +3272,12 @@ class StreamingProcessor {
32663272
3267 if ((this.type == 'swipe' || this.type === 'continue') && Array.isArray(chat[messageId]['swipes'])) {3273 if ((this.type == 'swipe' || this.type === 'continue') && Array.isArray(chat[messageId]['swipes'])) {
3268 chat[messageId]['swipes'][chat[messageId]['swipe_id']] = processedText;3274 chat[messageId]['swipes'][chat[messageId]['swipe_id']] = processedText;
3269 chat[messageId]['swipe_info'][chat[messageId]['swipe_id']] = { 'send_date': chat[messageId]['send_date'], 'gen_started': chat[messageId]['gen_started'], 'gen_finished': chat[messageId]['gen_finished'], 'extra': JSON.parse(JSON.stringify(chat[messageId]['extra'])) };3275 chat[messageId]['swipe_info'][chat[messageId]['swipe_id']] = {
3276 'send_date': chat[messageId]['send_date'],
3277 'gen_started': chat[messageId]['gen_started'],
3278 'gen_finished': chat[messageId]['gen_finished'],
3279 'extra': JSON.parse(JSON.stringify(chat[messageId]['extra']))
3280 };
3270 }3281 }
32713282
3272 const formattedText = messageFormatting(3283 const formattedText = messageFormatting(
@@ -3282,7 +3293,7 @@ class StreamingProcessor {
3282 this.messageTextDom.innerHTML = formattedText;3293 this.messageTextDom.innerHTML = formattedText;
3283 }3294 }
32843295
3285 const timePassed = formatGenerationTimer(this.timeStarted, currentTime, currentTokenCount, this.reasoningHandler.getDuration());3296 const timePassed = formatGenerationTimer(this.timeStarted, currentTime, currentTokenCount, this.reasoningHandler.getDuration(), this.timeToFirstToken);
3286 if (this.messageTimerDom instanceof HTMLElement) {3297 if (this.messageTimerDom instanceof HTMLElement) {
3287 this.messageTimerDom.textContent = timePassed.timerValue;3298 this.messageTimerDom.textContent = timePassed.timerValue;
3288 this.messageTimerDom.title = timePassed.timerTitle;3299 this.messageTimerDom.title = timePassed.timerTitle;
@@ -3357,7 +3368,12 @@ class StreamingProcessor {
3357 if (this.type !== 'swipe' && this.type !== 'impersonate') {3368 if (this.type !== 'swipe' && this.type !== 'impersonate') {
3358 if (Array.isArray(chat[messageId]['swipes']) && chat[messageId]['swipes'].length === 1 && chat[messageId]['swipe_id'] === 0) {3369 if (Array.isArray(chat[messageId]['swipes']) && chat[messageId]['swipes'].length === 1 && chat[messageId]['swipe_id'] === 0) {
3359 chat[messageId]['swipes'][0] = chat[messageId]['mes'];3370 chat[messageId]['swipes'][0] = chat[messageId]['mes'];
3360 chat[messageId]['swipe_info'][0] = { 'send_date': chat[messageId]['send_date'], 'gen_started': chat[messageId]['gen_started'], 'gen_finished': chat[messageId]['gen_finished'], 'extra': JSON.parse(JSON.stringify(chat[messageId]['extra'])) };3371 chat[messageId]['swipe_info'][0] = {
3372 'send_date': chat[messageId]['send_date'],
3373 'gen_started': chat[messageId]['gen_started'],
3374 'gen_finished': chat[messageId]['gen_finished'],
3375 'extra': JSON.parse(JSON.stringify(chat[messageId]['extra'])),
3376 };
3361 }3377 }
3362 }3378 }
3363 }3379 }
@@ -3391,7 +3407,11 @@ class StreamingProcessor {
3391 const sw = new Stopwatch(1000 / power_user.streaming_fps);3407 const sw = new Stopwatch(1000 / power_user.streaming_fps);
3392 const timestamps = [];3408 const timestamps = [];
3393 for await (const { text, swipes, logprobs, toolCalls, state } of this.generator()) {3409 for await (const { text, swipes, logprobs, toolCalls, state } of this.generator()) {
3394 timestamps.push(Date.now());3410 const now = Date.now();
3411 timestamps.push(now);
3412 if (!this.timeToFirstToken) {
3413 this.timeToFirstToken = now - this.createdAt.getTime();
3414 }
3395 if (this.isStopped || this.abortController.signal.aborted) {3415 if (this.isStopped || this.abortController.signal.aborted) {
3396 return this.result;3416 return this.result;
3397 }3417 }
@@ -8830,7 +8850,12 @@ const swipe_right = () => {
8830 chat[chat.length - 1]['swipes'] = []; // empty the array8850 chat[chat.length - 1]['swipes'] = []; // empty the array
8831 chat[chat.length - 1]['swipe_info'] = [];8851 chat[chat.length - 1]['swipe_info'] = [];
8832 chat[chat.length - 1]['swipes'][0] = chat[chat.length - 1]['mes']; //assign swipe array with last message from chat8852 chat[chat.length - 1]['swipes'][0] = chat[chat.length - 1]['mes']; //assign swipe array with last message from chat
8833 chat[chat.length - 1]['swipe_info'][0] = { 'send_date': chat[chat.length - 1]['send_date'], 'gen_started': chat[chat.length - 1]['gen_started'], 'gen_finished': chat[chat.length - 1]['gen_finished'], 'extra': JSON.parse(JSON.stringify(chat[chat.length - 1]['extra'])) };8853 chat[chat.length - 1]['swipe_info'][0] = {
8854 'send_date': chat[chat.length - 1]['send_date'],
8855 'gen_started': chat[chat.length - 1]['gen_started'],
8856 'gen_finished': chat[chat.length - 1]['gen_finished'],
8857 'extra': JSON.parse(JSON.stringify(chat[chat.length - 1]['extra'])),
8858 };
8834 //assign swipe info array with last message from chat8859 //assign swipe info array with last message from chat
8835 }8860 }
8836 if (chat.length === 1 && chat[0]['swipe_id'] !== undefined && chat[0]['swipe_id'] === chat[0]['swipes'].length - 1) { // if swipe_right is called on the last alternate greeting, loop back around8861 if (chat.length === 1 && chat[0]['swipe_id'] !== undefined && chat[0]['swipe_id'] === chat[0]['swipes'].length - 1) { // if swipe_right is called on the last alternate greeting, loop back around