Merge pull request #3610 from bmen25124/time_to_first_token Added time to first token

c36607be6f72074030c72e9e22eb7e451e549f06

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: 3b9fcfae60ddd54612801a9065538d28ac49ee1a fb5f3e0f97602682c4aec498a318b4c589f30a5f
1 files changed, +33 -8Showing whitespace changes
public/script.js+33 -8
@@ -2477,7 +2477,7 @@ export function addOneMessage(mes, { type = 'normal', insertAfter = null, scroll
24772477 timestamp: timestamp,
24782478 extra: mes.extra,
24792479 tokenCount: mes.extra?.token_count ?? 0,
24802480 ...formatGenerationTimer(mes.gen_started, mes.gen_finished, mes.extra?.token_count, mes.extra?.reasoning_duration, mes.extra?.time_to_first_token),
24812481 };
24822482
24832483 const renderedMessage = getMessageFromTemplate(params);
@@ -2598,13 +2598,14 @@ export function formatCharacterAvatar(characterAvatar) {
25982598 * @param {Date} gen_finished Date when generation was finished
25992599 * @param {number} tokenCount Number of tokens generated (0 if not available)
26002600 * @param {number?} [reasoningDuration=null] Reasoning duration (null if no reasoning was done)
2601+ * @param {number?} [timeToFirstToken=null] Time to first token
26012602 * @returns {Object} Object containing the formatted timer value and title
26022603 * @example
26032604 * const { timerValue, timerTitle } = formatGenerationTimer(gen_started, gen_finished, tokenCount);
26042605 * console.log(timerValue); // 1.2s
26052606 * console.log(timerTitle); // Generation queued: 12:34:56 7 Jan 2021\nReply received: 12:34:57 7 Jan 2021\nTime to generate: 1.2 seconds\nToken rate: 5 t/s
26062607 */
26072608function formatGenerationTimer(gen_started, gen_finished, tokenCount, reasoningDuration = null, timeToFirstToken = null) {
26082609 if (!gen_started || !gen_finished) {
26092610 return {};
26102611 }
@@ -2618,8 +2619,9 @@ function formatGenerationTimer(gen_started, gen_finished, tokenCount, reasoningD
26182619 `Generation queued: ${start.format(dateFormat)}`,
26192620 `Reply received: ${finish.format(dateFormat)}`,
26202621 `Time to generate: ${seconds} seconds`,
2622+ timeToFirstToken ? `Time to first token: ${timeToFirstToken / 1000} seconds` : '',
26212623 reasoningDuration > 0 ? `Time to think: ${reasoningDuration / 1000} seconds` : '',
26222624 tokenCount > 0 ? `Token rate: ${Number(tokenCount / seconds).toFixed(13)} t/s` : '',
26232625 ].filter(x => x).join('\n').trim();
26242626
26252627 if (isNaN(seconds) || seconds < 0) {
@@ -3158,6 +3160,9 @@ class StreamingProcessor {
31583160 this.abortController = new AbortController();
31593161 this.firstMessageText = '...';
31603162 this.timeStarted = timeStarted;
3163+ /** @type {number?} */
3164+ this.timeToFirstToken = null;
3165+ this.createdAt = new Date();
31613166 this.continueMessage = type === 'continue' ? continueMessage : '';
31623167 this.swipes = [];
31633168 /** @type {import('./scripts/logprobs.js').TokenLogprobs[]} */
@@ -3249,6 +3254,7 @@ class StreamingProcessor {
32493254 if (!chat[messageId]['extra']) {
32503255 chat[messageId]['extra'] = {};
32513256 }
3257+ chat[messageId]['extra']['time_to_first_token'] = this.timeToFirstToken;
32523258
32533259 // Update reasoning
32543260 await this.reasoningHandler.process(messageId, mesChanged);
@@ -3266,7 +3272,12 @@ class StreamingProcessor {
32663272
32673273 if ((this.type == 'swipe' || this.type === 'continue') && Array.isArray(chat[messageId]['swipes'])) {
32683274 chat[messageId]['swipes'][chat[messageId]['swipe_id']] = processedText;
3269- chat[messageId]['swipe_info'][chat[messageId]['swipe_id']] = { 'send_date': chat[messageId]['send_date'], 'gen_started': chat[messageId]['gen_started'], 'gen_finished': chat[messageId]['gen_finished'], 'extra': JSON.parse(JSON.stringify(chat[messageId]['extra'])) };
3275+ chat[messageId]['swipe_info'][chat[messageId]['swipe_id']] = {
3276+ 'send_date': chat[messageId]['send_date'],
3277+ 'gen_started': chat[messageId]['gen_started'],
3278+ 'gen_finished': chat[messageId]['gen_finished'],
3279+ 'extra': JSON.parse(JSON.stringify(chat[messageId]['extra']))
3280+ };
32703281 }
32713282
32723283 const formattedText = messageFormatting(
@@ -3282,7 +3293,7 @@ class StreamingProcessor {
32823293 this.messageTextDom.innerHTML = formattedText;
32833294 }
32843295
32853296 const timePassed = formatGenerationTimer(this.timeStarted, currentTime, currentTokenCount, this.reasoningHandler.getDuration(), this.timeToFirstToken);
32863297 if (this.messageTimerDom instanceof HTMLElement) {
32873298 this.messageTimerDom.textContent = timePassed.timerValue;
32883299 this.messageTimerDom.title = timePassed.timerTitle;
@@ -3357,7 +3368,12 @@ class StreamingProcessor {
33573368 if (this.type !== 'swipe' && this.type !== 'impersonate') {
33583369 if (Array.isArray(chat[messageId]['swipes']) && chat[messageId]['swipes'].length === 1 && chat[messageId]['swipe_id'] === 0) {
33593370 chat[messageId]['swipes'][0] = chat[messageId]['mes'];
3360- chat[messageId]['swipe_info'][0] = { 'send_date': chat[messageId]['send_date'], 'gen_started': chat[messageId]['gen_started'], 'gen_finished': chat[messageId]['gen_finished'], 'extra': JSON.parse(JSON.stringify(chat[messageId]['extra'])) };
3371+ chat[messageId]['swipe_info'][0] = {
3372+ 'send_date': chat[messageId]['send_date'],
3373+ 'gen_started': chat[messageId]['gen_started'],
3374+ 'gen_finished': chat[messageId]['gen_finished'],
3375+ 'extra': JSON.parse(JSON.stringify(chat[messageId]['extra'])),
3376+ };
33613377 }
33623378 }
33633379 }
@@ -3391,7 +3407,11 @@ class StreamingProcessor {
33913407 const sw = new Stopwatch(1000 / power_user.streaming_fps);
33923408 const timestamps = [];
33933409 for await (const { text, swipes, logprobs, toolCalls, state } of this.generator()) {
33943410 timestamps.push(const now = Date.now());
3411+ timestamps.push(now);
3412+ if (!this.timeToFirstToken) {
3413+ this.timeToFirstToken = now - this.createdAt.getTime();
3414+ }
33953415 if (this.isStopped || this.abortController.signal.aborted) {
33963416 return this.result;
33973417 }
@@ -8830,7 +8850,12 @@ const swipe_right = () => {
88308850 chat[chat.length - 1]['swipes'] = []; // empty the array
88318851 chat[chat.length - 1]['swipe_info'] = [];
88328852 chat[chat.length - 1]['swipes'][0] = chat[chat.length - 1]['mes']; //assign swipe array with last message from chat
8833- chat[chat.length - 1]['swipe_info'][0] = { 'send_date': chat[chat.length - 1]['send_date'], 'gen_started': chat[chat.length - 1]['gen_started'], 'gen_finished': chat[chat.length - 1]['gen_finished'], 'extra': JSON.parse(JSON.stringify(chat[chat.length - 1]['extra'])) };
8853+ chat[chat.length - 1]['swipe_info'][0] = {
8854+ 'send_date': chat[chat.length - 1]['send_date'],
8855+ 'gen_started': chat[chat.length - 1]['gen_started'],
8856+ 'gen_finished': chat[chat.length - 1]['gen_finished'],
8857+ 'extra': JSON.parse(JSON.stringify(chat[chat.length - 1]['extra'])),
8858+ };
88348859 //assign swipe info array with last message from chat
88358860 }
88368861 if (chat.length === 1 && chat[0]['swipe_id'] !== undefined && chat[0]['swipe_id'] === chat[0]['swipes'].length - 1) { // if swipe_right is called on the last alternate greeting, loop back around