Cull the usage of MAG in streaming

38f0929e3d7d591171324108ee8227f269e0726a

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: c84a685e7b3bb63e5b1a0acfc200ffc1095ed2e9
1 files changed, +13 -11Ignore whitespace
public/script.js+13 -11
@@ -2844,7 +2844,14 @@ function hideStopButton() {
2844}2844}
28452845
2846class StreamingProcessor {2846class StreamingProcessor {
2847 constructor(type, force_name2, timeStarted, messageAlreadyGenerated) {2847 /**
2848 * Creates a new streaming processor.
2849 * @param {string} type Generation type
2850 * @param {boolean} forceName2 If true, force the use of name2
2851 * @param {Date} timeStarted Date when generation was started
2852 * @param {string} continueMessage Previous message if the type is 'continue'
2853 */
2854 constructor(type, forceName2, timeStarted, continueMessage) {
2848 this.result = '';2855 this.result = '';
2849 this.messageId = -1;2856 this.messageId = -1;
2850 this.messageDom = null;2857 this.messageDom = null;
@@ -2854,14 +2861,14 @@ class StreamingProcessor {
2854 /** @type {HTMLTextAreaElement} */2861 /** @type {HTMLTextAreaElement} */
2855 this.sendTextarea = document.querySelector('#send_textarea');2862 this.sendTextarea = document.querySelector('#send_textarea');
2856 this.type = type;2863 this.type = type;
2857 this.force_name2 = force_name2;2864 this.force_name2 = forceName2;
2858 this.isStopped = false;2865 this.isStopped = false;
2859 this.isFinished = false;2866 this.isFinished = false;
2860 this.generator = this.nullStreamingGeneration;2867 this.generator = this.nullStreamingGeneration;
2861 this.abortController = new AbortController();2868 this.abortController = new AbortController();
2862 this.firstMessageText = '...';2869 this.firstMessageText = '...';
2863 this.timeStarted = timeStarted;2870 this.timeStarted = timeStarted;
2864 this.messageAlreadyGenerated = messageAlreadyGenerated;2871 this.continueMessage = type === 'continue' ? continueMessage : '';
2865 this.swipes = [];2872 this.swipes = [];
2866 /** @type {import('./scripts/logprobs.js').TokenLogprobs[]} */2873 /** @type {import('./scripts/logprobs.js').TokenLogprobs[]} */
2867 this.messageLogprobs = [];2874 this.messageLogprobs = [];
@@ -3014,8 +3021,7 @@ class StreamingProcessor {
3014 await eventSource.emit(event_types.IMPERSONATE_READY, text);3021 await eventSource.emit(event_types.IMPERSONATE_READY, text);
3015 }3022 }
30163023
3017 const continueMsg = this.type === 'continue' ? this.messageAlreadyGenerated : undefined;3024 saveLogprobsForActiveMessage(this.messageLogprobs.filter(Boolean), this.continueMessage);
3018 saveLogprobsForActiveMessage(this.messageLogprobs.filter(Boolean), continueMsg);
3019 await saveChatConditional();3025 await saveChatConditional();
3020 unblockGeneration();3026 unblockGeneration();
3021 generatedPromptCache = '';3027 generatedPromptCache = '';
@@ -3111,7 +3117,7 @@ class StreamingProcessor {
3111 if (logprobs) {3117 if (logprobs) {
3112 this.messageLogprobs.push(...(Array.isArray(logprobs) ? logprobs : [logprobs]));3118 this.messageLogprobs.push(...(Array.isArray(logprobs) ? logprobs : [logprobs]));
3113 }3119 }
3114 await sw.tick(() => this.onProgressStreaming(this.messageId, this.messageAlreadyGenerated + text));3120 await sw.tick(() => this.onProgressStreaming(this.messageId, this.continueMessage + text));
3115 }3121 }
3116 const seconds = (timestamps[timestamps.length - 1] - timestamps[0]) / 1000;3122 const seconds = (timestamps[timestamps.length - 1] - timestamps[0]) / 1000;
3117 console.warn(`Stream stats: ${timestamps.length} tokens, ${seconds.toFixed(2)} seconds, rate: ${Number(timestamps.length / seconds).toFixed(2)} TPS`);3123 console.warn(`Stream stats: ${timestamps.length} tokens, ${seconds.toFixed(2)} seconds, rate: ${Number(timestamps.length / seconds).toFixed(2)} TPS`);
@@ -3304,8 +3310,6 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
3304 const isInstruct = power_user.instruct.enabled && main_api !== 'openai';3310 const isInstruct = power_user.instruct.enabled && main_api !== 'openai';
3305 const isImpersonate = type == 'impersonate';3311 const isImpersonate = type == 'impersonate';
33063312
3307 let message_already_generated = isImpersonate ? `${name1}: ` : `${name2}: `;
3308
3309 if (!(dryRun || type == 'regenerate' || type == 'swipe' || type == 'quiet')) {3313 if (!(dryRun || type == 'regenerate' || type == 'swipe' || type == 'quiet')) {
3310 const interruptedByCommand = await processCommands(String($('#send_textarea').val()));3314 const interruptedByCommand = await processCommands(String($('#send_textarea').val()));
33113315
@@ -3744,7 +3748,6 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
3744 let oaiMessageExamples = [];3748 let oaiMessageExamples = [];
37453749
3746 if (main_api === 'openai') {3750 if (main_api === 'openai') {
3747 message_already_generated = '';
3748 oaiMessages = setOpenAIMessages(coreChat);3751 oaiMessages = setOpenAIMessages(coreChat);
3749 oaiMessageExamples = setOpenAIMessageExamples(mesExamplesArray);3752 oaiMessageExamples = setOpenAIMessageExamples(mesExamplesArray);
3750 }3753 }
@@ -3887,7 +3890,6 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
3887 cyclePrompt += oai_settings.continue_postfix;3890 cyclePrompt += oai_settings.continue_postfix;
3888 continue_mag += oai_settings.continue_postfix;3891 continue_mag += oai_settings.continue_postfix;
3889 }3892 }
3890 message_already_generated = continue_mag;
3891 }3893 }
38923894
3893 const originalType = type;3895 const originalType = type;
@@ -4314,7 +4316,7 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
4314 console.debug(`pushed prompt bits to itemizedPrompts array. Length is now: ${itemizedPrompts.length}`);4316 console.debug(`pushed prompt bits to itemizedPrompts array. Length is now: ${itemizedPrompts.length}`);
43154317
4316 if (isStreamingEnabled() && type !== 'quiet') {4318 if (isStreamingEnabled() && type !== 'quiet') {
4317 streamingProcessor = new StreamingProcessor(type, force_name2, generation_started, message_already_generated);4319 streamingProcessor = new StreamingProcessor(type, force_name2, generation_started, continue_mag);
4318 if (isContinue) {4320 if (isContinue) {
4319 // Save reply does add cycle text to the prompt, so it's not needed here4321 // Save reply does add cycle text to the prompt, so it's not needed here
4320 streamingProcessor.firstMessageText = '';4322 streamingProcessor.firstMessageText = '';