Use common methods for extras summary

5b3388d0c81091faea8b103d7f73ad0fd01e36e8

Cohee <18619528+Cohee1207@users.noreply.github.com>

public/scripts/extensions/memory/index.js+4 -4 Viewed

@@ -65,7 +65,7 @@ async function getSourceContextSize() {
65	}	65	}
66		66
67	if (extension_settings.source === summary_sources.extras) {	67	if (extension_settings.source === summary_sources.extras) {
68	return 1024;	68	return 1024 - 64;
69	}	69	}
70		70
71	return getMaxContextSize(overrideLength);	71	return getMaxContextSize(overrideLength);

     const reversedChat = chat.slice().reverse();
     reversedChat.shift();
     const memoryBuffer = [];
-    const CONTEXT_SIZE = 1024 - 64;
+    const CONTEXT_SIZE = await getSourceContextSize();
     for (const message of reversedChat) {
         // we reached the point of latest memory

         memoryBuffer.push(entry);
         // check if token limit was reached
-        const tokens = getTextTokens(tokenizers.GPT2, getMemoryString()).length;
+        const tokens = await countSourceTokens(getMemoryString());
         if (tokens >= CONTEXT_SIZE) {
             break;
+        }
+    }
     const resultingString = getMemoryString();
-    const resultingTokens = getTextTokens(tokenizers.GPT2, resultingString).length;
+    const resultingTokens = await countSourceTokens(resultingString);
     if (!resultingString || resultingTokens < CONTEXT_SIZE) {
         console.debug('Not enough context to summarize');