Merge pull request #2988 from QuantumEntangledAndy/feat/cachedVectorSummaries Add client side cacheing of vector summaries

e01a243ce5ee0abd6479d17a2b6e2f58ebf781f5

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

Parents: b6af986e582f9e635e904d7e49431afd18f7cd44 69283f511f518eecfdda192345908e1fd74b4115

public/scripts/extensions/vectors/index.js+84 -73 Viewed

 /**
  * @typedef {object} HashedMessage
  * @property {string} text - The hashed message text
 + * @property {number} hash - The hash used as the vector key
  */
 const MODULE_NAME = 'vectors';

 const moduleWorker = new ModuleWorkerWrapper(synchronizeChat);
 +const cachedSummaries = new Map();
++
 /**
  * Gets the Collection ID for a file embedded in the chat.
  * @param {string} fileUrl URL of the file

             return;
+        }
 +        // Clear all cached summaries to ensure that new ones are created
 +        // upon request of a full vectorise
 +        cachedSummaries.clear();
++
         const batchSize = 5;
         const elapsedLog = [];
         let finished = false;

 /**
  * Summarizes messages using the Extras API method.
  * @param {HashedMessage[]} hashedMessages Array ofelement hashed messagesmessage
  * @returns {Promise<HashedMessage[]boolean>} Summarized messagesSucess
  */
 async function summarizeExtra(hashedMessageselement) {
 -    for (const element of hashedMessages) {
 +    try {
 -        try {
 +        const url = new URL(getApiUrl());
 -            const url = new URL(getApiUrl());
 +        url.pathname = '/api/summarize';
 -            url.pathname = '/api/summarize';
+-
 -            const apiResult = await doExtrasFetch(url, {
 -                method: 'POST',
 -                headers: {
 -                    'Content-Type': 'application/json',
 -                    'Bypass-Tunnel-Reminder': 'bypass',
 -                },
 -                body: JSON.stringify({
 -                    text: element.text,
 -                    params: {},
 -                }),
 -            });
 -            if (apiResult.ok) {
 +        const apiResult = await doExtrasFetch(url, {
 -                const data = await apiResult.json();
 +            method: 'POST',
 -                element.text = data.summary;
 +            headers: {
 -            }
 +                'Content-Type': 'application/json',
 -        }
 +                'Bypass-Tunnel-Reminder': 'bypass',
 -        catch (error) {
 +            },
 -            console.log(error);
 +            body: JSON.stringify({
 +                text: element.text,
 +                params: {},
 +            }),
 +        });
++
 +        if (apiResult.ok) {
 +            const data = await apiResult.json();
 +            element.text = data.summary;
+        }
+    }
 +    catch (error) {
 +        console.log(error);
 +        return false;
 +    }
     return hashedMessagestrue;
+}
 /**
  * Summarizes messages using the main API method.
  * @param {HashedMessage[]} hashedMessages Array ofelement hashed messagesmessage
  * @returns {Promise<HashedMessage[]boolean>} Summarized messagesSucess
  */
 async function summarizeMain(hashedMessageselement) {
 -    for (const element of hashedMessages) {
 +    element.text = await generateRaw(element.text, '', false, false, settings.summary_prompt);
 -        element.text = await generateRaw(element.text, '', false, false, settings.summary_prompt);
 +    return true;
 -    }
+-
 -    return hashedMessages;
+}
 /**
  * Summarizes messages using WebLLM.
  * @param {HashedMessage[]} hashedMessages Array ofelement hashed messagesmessage
  * @returns {Promise<HashedMessage[]boolean>} Summarized messagesSucess
  */
 async function summarizeWebLLM(hashedMessageselement) {
     if (!isWebLlmSupported()) {
         console.warn('Vectors: WebLLM is not supported');
         return hashedMessagesfalse;
+    }
 -    for (const element of hashedMessages) {
 +    const messages = [{ role: 'system', content: settings.summary_prompt }, { role: 'user', content: element.text }];
 -        const messages = [{ role:'system', content: settings.summary_prompt }, { role:'user', content: element.text }];
 +    element.text = await generateWebLlmChatPrompt(messages);
 -        element.text = await generateWebLlmChatPrompt(messages);
 -    }
     return hashedMessagestrue;
+}
 /**

  * @returns {Promise<HashedMessage[]>} Summarized messages
  */
 async function summarize(hashedMessages, endpoint = 'main') {
     switchfor (endpointconst element of hashedMessages) {
 -        case 'main':
 +        const cachedSummary = cachedSummaries.get(element.hash);
 -            return await summarizeMain(hashedMessages);
 +        if (!cachedSummary) {
 -        case 'extras':
 +            let success = true;
 -            return await summarizeExtra(hashedMessages);
 +            switch (endpoint) {
                         case 'webllmmain':
             return                    success = await summarizeWebLLMsummarizeMain(hashedMessageselement);
 -        default:
 +                    break;
 -            console.error('Unsupported endpoint', endpoint);
 +                case 'extras':
 +                    success = await summarizeExtra(element);
 +                    break;
 +                case 'webllm':
 +                    success = await summarizeWebLLM(element);
 +                    break;
 +                default:
 +                    console.error('Unsupported endpoint', endpoint);
 +                    success = false;
 +                    break;
 +            }
 +            if (success) {
 +                cachedSummaries.set(element.hash, element.text);
 +            } else {
 +                break;
 +            }
 +        } else {
 +            element.text = cachedSummary;
 +        }
+    }
 +    return hashedMessages;
+}
 async function synchronizeChat(batchSize = 5) {

             return -1;
+        }
         letconst hashedMessages = context.chat.filter(x => !x.is_system).map(x => ({ text: String(substituteParams(x.mes)), hash: getStringHash(substituteParams(x.mes)), index: context.chat.indexOf(x) }));
         const hashesInCollection = await getSavedHashes(chatId);
 -        if (settings.summarize) {
 +        let newVectorItems = hashedMessages.filter(x => !hashesInCollection.includes(x.hash));
 -            hashedMessages = await summarize(hashedMessages, settings.summary_source);
 -        }
+-
 -        const newVectorItems = hashedMessages.filter(x => !hashesInCollection.includes(x.hash));
         const deletedHashes = hashesInCollection.filter(x => !hashedMessages.some(y => y.hash === x));
 +        if (settings.summarize) {
 +            newVectorItems = await summarize(newVectorItems, settings.summary_source);
 +        }
         if (newVectorItems.length > 0) {
             const chunkedBatch = splitByChunks(newVectorItems.slice(0, batchSize));

  * @returns {Promise<string>} Text to query
  */
 async function getQueryText(chat, initiator) {
     let queryTexthashedMessages = '';chat
 -    let i = 0;
 +        .map(x => ({ text: String(substituteParams(x.mes)), hash: getStringHash(substituteParams(x.mes)) }))
+-
 +        .filter(x => x.text)
 -    let hashedMessages = chat.map(x => ({ text: String(substituteParams(x.mes)) }));
 +        .reverse()
 +        .slice(0, settings.query);
     if (initiator === 'chat' && settings.enabled_chats && settings.summarize && settings.summarize_sent) {
         hashedMessages = await summarize(hashedMessages, settings.summary_source);
+    }
 -    for (const message of hashedMessages.slice().reverse()) {
 +    const queryText = hashedMessages.map(x => x.text).join('\n');
 -        if (message.text) {
 -            queryText += message.text + '\n';
 -            i++;
 -        }
+-
 -        if (i === settings.query) {
 -            break;
 -        }
 -    }
     return collapseNewlines(queryText).trim();
+}