Data Bank: display file ingestion progress

aaed09f6063e47543da807fcc20ea8d256f1d4b5

Cohee <18619528+Cohee1207@users.noreply.github.com>

public/scripts/extensions/vectors/index.js+13 -3 Viewed

 export const EXTENSION_PROMPT_TAG = '3_vectors';
 export const EXTENSION_PROMPT_TAG_DB = '4_vectors_data_bank';
 +// Force solo chunks for sources that don't support batching.
 +const getBatchSize = () => ['transformers', 'palm', 'ollama'].includes(settings.source) ? 1 : 5;
++
 const settings = {
     // For both
     source: 'transformers',

         // upon request of a full vectorise
         cachedSummaries.clear();
         const batchSize = 5getBatchSize();
         const elapsedLog = [];
         let finished = false;
         $('#vectorize_progress').show();

             fileText = translatedText;
+        }
 -        const toast = toastr.info('Vectorization may take some time, please wait...', `Ingesting file ${fileName}`);
 +        const batchSize = getBatchSize();
 +        const toastBody = $('<span>').text('This may take a while. Please wait...');
 +        const toast = toastr.info(toastBody, `Ingesting file ${fileName}`, { closeButton: false, escapeHtml: false, timeOut: 0, extendedTimeOut: 0 });
         const overlapSize = Math.round(chunkSize * overlapPercent / 100);
         const delimiters = getChunkDelimiters();
         // Overlap should not be included in chunk size. It will be later compensated by overlapChunks

         console.debug(`Vectors: Split file ${fileName} into ${chunks.length} chunks with ${overlapPercent}% overlap`, chunks);
         const items = chunks.map((chunk, index) => ({ hash: getStringHash(chunk), text: chunk, index: index }));
 -        await insertVectorItems(collectionId, items);
++
 +        for (let i = 0; i < items.length; i += batchSize) {
 +            toastBody.text(`${i}/${items.length} (${Math.round((i / items.length) * 100)}%) chunks processed`);
 +            const chunkedBatch = items.slice(i, i + batchSize);
 +            await insertVectorItems(collectionId, chunkedBatch);
 +        }
         toastr.clear(toast);
         console.log(`Vectors: Inserted ${chunks.length} vector items for file ${fileName} into ${collectionId}`);