Vectors: Allow chunk overlap when forced chunking on delimiter

441dffc0da43f016ea702ae6a4e0b85b57c698f2

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: cb8fd4f66390344eda01cee7cffadbfd0b4b92b9
1 files changed, +3 -2Ignore whitespace
public/scripts/extensions/vectors/index.js+3 -2
@@ -586,9 +586,10 @@ async function vectorizeFile(fileText, fileName, collectionId, chunkSize, overla
586 const delimiters = getChunkDelimiters();586 const delimiters = getChunkDelimiters();
587 // Overlap should not be included in chunk size. It will be later compensated by overlapChunks587 // Overlap should not be included in chunk size. It will be later compensated by overlapChunks
588 chunkSize = overlapSize > 0 ? (chunkSize - overlapSize) : chunkSize;588 chunkSize = overlapSize > 0 ? (chunkSize - overlapSize) : chunkSize;
589 const applyOverlap = (x, y, z) => overlapSize > 0 ? overlapChunks(x, y, z, overlapSize) : x;
589 const chunks = settings.only_custom_boundary && settings.force_chunk_delimiter590 const chunks = settings.only_custom_boundary && settings.force_chunk_delimiter
590 ? fileText.split(settings.force_chunk_delimiter)591 ? fileText.split(settings.force_chunk_delimiter).map(applyOverlap)
591 : splitRecursive(fileText, chunkSize, delimiters).map((x, y, z) => overlapSize > 0 ? overlapChunks(x, y, z, overlapSize) : x);592 : splitRecursive(fileText, chunkSize, delimiters).map(applyOverlap);
592 console.debug(`Vectors: Split file ${fileName} into ${chunks.length} chunks with ${overlapPercent}% overlap`, chunks);593 console.debug(`Vectors: Split file ${fileName} into ${chunks.length} chunks with ${overlapPercent}% overlap`, chunks);
593594
594 const items = chunks.map((chunk, index) => ({ hash: getStringHash(chunk), text: chunk, index: index }));595 const items = chunks.map((chunk, index) => ({ hash: getStringHash(chunk), text: chunk, index: index }));