Vectors: Add only custom boundary chunking

77bee453abe1b325e96700684def04832f6948bd

Cohee <18619528+Cohee1207@users.noreply.github.com>

public/scripts/extensions/vectors/index.js+10 -1 Viewed

     chunk_size: 5000,
     chunk_count: 2,
     overlap_percent: 0,
 +    only_custom_boundary: false,
     // For Data Bank
     size_threshold_db: 5,

         const delimiters = getChunkDelimiters();
         // Overlap should not be included in chunk size. It will be later compensated by overlapChunks
         chunkSize = overlapSize > 0 ? (chunkSize - overlapSize) : chunkSize;
 -        const chunks = splitRecursive(fileText, chunkSize, delimiters).map((x, y, z) => overlapSize > 0 ? overlapChunks(x, y, z, overlapSize) : x);
 +        const chunks = settings.only_custom_boundary && settings.force_chunk_delimiter
 +            ? fileText.split(settings.force_chunk_delimiter)
 +            : splitRecursive(fileText, chunkSize, delimiters).map((x, y, z) => overlapSize > 0 ? overlapChunks(x, y, z, overlapSize) : x);
         console.debug(`Vectors: Split file ${fileName} into ${chunks.length} chunks with ${overlapPercent}% overlap`, chunks);
         const items = chunks.map((chunk, index) => ({ hash: getStringHash(chunk), text: chunk, index: index }));

         saveSettingsDebounced();
     });
 +    $('#vectors_only_custom_boundary').prop('checked', settings.only_custom_boundary).on('input', () => {
 +        settings.only_custom_boundary = !!$('#vectors_only_custom_boundary').prop('checked');
 +        Object.assign(extension_settings.vectors, settings);
 +        saveSettingsDebounced();
 +    });
++
     $('#vectors_ollama_pull').on('click', (e) => {
         const presetModel = extension_settings.vectors.ollama_model || '';
         e.preventDefault();

public/scripts/extensions/vectors/settings.html+6 -0 Viewed

             </label>
             <div id="vectors_files_settings" class="marginTopBot5">
 +                <label class="checkbox_label" for="vectors_only_custom_boundary" title="Create a chunk for every custom Chunk boundary detected in the file text. Don't chunk on default boundaries.">
 +                    <input id="vectors_only_custom_boundary" type="checkbox" class="checkbox">
 +                    <span data-i18n="Only chunk on custom boundary">
 +                        Only chunk on custom boundary
 +                    </span>
 +                </label>
                 <label class="checkbox_label" for="vectors_translate_files" title="This can help with retrieval accuracy if using embedding models that are trained on English data. Uses the selected API from Chat Translation extension settings.">
                     <input id="vectors_translate_files" type="checkbox" class="checkbox">
                     <span data-i18n="Translate files into English before processing">