Merge pull request #2479 from SillyTavern/wi-processing-refactoring WI scan refactoring: Extended logging, updated code flow, changed Min Activations

9268fc3ef2909f4359938d316e2617576be96083

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
1 files changed, +264 -161Ignore whitespace
public/scripts/world-info.js+264 -161
@@ -169,7 +169,7 @@ class WorldInfoBuffer {
169169 #skew = 0;
170170
171171 /**
172- * @type {number} The starting depth of the global scan depth. Incremented by "min activations" feature to not repeat scans. When > 0 it means a complete scan was done up to #startDepth already, and `advanceScanPosition` was called.
172+ * @type {number} The starting depth of the global scan depth.
173173 */
174174 #startDepth = 0;
175175
@@ -222,12 +222,12 @@ class WorldInfoBuffer {
222222 }
223223
224224 if (depth < 0) {
225225 console.error(`[WI] Invalid WI scan depth ${depth}. Must be >= 0`);
226226 return '';
227227 }
228228
229229 if (depth > MAX_SCAN_DEPTH) {
230230 console.warn(`[WI] Invalid WI scan depth ${depth}. Truncating to ${MAX_SCAN_DEPTH}`);
231231 depth = MAX_SCAN_DEPTH;
232232 }
233233
@@ -301,10 +301,17 @@ class WorldInfoBuffer {
301301 }
302302
303303 /**
304304 * IncrementsChecks skewif andthe setsrecursion startDepthbuffer tois previousnot depthempty.
305+ * @returns {boolean} Returns true if the recursion buffer is not empty, otherwise false
305306 */
306307 advanceScanPositionhasRecurse() {
307308 return this.#startDepthrecurseBuffer.length => this.getDepth()0;
309+ }
310+
311+ /**
312+ * Increments skew to advance the scan range.
313+ */
314+ advanceScan() {
308315 this.#skew++;
309316 }
310317
@@ -436,7 +443,7 @@ class WorldInfoTimedEffects {
436443 const key = this.#getEntryKey(entry);
437444 const effect = this.#getEntryTimedEffect('cooldown', entry, true);
438445 chat_metadata.timedWorldInfo.cooldown[key] = effect;
439446 console.log(`[WI] Adding cooldown entry ${key} on ended sticky: start=${effect.start}, end=${effect.end}, protected=${effect.protected}`);
440447 // Set the cooldown immediately for this evaluation
441448 this.#buffer.cooldown.push(entry);
442449 },
@@ -447,10 +454,10 @@ class WorldInfoTimedEffects {
447454 * @param {WIScanEntry} entry Entry that ended cooldown
448455 */
449456 'cooldown': (entry) => {
450457 console.debug('[WI] Cooldown ended for entry', entry.uid);
451458 },
452459
453460 'delay': () => { },
454461 };
455462
456463 /**
@@ -537,11 +544,11 @@ class WorldInfoTimedEffects {
537544 /** @type {[string, WITimedEffect][]} */
538545 const effects = Object.entries(chat_metadata.timedWorldInfo[type]);
539546 for (const [key, value] of effects) {
540547 console.log(`[WI] Processing ${type} entry ${key}`, value);
541548 const entry = this.#entries.find(x => String(this.#getEntryHash(x)) === String(value.hash));
542549
543550 if (this.#chat.length <= Number(value.start) && !value.protected) {
544551 console.log(`[WI] Removing ${type} entry ${key} from timedWorldInfo: chat not advanced`, value);
545552 delete chat_metadata.timedWorldInfo[type][key];
546553 continue;
547554 }
@@ -549,7 +556,7 @@ class WorldInfoTimedEffects {
549556 // Missing entries (they could be from another character's lorebook)
550557 if (!entry) {
551558 if (this.#chat.length >= Number(value.end)) {
552559 console.log(`[WI] Removing ${type} entry from timedWorldInfo: entry not found and interval passed`, entry);
553560 delete chat_metadata.timedWorldInfo[type][key];
554561 }
555562 continue;
@@ -557,13 +564,13 @@ class WorldInfoTimedEffects {
557564
558565 // Ignore invalid entries (not configured for timed effects)
559566 if (!entry[type]) {
560567 console.log(`[WI] Removing ${type} entry from timedWorldInfo: entry not ${type}`, entry);
561568 delete chat_metadata.timedWorldInfo[type][key];
562569 continue;
563570 }
564571
565572 if (this.#chat.length >= Number(value.end)) {
566573 console.log(`[WI] Removing ${type} entry from timedWorldInfo: ${type} interval passed`, entry);
567574 delete chat_metadata.timedWorldInfo[type][key];
568575 if (typeof onEnded === 'function') {
569576 onEnded(entry);
@@ -572,7 +579,7 @@ class WorldInfoTimedEffects {
572579 }
573580
574581 buffer.push(entry);
575582 console.log(`[WI] Timed effect "${type}" applied to entry`, entry);
576583 }
577584 }
578585
@@ -588,7 +595,7 @@ class WorldInfoTimedEffects {
588595
589596 if (this.#chat.length < entry.delay) {
590597 buffer.push(entry);
591598 console.log('[WI] Timed effect "delay" applied to entry', entry);
592599 }
593600 }
594601
@@ -635,7 +642,7 @@ class WorldInfoTimedEffects {
635642 const effect = this.#getEntryTimedEffect(type, entry, false);
636643 chat_metadata.timedWorldInfo[type][key] = effect;
637644
638645 console.log(`[WI] Adding ${type} entry ${key}: start=${effect.start}, end=${effect.end}, protected=${effect.protected}`);
639646 }
640647 }
641648
@@ -667,7 +674,7 @@ class WorldInfoTimedEffects {
667674 if (newState) {
668675 const effect = this.#getEntryTimedEffect(type, entry, false);
669676 chat_metadata.timedWorldInfo[type][key] = effect;
670677 console.log(`[WI] Adding ${type} entry ${key}: start=${effect.start}, end=${effect.end}, protected=${effect.protected}`);
671678 }
672679 }
673680
@@ -3419,13 +3426,12 @@ async function createNewWorldInfo(worldName, { interactive = false } = {}) {
34193426async function getCharacterLore() {
34203427 const character = characters[this_chid];
34213428 const name = character?.name;
3429+ /** @type {Set<string>} */
34223430 let worldsToSearch = new Set();
34233431
34243432 const baseWorldName = character?.data?.extensions?.world;
34253433 if (baseWorldName) {
34263434 worldsToSearch.add(baseWorldName);
3427- } else {
3428- console.debug(`Character ${name}'s base world could not be found or is empty! Skipping...`);
34293435 }
34303436
34313437 // TODO: Maybe make the utility function not use the window context?
@@ -3435,40 +3441,48 @@ async function getCharacterLore() {
34353441 worldsToSearch = new Set([...worldsToSearch, ...extraCharLore.extraBooks]);
34363442 }
34373443
3444+ if (!worldsToSearch.size) {
3445+ return [];
3446+ }
3447+
34383448 let entries = [];
34393449 for (const worldName of worldsToSearch) {
34403450 if (selected_world_info.includes(worldName)) {
34413451 console.debug(`[WI] Character ${name}'s world ${worldName} is already activated in global world info! Skipping...`);
34423452 continue;
34433453 }
34443454
34453455 if (chat_metadata[METADATA_KEY] === worldName) {
34463456 console.debug(`[WI] Character ${name}'s world ${worldName} is already activated in chat lore! Skipping...`);
34473457 continue;
34483458 }
34493459
34503460 const data = await loadWorldInfoData(worldName);
34513461 const newEntries = data ? Object.keys(data.entries).map((x) => data.entries[x]).map(x({ uid, ...rest }) => ({ ...xuid, world: worldName, ...rest })) : [];
34523462 entries = entries.concat(newEntries);
3463+
3464+ if (!newEntries.length) {
3465+ console.debug(`[WI] Character ${name}'s world ${worldName} could not be found or is empty`);
3466+ }
34533467 }
34543468
34553469 console.debug(`[WI] Character ${name}'s lore (${Array.from(worldsToSearch)}) has ${entries.length} world info entries`, [...worldsToSearch]);
34563470 return entries;
34573471}
34583472
34593473async function getGlobalLore() {
34603474 if (!selected_world_info?.length) {
34613475 return [];
34623476 }
34633477
34643478 let entries = [];
34653479 for (const worldName of selected_world_info) {
34663480 const data = await loadWorldInfoData(worldName);
34673481 const newEntries = data ? Object.keys(data.entries).map((x) => data.entries[x]).map(x({ uid, ...rest }) => ({ ...xuid, world: worldName, ...rest })) : [];
34683482 entries = entries.concat(newEntries);
34693483 }
34703484
34713485 console.debug(`[WI] Global world info has ${entries.length} entries`, selected_world_info);
34723486
34733487 return entries;
34743488}
@@ -3481,14 +3495,14 @@ async function getChatLore() {
34813495 }
34823496
34833497 if (selected_world_info.includes(chatWorld)) {
34843498 console.debug(`[WI] Chat world ${chatWorld} is already activated in global world info! Skipping...`);
34853499 return [];
34863500 }
34873501
34883502 const data = await loadWorldInfoData(chatWorld);
34893503 const entries = data ? Object.keys(data.entries).map((x) => data.entries[x]).map(x({ uid, ...rest }) => ({ ...xuid, world: chatWorld, ...rest })) : [];
34903504
34913505 console.debug(`[WI] Chat lore has ${entries.length} entries`, [chatWorld]);
34923506
34933507 return entries;
34943508}
@@ -3512,7 +3526,7 @@ export async function getSortedEntries() {
35123526 entries = [...globalLore.sort(sortFn), ...characterLore.sort(sortFn)];
35133527 break;
35143528 default:
35153529 console.error('[WI] Unknown WI insertion strategy: ', world_info_character_strategy, 'defaulting to evenly');
35163530 entries = [...globalLore, ...characterLore].sort(sortFn);
35173531 break;
35183532 }
@@ -3520,7 +3534,7 @@ export async function getSortedEntries() {
35203534 // Chat lore always goes first
35213535 entries = [...chatLore.sort(sortFn), ...entries];
35223536
35233537 console.debug(`Sorted[WI] Found ${entries.length} world lore entries. usingSorted by strategy`, ${Object.entries(world_info_insertion_strategy).find((x) => x[1] === world_info_character_strategy}`));
35243538
35253539 // Need to deep clone the entries to avoid modifying the cached data
35263540 return structuredClone(entries);
@@ -3543,6 +3557,8 @@ async function checkWorldInfo(chat, maxContext, isDryRun) {
35433557 const context = getContext();
35443558 const buffer = new WorldInfoBuffer(chat);
35453559
3560+ console.debug(`[WI] --- START WI SCAN (on ${chat.length} messages) ---`);
3561+
35463562 // Combine the chat
35473563
35483564 // Add the depth or AN if enabled
@@ -3566,11 +3582,11 @@ async function checkWorldInfo(chat, maxContext, isDryRun) {
35663582 let budget = Math.round(world_info_budget * maxContext / 100) || 1;
35673583
35683584 if (world_info_budget_cap > 0 && budget > world_info_budget_cap) {
35693585 console.debug(`[WI] Budget ${budget} exceeds cap ${world_info_budget_cap}, using cap`);
35703586 budget = world_info_budget_cap;
35713587 }
35723588
35733589 console.debug(`[WI] Context size: ${maxContext}; WI budget: ${budget} (max% = ${world_info_budget}%, cap = ${world_info_budget_cap})`);
35743590 const sortedEntries = await getSortedEntries();
35753591 const timedEffects = new WorldInfoTimedEffects(chat, sortedEntries);
35763592
@@ -3580,21 +3596,48 @@ async function checkWorldInfo(chat, maxContext, isDryRun) {
35803596 return { worldInfoBefore: '', worldInfoAfter: '', WIDepthEntries: [], EMEntries: [], allActivatedEntries: new Set() };
35813597 }
35823598
3599+ console.debug(`[WI] --- SEARCHING ENTRIES (on ${sortedEntries.length} entries) ---`);
3600+
35833601 while (scanState) {
35843602 // Track how many times the loop has run. May be useful for debugging.
3585- // eslint-disable-next-line no-unused-vars
35863603 count++;
35873604
3605+ console.debug(`[WI] Loop #${count}. Search state`, Object.entries(scan_state).find(x => x[1] === scanState));
3606+
3607+ // Until decided otherwise, we set the loop to stop scanning after this
3608+ let nextScanState = scan_state.NONE;
3609+
3610+ // Loop and find all entries that can activate here
35883611 let activatedNow = new Set();
35893612
35903613 for (let entry of sortedEntries) {
3614+ // Logging preparation
3615+ let headerLogged = false;
3616+ function log(...args) {
3617+ if (!headerLogged) {
3618+ console.debug(`[WI] Entry ${entry.uid}`, `from '${entry.world}' processing`, entry);
3619+ headerLogged = true;
3620+ }
3621+ console.debug(`[WI] Entry ${entry.uid}`, ...args);
3622+ }
3623+
3624+ // Already processed, considered and then skipped entries should still be skipped
3625+ if (failedProbabilityChecks.has(entry) || allActivatedEntries.has(entry)) {
3626+ continue;
3627+ }
3628+
3629+ if (entry.disable == true) {
3630+ log('disabled');
3631+ continue;
3632+ }
3633+
35913634 // Check if this entry applies to the character or if it's excluded
35923635 if (entry.characterFilter && entry.characterFilter?.names?.length > 0) {
35933636 const nameIncluded = entry.characterFilter.names.includes(getCharaFilename());
35943637 const filtered = entry.characterFilter.isExclude ? nameIncluded : !nameIncluded;
35953638
35963639 if (filtered) {
35973640 console.debuglog(`WI entry ${entry.uid} 'filtered out by character`');
35983641 continue;
35993642 }
36003643 }
@@ -3611,7 +3654,7 @@ async function checkWorldInfo(chat, maxContext, isDryRun) {
36113654 const filtered = entry.characterFilter.isExclude ? includesTag : !includesTag;
36123655
36133656 if (filtered) {
36143657 console.debuglog(`WI entry ${entry.uid} 'filtered out by tag`');
36153658 continue;
36163659 }
36173660 }
@@ -3623,186 +3666,242 @@ async function checkWorldInfo(chat, maxContext, isDryRun) {
36233666 const isDelay = timedEffects.isEffectActive('delay', entry);
36243667
36253668 if (isDelay) {
3626- console.debug(`WI entry ${entry.uid} suppressed by delay`, entry);
3669+ log('suppressed by delay');
36273670 continue;
36283671 }
36293672
36303673 if (isCooldown && !isSticky) {
3631- console.debug(`WI entry ${entry.uid} suppressed by cooldown`, entry);
3674+ log('suppressed by cooldown');
36323675 continue;
36333676 }
36343677
3635- if (failedProbabilityChecks.has(entry)) {
3678+ // Only use checks for recursion flags if the scan step was activated by recursion
3679+ if (scanState !== scan_state.RECURSION && entry.delayUntilRecursion) {
3680+ log('suppressed by delay until recursion');
36363681 continue;
36373682 }
36383683
3639- if (allActivatedEntries.has(entry) || entry.disable == true) {
3684+ if (scanState === scan_state.RECURSION && world_info_recursive && entry.excludeRecursion) {
3685+ log('suppressed by exclude recursion');
36403686 continue;
36413687 }
36423688
3643- // Only use checks for recursion flags if the scan step was activated by recursion
3689+ // Now do checks for immediate activations
36443690 if (scanState !== scan_state.RECURSION && entry.delayUntilRecursionconstant) {
3645- console.debug(`WI entry ${entry.uid} suppressed by delay until recursion`, entry);
3691+ log('activated because of constant');
3692+ activatedNow.add(entry);
36463693 continue;
36473694 }
36483695
3649- if (scanState === scan_state.RECURSION && world_info_recursive && entry.excludeRecursion) {
3696+ if (buffer.isExternallyActivated(entry)) {
3650- console.debug(`WI entry ${entry.uid} suppressed by exclude recursion`, entry);
3697+ log('externally activated');
3698+ activatedNow.add(entry);
36513699 continue;
36523700 }
36533701
3654- if (entry.constant || buffer.isExternallyActivated(entry) || isSticky) {
3702+ if (isSticky) {
3703+ log('activated because active sticky');
36553704 activatedNow.add(entry);
36563705 continue;
36573706 }
36583707
36593708 if (!Array.isArray(entry.key) &&|| !entry.key.length) { //check for keywords existing
3660- // If selectiveLogic isn't found, assume it's AND, only do this once per entry
3709+ log('has no keys defined, skipped');
3661- const selectiveLogic = entry.selectiveLogic ?? 0;
3710+ continue;
3662-
3711+ }
3663- primary: for (let key of entry.key) {
3664- const substituted = substituteParams(key);
3665- const textToScan = buffer.get(entry, scanState);
3666-
3667- if (substituted && buffer.matchKeys(textToScan, substituted.trim(), entry)) {
3668- console.debug(`WI UID ${entry.uid} found by primary match: ${substituted}.`);
3669-
3670- //selective logic begins
3671- if (
3672- entry.selective && //all entries are selective now
3673- Array.isArray(entry.keysecondary) && //always true
3674- entry.keysecondary.length //ignore empties
3675- ) {
3676- console.debug(`WI UID:${entry.uid} found. Checking logic: ${entry.selectiveLogic}`);
3677- let hasAnyMatch = false;
3678- let hasAllMatch = true;
3679- secondary: for (let keysecondary of entry.keysecondary) {
3680- const secondarySubstituted = substituteParams(keysecondary);
3681- const hasSecondaryMatch = secondarySubstituted && buffer.matchKeys(textToScan, secondarySubstituted.trim(), entry);
3682- console.debug(`WI UID:${entry.uid}: Filtering for secondary keyword - "${secondarySubstituted}".`);
3683-
3684- if (hasSecondaryMatch) {
3685- hasAnyMatch = true;
3686- }
3687-
3688- if (!hasSecondaryMatch) {
3689- hasAllMatch = false;
3690- }
3691-
3692- // Simplified AND ANY / NOT ALL if statement. (Proper fix for PR#1356 by Bronya)
3693- // If AND ANY logic and the main checks pass OR if NOT ALL logic and the main checks do not pass
3694- if ((selectiveLogic === world_info_logic.AND_ANY && hasSecondaryMatch) || (selectiveLogic === world_info_logic.NOT_ALL && !hasSecondaryMatch)) {
3695- // Differ both logic statements in the debugger
3696- if (selectiveLogic === world_info_logic.AND_ANY) {
3697- console.debug(`(AND ANY Check) Activating WI Entry ${entry.uid}. Found match for word: ${substituted} ${secondarySubstituted}`);
3698- } else {
3699- console.debug(`(NOT ALL Check) Activating WI Entry ${entry.uid}. Found match for word "${substituted}" without secondary keyword: ${secondarySubstituted}`);
3700- }
3701- activatedNow.add(entry);
3702- break secondary;
3703- }
3704- }
37053712
3706- // Handle NOT ANY logic
3713+ // Cache the text to scan before the loop, it won't change its content
3707- if (selectiveLogic === world_info_logic.NOT_ANY && !hasAnyMatch) {
3714+ const textToScan = buffer.get(entry, scanState);
3708- console.debug(`(NOT ANY Check) Activating WI Entry ${entry.uid}, no secondary keywords found.`);
3709- activatedNow.add(entry);
3710- }
37113715
3712- // Handle AND ALL logic
3716+ // PRIMARY KEYWORDS
3713- if (selectiveLogic === world_info_logic.AND_ALL && hasAllMatch) {
3717+ let primaryKeyMatch = entry.key.find(key => {
3714- console.debug(`(AND ALL Check) Activating WI Entry ${entry.uid}, all secondary keywords found.`);
3718+ const substituted = substituteParams(key);
3715- activatedNow.add(entry);
3719+ return substituted && buffer.matchKeys(textToScan, substituted.trim(), entry);
3716- }
3720+ });
3717- } else {
3721+
3718- // Handle cases where secondary is empty
3722+ if (!primaryKeyMatch) {
3719- console.debug(`WI UID ${entry.uid}: Activated without filter logic.`);
3723+ // Don't write logs for simple no-matches
3720- activatedNow.add(entry);
3724+ continue;
3721- break primary;
3725+ }
3722- }
3726+
3727+ const hasSecondaryKeywords = (
3728+ entry.selective && //all entries are selective now
3729+ Array.isArray(entry.keysecondary) && //always true
3730+ entry.keysecondary.length //ignore empties
3731+ );
3732+
3733+ if (!hasSecondaryKeywords) {
3734+ // Handle cases where secondary is empty
3735+ log('activated by primary key match', primaryKeyMatch);
3736+ activatedNow.add(entry);
3737+ continue;
3738+ }
3739+
3740+
3741+ // SECONDARY KEYWORDS
3742+ const selectiveLogic = entry.selectiveLogic ?? 0; // If selectiveLogic isn't found, assume it's AND, only do this once per entry
3743+ log('Entry with primary key match', primaryKeyMatch, 'has secondary keywords. Checking with logic logic', Object.entries(world_info_logic).find(x => x[1] === entry.selectiveLogic));
3744+
3745+ /** @type {() => boolean} */
3746+ function matchSecondaryKeys() {
3747+ let hasAnyMatch = false;
3748+ let hasAllMatch = true;
3749+ for (let keysecondary of entry.keysecondary) {
3750+ const secondarySubstituted = substituteParams(keysecondary);
3751+ const hasSecondaryMatch = secondarySubstituted && buffer.matchKeys(textToScan, secondarySubstituted.trim(), entry);
3752+
3753+ if (hasSecondaryMatch) hasAnyMatch = true;
3754+ if (!hasSecondaryMatch) hasAllMatch = false;
3755+
3756+ // Simplified AND ANY / NOT ALL if statement. (Proper fix for PR#1356 by Bronya)
3757+ // If AND ANY logic and the main checks pass OR if NOT ALL logic and the main checks do not pass
3758+ if (selectiveLogic === world_info_logic.AND_ANY && hasSecondaryMatch) {
3759+ log('activated. (AND ANY) Found match secondary keyword', secondarySubstituted);
3760+ return true;
37233761 }
3762+ if (selectiveLogic === world_info_logic.NOT_ALL && !hasSecondaryMatch) {
3763+ log('activated. (NOT ALL) Found not matching secondary keyword', secondarySubstituted);
3764+ return true;
3765+ }
3766+ }
3767+
3768+ // Handle NOT ANY logic
3769+ if (selectiveLogic === world_info_logic.NOT_ANY && !hasAnyMatch) {
3770+ log('activated. (NOT ANY) No secondary keywords found', entry.keysecondary);
3771+ return true;
3772+ }
3773+
3774+ // Handle AND ALL logic
3775+ if (selectiveLogic === world_info_logic.AND_ALL && hasAllMatch) {
3776+ log('activated. (AND ALL) All secondary keywords found', entry.keysecondary);
3777+ return true;
37243778 }
3779+
3780+ return false;
37253781 }
3782+
3783+ const matched = matchSecondaryKeys();
3784+ if (!matched) {
3785+ log('skipped. Secondary keywords not satisfied', entry.keysecondary);
3786+ continue;
3787+ }
3788+
3789+ // Success logging was already done inside the function, so just add the entry
3790+ activatedNow.add(entry);
3791+ continue;
37263792 }
37273793
3728- scanState = world_info_recursive && activatedNow.size > 0 ? scan_state.RECURSION : scan_state.NONE;
3794+ console.debug(`[WI] Search done. Found ${activatedNow.size} possible entries.`);
37293795 const newEntries = [...activatedNow]
37303796 .sort((a, b) => sortedEntries.indexOf(a) - sortedEntries.indexOf(b));
37313797 let newContent = '';
37323798 const textToScanTokens = await getTokenCountAsync(allActivatedText);
3733- const probabilityChecksBefore = failedProbabilityChecks.size;
37343799
37353800 filterByInclusionGroups(newEntries, allActivatedEntries, buffer, scanState);
37363801
37373802 console.debug('[WI] --- PROBABILITY CHECKS BEGIN ---');
37383803 for (const entry of newEntries) {
3739- const rollValue = Math.random() * 100;
3804+ function verifyProbability() {
3805+ // If we don't need to roll, it's always true
3806+ if (!entry.useProbability || entry.probability === 100) {
3807+ console.debug(`WI entry ${entry.uid} does not use probability`);
3808+ return true;
3809+ }
37403810
3741- if (entry.useProbability && rollValue > entry.probability) {
37423811 const isSticky = timedEffects.isEffectActive('sticky', entry);
37433812 if (!isSticky) {
37443813 console.debug(`WI entry ${entry.uid} ${entry.key}is failedsticky, probabilitydoes check,not skippingneed to re-roll probability`);
3745- failedProbabilityChecks.add(entry);
3814+ return true;
3746- continue;
3815+ }
3816+
3817+ const rollValue = Math.random() * 100;
3818+ if (rollValue <= entry.probability) {
3819+ console.debug(`WI entry ${entry.uid} passed probability check of ${entry.probability}%`);
3820+ return true;
37473821 }
3748- } else { console.debug(`uid:${entry.uid} passed probability check, inserting to prompt`); }
3822+
3823+ failedProbabilityChecks.add(entry);
3824+ return false;
3825+ }
3826+
3827+ const success = verifyProbability();
3828+ if (!success) {
3829+ console.debug(`WI entry ${entry.uid} failed probability check, removing from activated entries`, entry);
3830+ continue;
3831+ }
37493832
37503833 // Substitute macros inline, for both this checking and also future processing
37513834 entry.content = substituteParams(entry.content);
37523835 newContent += `${entry.content}\n`;
37533836
37543837 if ((textToScanTokens + (await getTokenCountAsync(newContent))) >= budget) {
3755- console.debug('WI budget reached, stopping');
37563838 if (world_info_overflow_alert) {
3757- console.log('Alerting');
3839+ console.warn(`[WI] budget of ${budget} reached, stopping after ${allActivatedEntries.size} entries`);
37583840 toastr.warning(`World info budget reached after ${allActivatedEntries.size} entries.`, 'World Info');
3841+ } else {
3842+ console.debug(`[WI] budget of ${budget} reached, stopping after ${allActivatedEntries.size} entries`);
37593843 }
3760- scanState = scan_state.NONE;
37613844 token_budget_overflowed = true;
37623845 break;
37633846 }
37643847
37653848 allActivatedEntries.add(entry);
37663849 console.debug('`[WI] Entry ${entry.uid} activated:'activation successful, adding to prompt`, entry);
37673850 }
37683851
37693852 const probabilityChecksAftersuccessfulNewEntries = newEntries.filter(x => !failedProbabilityChecks.sizehas(x));
3853+ const successfulNewEntriesForRecursion = successfulNewEntries.filter(x => !x.preventRecursion);
37703854
3771- if ((probabilityChecksAfter - probabilityChecksBefore) === activatedNow.size) {
3855+ if (!newEntries.length) {
37723856 console.debug('[WI probability checks] failedNo fornew allentries activated entries, stopping');
3773- scanState = scan_state.NONE;
3857+ } else if (!successfulNewEntries.length) {
3858+ console.debug('[WI] Probability checks failed for all activated entries, stopping');
3859+ } else {
3860+ console.debug(`[WI] Successfully activated ${successfulNewEntries.length} new entries to prompt. ${allActivatedEntries.size} total entries activated.`, successfulNewEntries);
37743861 }
37753862
3776- if (newEntries.length === 0) {
3863+ // After processing and rolling entries is done, see if we should continue with normal recursion
3777- console.debug('No new entries activated, stopping');
3864+ if (world_info_recursive && !token_budget_overflowed && successfulNewEntriesForRecursion.length) {
37783865 scanStatenextScanState = scan_state.NONERECURSION;
37793866 }
37803867
3781- if (scanState) {
3868+ // If we are inside min activations scan, and we have recursive buffer, we should do a recursive scan before increasing the buffer again
3782- const text = newEntries
3869+ // There might be recurse-trigger-able entries that match the buffer, so we need to check that
3783- .filter(x => !failedProbabilityChecks.has(x))
3870+ if (world_info_recursive && !token_budget_overflowed && scanState === scan_state.MIN_ACTIVATIONS && buffer.hasRecurse()) {
3784- .filter(x => !x.preventRecursion)
3871+ nextScanState = scan_state.RECURSION;
3785- .map(x => x.content).join('\n');
3786- buffer.addRecurse(text);
3787- allActivatedText = (text + '\n' + allActivatedText);
37883872 }
37893873
3790- // world_info_min_activations
3874+ // If scanning is planned to stop, but min activations is set and not satisfied, check if we should continue
3791- if (!scanState && !token_budget_overflowed) {
3875+ const minActivationsNotSatisfied = world_info_min_activations > 0 && (allActivatedEntries.size < world_info_min_activations);
37923876 if (world_info_min_activations > 0!nextScanState && (allActivatedEntries.size!token_budget_overflowed <&& world_info_min_activations)minActivationsNotSatisfied) {
3793- let over_max = (
3877+ console.debug('[WI] --- MIN ACTIVATIONS CHECK ---');
3794- world_info_min_activations_depth_max > 0 &&
3795- buffer.getDepth() > world_info_min_activations_depth_max
3796- ) || (buffer.getDepth() > chat.length);
37973878
3798- if (!over_max) {
3879+ let over_max = (
3799- scanState = scan_state.MIN_ACTIVATIONS; // loop
3880+ world_info_min_activations_depth_max > 0 &&
3800- buffer.advanceScanPosition();
3881+ buffer.getDepth() > world_info_min_activations_depth_max
3801- }
3882+ ) || (buffer.getDepth() > chat.length);
3883+
3884+ if (!over_max) {
3885+ console.debug(`[WI] Min activations not reached (${allActivatedEntries.size}/${world_info_min_activations}), advancing depth to ${buffer.getDepth() + 1} and checking again`);
3886+ nextScanState = scan_state.MIN_ACTIVATIONS; // loop
3887+ buffer.advanceScan();
3888+ } else {
3889+ console.debug(`[WI] Min activations not reached (${allActivatedEntries.size}/${world_info_min_activations}), but reached on of depth. Stopping`);
38023890 }
38033891 }
3892+
3893+ // Final check if we should really continue scan, and extend the current WI recurse buffer
3894+ scanState = nextScanState;
3895+ if (scanState) {
3896+ const text = successfulNewEntriesForRecursion
3897+ .map(x => x.content).join('\n');
3898+ buffer.addRecurse(text);
3899+ allActivatedText = (text + '\n' + allActivatedText);
3900+ }
38043901 }
38053902
3903+ console.debug('[WI] --- BUILDING PROMPT ---');
3904+
38063905 // Forward-sorted list of entries for joining
38073906 const WIBeforeEntries = [];
38083907 const WIAfterEntries = [];
@@ -3818,7 +3917,7 @@ async function checkWorldInfo(chat, maxContext, isDryRun) {
38183917 const content = getRegexedString(entry.content, regex_placement.WORLD_INFO, { depth: regexDepth, isMarkdown: false, isPrompt: true });
38193918
38203919 if (!content) {
38213920 console.debug('Skipping adding `[WI] Entry ${entry.uid}`, 'skipped adding to prompt due to empty content:', entry);
38223921 return;
38233922 }
38243923
@@ -3876,6 +3975,9 @@ async function checkWorldInfo(chat, maxContext, isDryRun) {
38763975 buffer.resetExternalEffects();
38773976 timedEffects.cleanUp();
38783977
3978+ console.log(`[WI] Adding ${allActivatedEntries.size} entries to prompt`, Array.from(allActivatedEntries));
3979+ console.debug('[WI] --- DONE ---');
3980+
38793981 return { worldInfoBefore, worldInfoAfter, EMEntries, WIDepthEntries, allActivatedEntries };
38803982}
38813983
@@ -3890,13 +3992,13 @@ function filterGroupsByScoring(groups, buffer, removeEntry, scanState) {
38903992 for (const [key, group] of Object.entries(groups)) {
38913993 // Group scoring is disabled both globally and for the group entries
38923994 if (!world_info_use_group_scoring && !group.some(x => x.useGroupScoring)) {
38933995 console.debug(`[WI] Skipping group scoring for group '${key}'`);
38943996 continue;
38953997 }
38963998
38973999 const scores = group.map(entry => buffer.getScore(entry, scanState));
38984000 const maxScore = Math.max(...scores);
38994001 console.debug(`[WI] Group '${key}' max score:`, ${maxScore}`);
39004002 //console.table(group.map((entry, i) => ({ uid: entry.uid, key: JSON.stringify(entry.key), score: scores[i] })));
39014003
39024004 for (let i = 0; i < group.length; i++) {
@@ -3907,7 +4009,7 @@ function filterGroupsByScoring(groups, buffer, removeEntry, scanState) {
39074009 }
39084010
39094011 if (scores[i] < maxScore) {
39104012 console.debug(`Removing[WI] Entry ${group[i].uid}`, `removed as score loser from inclusion group '${key}' entry '${group[i].uid}'`, group[i]);
39114013 removeEntry(group[i]);
39124014 group.splice(i, 1);
39134015 scores.splice(i, 1);
@@ -3925,7 +4027,8 @@ function filterGroupsByScoring(groups, buffer, removeEntry, scanState) {
39254027 * @param {number} scanState The current scan state
39264028 */
39274029function filterByInclusionGroups(newEntries, allActivatedEntries, buffer, scanState) {
39284030 console.debug('[WI] --- INCLUSION GROUP CHECKS BEGIN ---');
4031+
39294032 const grouped = newEntries.filter(x => x.group).reduce((acc, item) => {
39304033 item.group.split(/,\s*/).filter(x => x).forEach(group => {
39314034 if (!acc[group]) {
@@ -3937,7 +4040,7 @@ function filterByInclusionGroups(newEntries, allActivatedEntries, buffer, scanSt
39374040 }, {});
39384041
39394042 if (Object.keys(grouped).length === 0) {
39404043 console.debug('[WI] No inclusion groups found');
39414044 return;
39424045 }
39434046
@@ -3948,7 +4051,7 @@ function filterByInclusionGroups(newEntries, allActivatedEntries, buffer, scanSt
39484051 continue;
39494052 }
39504053
39514054 if (logging) console.debug(`Removing[WI] Entry ${entry.uid}`, `removed as loser from inclusion group '${entry.group}' entry '${entry.uid}'`, entry);
39524055 removeEntry(entry);
39534056 }
39544057 }
@@ -3956,24 +4059,24 @@ function filterByInclusionGroups(newEntries, allActivatedEntries, buffer, scanSt
39564059 filterGroupsByScoring(grouped, buffer, removeEntry, scanState);
39574060
39584061 for (const [key, group] of Object.entries(grouped)) {
39594062 console.debug(`[WI] Checking inclusion group '${key}' with ${group.length} entries`, group);
39604063
39614064 if (Array.from(allActivatedEntries).some(x => x.group === key)) {
39624065 console.debug(`[WI] Skipping inclusion group check, group already activated '${key}' was already activated`);
39634066 // We need to forcefully deactivate all other entries in the group
39644067 removeAllBut(group, null, false);
39654068 continue;
39664069 }
39674070
39684071 if (!Array.isArray(group) || group.length <= 1) {
39694072 console.debug('[WI] Skipping inclusion group check, only one entry');
39704073 continue;
39714074 }
39724075
39734076 // Check for group prio
39744077 const prios = group.filter(x => x.groupOverride).sort(sortFn);
39754078 if (prios.length) {
39764079 console.debug(`Activated inclusion[WI] groupEntry '${keyprios[0].uid}'`, with`activated byas prio winner entryfrom inclusion group '${prios[0].uidkey}'`, prios[0]);
39774080 removeAllBut(group, prios[0]);
39784081 continue;
39794082 }
@@ -3988,14 +4091,14 @@ function filterByInclusionGroups(newEntries, allActivatedEntries, buffer, scanSt
39884091 currentWeight += (entry.groupWeight ?? DEFAULT_WEIGHT);
39894092
39904093 if (rollValue <= currentWeight) {
39914094 console.debug(`Activated inclusion[WI] groupEntry '${keyentry.uid}'`, with`activated as roll winner entryfrom inclusion group '${entry.uidkey}'`, entry);
39924095 winner = entry;
39934096 break;
39944097 }
39954098 }
39964099
39974100 if (!winner) {
39984101 console.debug(`[WI] Failed to activate inclusion group '${key}', no winner found`);
39994102 continue;
40004103 }
40014104