Merge pull request #2479 from SillyTavern/wi-processing-refactoring WI scan refactoring: Extended logging, updated code flow, changed Min Activations
Signed| @@ -169,7 +169,7 @@ class WorldInfoBuffer { | ||
| 169 | 169 | #skew = 0; |
| 170 | 170 | |
| 171 | 171 | /** |
| 172 | - * @type {number} The starting depth of the global scan depth. Incremented by "min activations" feature to not repeat scans. When > 0 it means a complete scan was done up to #startDepth already, and `advanceScanPosition` was called. | |
| 172 | + * @type {number} The starting depth of the global scan depth. | |
| 173 | 173 | */ |
| 174 | 174 | #startDepth = 0; |
| 175 | 175 | |
| @@ -222,12 +222,12 @@ class WorldInfoBuffer { | ||
| 222 | 222 | } |
| 223 | 223 | |
| 224 | 224 | if (depth < 0) { |
| 225 | 225 | console.error(`[WI] Invalid WI scan depth ${depth}. Must be >= 0`); |
| 226 | 226 | return ''; |
| 227 | 227 | } |
| 228 | 228 | |
| 229 | 229 | if (depth > MAX_SCAN_DEPTH) { |
| 230 | 230 | console.warn(`[WI] Invalid WI scan depth ${depth}. Truncating to ${MAX_SCAN_DEPTH}`); |
| 231 | 231 | depth = MAX_SCAN_DEPTH; |
| 232 | 232 | } |
| 233 | 233 | |
| @@ -301,10 +301,17 @@ class WorldInfoBuffer { | ||
| 301 | 301 | } |
| 302 | 302 | |
| 303 | 303 | /** |
| 304 | 304 | * IncrementsChecks skewif andthe setsrecursion startDepthbuffer tois previousnot depthempty. |
| 305 | + * @returns {boolean} Returns true if the recursion buffer is not empty, otherwise false | |
| 305 | 306 | */ |
| 306 | 307 | advanceScanPositionhasRecurse() { |
| 307 | 308 | return this.#startDepthrecurseBuffer.length => this.getDepth()0; |
| 309 | + } | |
| 310 | + | |
| 311 | + /** | |
| 312 | + * Increments skew to advance the scan range. | |
| 313 | + */ | |
| 314 | + advanceScan() { | |
| 308 | 315 | this.#skew++; |
| 309 | 316 | } |
| 310 | 317 | |
| @@ -436,7 +443,7 @@ class WorldInfoTimedEffects { | ||
| 436 | 443 | const key = this.#getEntryKey(entry); |
| 437 | 444 | const effect = this.#getEntryTimedEffect('cooldown', entry, true); |
| 438 | 445 | chat_metadata.timedWorldInfo.cooldown[key] = effect; |
| 439 | 446 | console.log(`[WI] Adding cooldown entry ${key} on ended sticky: start=${effect.start}, end=${effect.end}, protected=${effect.protected}`); |
| 440 | 447 | // Set the cooldown immediately for this evaluation |
| 441 | 448 | this.#buffer.cooldown.push(entry); |
| 442 | 449 | }, |
| @@ -447,10 +454,10 @@ class WorldInfoTimedEffects { | ||
| 447 | 454 | * @param {WIScanEntry} entry Entry that ended cooldown |
| 448 | 455 | */ |
| 449 | 456 | 'cooldown': (entry) => { |
| 450 | 457 | console.debug('[WI] Cooldown ended for entry', entry.uid); |
| 451 | 458 | }, |
| 452 | 459 | |
| 453 | 460 | 'delay': () => { }, |
| 454 | 461 | }; |
| 455 | 462 | |
| 456 | 463 | /** |
| @@ -537,11 +544,11 @@ class WorldInfoTimedEffects { | ||
| 537 | 544 | /** @type {[string, WITimedEffect][]} */ |
| 538 | 545 | const effects = Object.entries(chat_metadata.timedWorldInfo[type]); |
| 539 | 546 | for (const [key, value] of effects) { |
| 540 | 547 | console.log(`[WI] Processing ${type} entry ${key}`, value); |
| 541 | 548 | const entry = this.#entries.find(x => String(this.#getEntryHash(x)) === String(value.hash)); |
| 542 | 549 | |
| 543 | 550 | if (this.#chat.length <= Number(value.start) && !value.protected) { |
| 544 | 551 | console.log(`[WI] Removing ${type} entry ${key} from timedWorldInfo: chat not advanced`, value); |
| 545 | 552 | delete chat_metadata.timedWorldInfo[type][key]; |
| 546 | 553 | continue; |
| 547 | 554 | } |
| @@ -549,7 +556,7 @@ class WorldInfoTimedEffects { | ||
| 549 | 556 | // Missing entries (they could be from another character's lorebook) |
| 550 | 557 | if (!entry) { |
| 551 | 558 | if (this.#chat.length >= Number(value.end)) { |
| 552 | 559 | console.log(`[WI] Removing ${type} entry from timedWorldInfo: entry not found and interval passed`, entry); |
| 553 | 560 | delete chat_metadata.timedWorldInfo[type][key]; |
| 554 | 561 | } |
| 555 | 562 | continue; |
| @@ -557,13 +564,13 @@ class WorldInfoTimedEffects { | ||
| 557 | 564 | |
| 558 | 565 | // Ignore invalid entries (not configured for timed effects) |
| 559 | 566 | if (!entry[type]) { |
| 560 | 567 | console.log(`[WI] Removing ${type} entry from timedWorldInfo: entry not ${type}`, entry); |
| 561 | 568 | delete chat_metadata.timedWorldInfo[type][key]; |
| 562 | 569 | continue; |
| 563 | 570 | } |
| 564 | 571 | |
| 565 | 572 | if (this.#chat.length >= Number(value.end)) { |
| 566 | 573 | console.log(`[WI] Removing ${type} entry from timedWorldInfo: ${type} interval passed`, entry); |
| 567 | 574 | delete chat_metadata.timedWorldInfo[type][key]; |
| 568 | 575 | if (typeof onEnded === 'function') { |
| 569 | 576 | onEnded(entry); |
| @@ -572,7 +579,7 @@ class WorldInfoTimedEffects { | ||
| 572 | 579 | } |
| 573 | 580 | |
| 574 | 581 | buffer.push(entry); |
| 575 | 582 | console.log(`[WI] Timed effect "${type}" applied to entry`, entry); |
| 576 | 583 | } |
| 577 | 584 | } |
| 578 | 585 | |
| @@ -588,7 +595,7 @@ class WorldInfoTimedEffects { | ||
| 588 | 595 | |
| 589 | 596 | if (this.#chat.length < entry.delay) { |
| 590 | 597 | buffer.push(entry); |
| 591 | 598 | console.log('[WI] Timed effect "delay" applied to entry', entry); |
| 592 | 599 | } |
| 593 | 600 | } |
| 594 | 601 | |
| @@ -635,7 +642,7 @@ class WorldInfoTimedEffects { | ||
| 635 | 642 | const effect = this.#getEntryTimedEffect(type, entry, false); |
| 636 | 643 | chat_metadata.timedWorldInfo[type][key] = effect; |
| 637 | 644 | |
| 638 | 645 | console.log(`[WI] Adding ${type} entry ${key}: start=${effect.start}, end=${effect.end}, protected=${effect.protected}`); |
| 639 | 646 | } |
| 640 | 647 | } |
| 641 | 648 | |
| @@ -667,7 +674,7 @@ class WorldInfoTimedEffects { | ||
| 667 | 674 | if (newState) { |
| 668 | 675 | const effect = this.#getEntryTimedEffect(type, entry, false); |
| 669 | 676 | chat_metadata.timedWorldInfo[type][key] = effect; |
| 670 | 677 | console.log(`[WI] Adding ${type} entry ${key}: start=${effect.start}, end=${effect.end}, protected=${effect.protected}`); |
| 671 | 678 | } |
| 672 | 679 | } |
| 673 | 680 | |
| @@ -3419,13 +3426,12 @@ async function createNewWorldInfo(worldName, { interactive = false } = {}) { | ||
| 3419 | 3426 | async function getCharacterLore() { |
| 3420 | 3427 | const character = characters[this_chid]; |
| 3421 | 3428 | const name = character?.name; |
| 3429 | + /** @type {Set<string>} */ | |
| 3422 | 3430 | let worldsToSearch = new Set(); |
| 3423 | 3431 | |
| 3424 | 3432 | const baseWorldName = character?.data?.extensions?.world; |
| 3425 | 3433 | if (baseWorldName) { |
| 3426 | 3434 | worldsToSearch.add(baseWorldName); |
| 3427 | - } else { | |
| 3428 | - console.debug(`Character ${name}'s base world could not be found or is empty! Skipping...`); | |
| 3429 | 3435 | } |
| 3430 | 3436 | |
| 3431 | 3437 | // TODO: Maybe make the utility function not use the window context? |
| @@ -3435,40 +3441,48 @@ async function getCharacterLore() { | ||
| 3435 | 3441 | worldsToSearch = new Set([...worldsToSearch, ...extraCharLore.extraBooks]); |
| 3436 | 3442 | } |
| 3437 | 3443 | |
| 3444 | + if (!worldsToSearch.size) { | |
| 3445 | + return []; | |
| 3446 | + } | |
| 3447 | + | |
| 3438 | 3448 | let entries = []; |
| 3439 | 3449 | for (const worldName of worldsToSearch) { |
| 3440 | 3450 | if (selected_world_info.includes(worldName)) { |
| 3441 | 3451 | console.debug(`[WI] Character ${name}'s world ${worldName} is already activated in global world info! Skipping...`); |
| 3442 | 3452 | continue; |
| 3443 | 3453 | } |
| 3444 | 3454 | |
| 3445 | 3455 | if (chat_metadata[METADATA_KEY] === worldName) { |
| 3446 | 3456 | console.debug(`[WI] Character ${name}'s world ${worldName} is already activated in chat lore! Skipping...`); |
| 3447 | 3457 | continue; |
| 3448 | 3458 | } |
| 3449 | 3459 | |
| 3450 | 3460 | const data = await loadWorldInfoData(worldName); |
| 3451 | 3461 | const newEntries = data ? Object.keys(data.entries).map((x) => data.entries[x]).map(x({ uid, ...rest }) => ({ ...xuid, world: worldName, ...rest })) : []; |
| 3452 | 3462 | entries = entries.concat(newEntries); |
| 3463 | + | |
| 3464 | + if (!newEntries.length) { | |
| 3465 | + console.debug(`[WI] Character ${name}'s world ${worldName} could not be found or is empty`); | |
| 3466 | + } | |
| 3453 | 3467 | } |
| 3454 | 3468 | |
| 3455 | 3469 | console.debug(`[WI] Character ${name}'s lore (${Array.from(worldsToSearch)}) has ${entries.length} world info entries`, [...worldsToSearch]); |
| 3456 | 3470 | return entries; |
| 3457 | 3471 | } |
| 3458 | 3472 | |
| 3459 | 3473 | async function getGlobalLore() { |
| 3460 | 3474 | if (!selected_world_info?.length) { |
| 3461 | 3475 | return []; |
| 3462 | 3476 | } |
| 3463 | 3477 | |
| 3464 | 3478 | let entries = []; |
| 3465 | 3479 | for (const worldName of selected_world_info) { |
| 3466 | 3480 | const data = await loadWorldInfoData(worldName); |
| 3467 | 3481 | const newEntries = data ? Object.keys(data.entries).map((x) => data.entries[x]).map(x({ uid, ...rest }) => ({ ...xuid, world: worldName, ...rest })) : []; |
| 3468 | 3482 | entries = entries.concat(newEntries); |
| 3469 | 3483 | } |
| 3470 | 3484 | |
| 3471 | 3485 | console.debug(`[WI] Global world info has ${entries.length} entries`, selected_world_info); |
| 3472 | 3486 | |
| 3473 | 3487 | return entries; |
| 3474 | 3488 | } |
| @@ -3481,14 +3495,14 @@ async function getChatLore() { | ||
| 3481 | 3495 | } |
| 3482 | 3496 | |
| 3483 | 3497 | if (selected_world_info.includes(chatWorld)) { |
| 3484 | 3498 | console.debug(`[WI] Chat world ${chatWorld} is already activated in global world info! Skipping...`); |
| 3485 | 3499 | return []; |
| 3486 | 3500 | } |
| 3487 | 3501 | |
| 3488 | 3502 | const data = await loadWorldInfoData(chatWorld); |
| 3489 | 3503 | const entries = data ? Object.keys(data.entries).map((x) => data.entries[x]).map(x({ uid, ...rest }) => ({ ...xuid, world: chatWorld, ...rest })) : []; |
| 3490 | 3504 | |
| 3491 | 3505 | console.debug(`[WI] Chat lore has ${entries.length} entries`, [chatWorld]); |
| 3492 | 3506 | |
| 3493 | 3507 | return entries; |
| 3494 | 3508 | } |
| @@ -3512,7 +3526,7 @@ export async function getSortedEntries() { | ||
| 3512 | 3526 | entries = [...globalLore.sort(sortFn), ...characterLore.sort(sortFn)]; |
| 3513 | 3527 | break; |
| 3514 | 3528 | default: |
| 3515 | 3529 | console.error('[WI] Unknown WI insertion strategy: ', world_info_character_strategy, 'defaulting to evenly'); |
| 3516 | 3530 | entries = [...globalLore, ...characterLore].sort(sortFn); |
| 3517 | 3531 | break; |
| 3518 | 3532 | } |
| @@ -3520,7 +3534,7 @@ export async function getSortedEntries() { | ||
| 3520 | 3534 | // Chat lore always goes first |
| 3521 | 3535 | entries = [...chatLore.sort(sortFn), ...entries]; |
| 3522 | 3536 | |
| 3523 | 3537 | console.debug(`Sorted[WI] Found ${entries.length} world lore entries. usingSorted by strategy`, ${Object.entries(world_info_insertion_strategy).find((x) => x[1] === world_info_character_strategy}`)); |
| 3524 | 3538 | |
| 3525 | 3539 | // Need to deep clone the entries to avoid modifying the cached data |
| 3526 | 3540 | return structuredClone(entries); |
| @@ -3543,6 +3557,8 @@ async function checkWorldInfo(chat, maxContext, isDryRun) { | ||
| 3543 | 3557 | const context = getContext(); |
| 3544 | 3558 | const buffer = new WorldInfoBuffer(chat); |
| 3545 | 3559 | |
| 3560 | + console.debug(`[WI] --- START WI SCAN (on ${chat.length} messages) ---`); | |
| 3561 | + | |
| 3546 | 3562 | // Combine the chat |
| 3547 | 3563 | |
| 3548 | 3564 | // Add the depth or AN if enabled |
| @@ -3566,11 +3582,11 @@ async function checkWorldInfo(chat, maxContext, isDryRun) { | ||
| 3566 | 3582 | let budget = Math.round(world_info_budget * maxContext / 100) || 1; |
| 3567 | 3583 | |
| 3568 | 3584 | if (world_info_budget_cap > 0 && budget > world_info_budget_cap) { |
| 3569 | 3585 | console.debug(`[WI] Budget ${budget} exceeds cap ${world_info_budget_cap}, using cap`); |
| 3570 | 3586 | budget = world_info_budget_cap; |
| 3571 | 3587 | } |
| 3572 | 3588 | |
| 3573 | 3589 | console.debug(`[WI] Context size: ${maxContext}; WI budget: ${budget} (max% = ${world_info_budget}%, cap = ${world_info_budget_cap})`); |
| 3574 | 3590 | const sortedEntries = await getSortedEntries(); |
| 3575 | 3591 | const timedEffects = new WorldInfoTimedEffects(chat, sortedEntries); |
| 3576 | 3592 | |
| @@ -3580,21 +3596,48 @@ async function checkWorldInfo(chat, maxContext, isDryRun) { | ||
| 3580 | 3596 | return { worldInfoBefore: '', worldInfoAfter: '', WIDepthEntries: [], EMEntries: [], allActivatedEntries: new Set() }; |
| 3581 | 3597 | } |
| 3582 | 3598 | |
| 3599 | + console.debug(`[WI] --- SEARCHING ENTRIES (on ${sortedEntries.length} entries) ---`); | |
| 3600 | + | |
| 3583 | 3601 | while (scanState) { |
| 3584 | 3602 | // Track how many times the loop has run. May be useful for debugging. |
| 3585 | - // eslint-disable-next-line no-unused-vars | |
| 3586 | 3603 | count++; |
| 3587 | 3604 | |
| 3605 | + console.debug(`[WI] Loop #${count}. Search state`, Object.entries(scan_state).find(x => x[1] === scanState)); | |
| 3606 | + | |
| 3607 | + // Until decided otherwise, we set the loop to stop scanning after this | |
| 3608 | + let nextScanState = scan_state.NONE; | |
| 3609 | + | |
| 3610 | + // Loop and find all entries that can activate here | |
| 3588 | 3611 | let activatedNow = new Set(); |
| 3589 | 3612 | |
| 3590 | 3613 | for (let entry of sortedEntries) { |
| 3614 | + // Logging preparation | |
| 3615 | + let headerLogged = false; | |
| 3616 | + function log(...args) { | |
| 3617 | + if (!headerLogged) { | |
| 3618 | + console.debug(`[WI] Entry ${entry.uid}`, `from '${entry.world}' processing`, entry); | |
| 3619 | + headerLogged = true; | |
| 3620 | + } | |
| 3621 | + console.debug(`[WI] Entry ${entry.uid}`, ...args); | |
| 3622 | + } | |
| 3623 | + | |
| 3624 | + // Already processed, considered and then skipped entries should still be skipped | |
| 3625 | + if (failedProbabilityChecks.has(entry) || allActivatedEntries.has(entry)) { | |
| 3626 | + continue; | |
| 3627 | + } | |
| 3628 | + | |
| 3629 | + if (entry.disable == true) { | |
| 3630 | + log('disabled'); | |
| 3631 | + continue; | |
| 3632 | + } | |
| 3633 | + | |
| 3591 | 3634 | // Check if this entry applies to the character or if it's excluded |
| 3592 | 3635 | if (entry.characterFilter && entry.characterFilter?.names?.length > 0) { |
| 3593 | 3636 | const nameIncluded = entry.characterFilter.names.includes(getCharaFilename()); |
| 3594 | 3637 | const filtered = entry.characterFilter.isExclude ? nameIncluded : !nameIncluded; |
| 3595 | 3638 | |
| 3596 | 3639 | if (filtered) { |
| 3597 | 3640 | console.debuglog(`WI entry ${entry.uid} 'filtered out by character`'); |
| 3598 | 3641 | continue; |
| 3599 | 3642 | } |
| 3600 | 3643 | } |
| @@ -3611,7 +3654,7 @@ async function checkWorldInfo(chat, maxContext, isDryRun) { | ||
| 3611 | 3654 | const filtered = entry.characterFilter.isExclude ? includesTag : !includesTag; |
| 3612 | 3655 | |
| 3613 | 3656 | if (filtered) { |
| 3614 | 3657 | console.debuglog(`WI entry ${entry.uid} 'filtered out by tag`'); |
| 3615 | 3658 | continue; |
| 3616 | 3659 | } |
| 3617 | 3660 | } |
| @@ -3623,186 +3666,242 @@ async function checkWorldInfo(chat, maxContext, isDryRun) { | ||
| 3623 | 3666 | const isDelay = timedEffects.isEffectActive('delay', entry); |
| 3624 | 3667 | |
| 3625 | 3668 | if (isDelay) { |
| 3626 | - console.debug(`WI entry ${entry.uid} suppressed by delay`, entry); | |
| 3669 | + log('suppressed by delay'); | |
| 3627 | 3670 | continue; |
| 3628 | 3671 | } |
| 3629 | 3672 | |
| 3630 | 3673 | if (isCooldown && !isSticky) { |
| 3631 | - console.debug(`WI entry ${entry.uid} suppressed by cooldown`, entry); | |
| 3674 | + log('suppressed by cooldown'); | |
| 3632 | 3675 | continue; |
| 3633 | 3676 | } |
| 3634 | 3677 | |
| 3635 | - if (failedProbabilityChecks.has(entry)) { | |
| 3678 | + // Only use checks for recursion flags if the scan step was activated by recursion | |
| 3679 | + if (scanState !== scan_state.RECURSION && entry.delayUntilRecursion) { | |
| 3680 | + log('suppressed by delay until recursion'); | |
| 3636 | 3681 | continue; |
| 3637 | 3682 | } |
| 3638 | 3683 | |
| 3639 | - if (allActivatedEntries.has(entry) || entry.disable == true) { | |
| 3684 | + if (scanState === scan_state.RECURSION && world_info_recursive && entry.excludeRecursion) { | |
| 3685 | + log('suppressed by exclude recursion'); | |
| 3640 | 3686 | continue; |
| 3641 | 3687 | } |
| 3642 | 3688 | |
| 3643 | - // Only use checks for recursion flags if the scan step was activated by recursion | |
| 3689 | + // Now do checks for immediate activations | |
| 3644 | 3690 | if (scanState !== scan_state.RECURSION && entry.delayUntilRecursionconstant) { |
| 3645 | - console.debug(`WI entry ${entry.uid} suppressed by delay until recursion`, entry); | |
| 3691 | + log('activated because of constant'); | |
| 3692 | + activatedNow.add(entry); | |
| 3646 | 3693 | continue; |
| 3647 | 3694 | } |
| 3648 | 3695 | |
| 3649 | - if (scanState === scan_state.RECURSION && world_info_recursive && entry.excludeRecursion) { | |
| 3696 | + if (buffer.isExternallyActivated(entry)) { | |
| 3650 | - console.debug(`WI entry ${entry.uid} suppressed by exclude recursion`, entry); | |
| 3697 | + log('externally activated'); | |
| 3698 | + activatedNow.add(entry); | |
| 3651 | 3699 | continue; |
| 3652 | 3700 | } |
| 3653 | 3701 | |
| 3654 | - if (entry.constant || buffer.isExternallyActivated(entry) || isSticky) { | |
| 3702 | + if (isSticky) { | |
| 3703 | + log('activated because active sticky'); | |
| 3655 | 3704 | activatedNow.add(entry); |
| 3656 | 3705 | continue; |
| 3657 | 3706 | } |
| 3658 | 3707 | |
| 3659 | 3708 | if (!Array.isArray(entry.key) &&|| !entry.key.length) { //check for keywords existing |
| 3660 | - // If selectiveLogic isn't found, assume it's AND, only do this once per entry | |
| 3709 | + log('has no keys defined, skipped'); | |
| 3661 | - const selectiveLogic = entry.selectiveLogic ?? 0; | |
| 3710 | + continue; | |
| 3662 | - | |
| 3711 | + } | |
| 3663 | - primary: for (let key of entry.key) { | |
| 3664 | - const substituted = substituteParams(key); | |
| 3665 | - const textToScan = buffer.get(entry, scanState); | |
| 3666 | - | |
| 3667 | - if (substituted && buffer.matchKeys(textToScan, substituted.trim(), entry)) { | |
| 3668 | - console.debug(`WI UID ${entry.uid} found by primary match: ${substituted}.`); | |
| 3669 | - | |
| 3670 | - //selective logic begins | |
| 3671 | - if ( | |
| 3672 | - entry.selective && //all entries are selective now | |
| 3673 | - Array.isArray(entry.keysecondary) && //always true | |
| 3674 | - entry.keysecondary.length //ignore empties | |
| 3675 | - ) { | |
| 3676 | - console.debug(`WI UID:${entry.uid} found. Checking logic: ${entry.selectiveLogic}`); | |
| 3677 | - let hasAnyMatch = false; | |
| 3678 | - let hasAllMatch = true; | |
| 3679 | - secondary: for (let keysecondary of entry.keysecondary) { | |
| 3680 | - const secondarySubstituted = substituteParams(keysecondary); | |
| 3681 | - const hasSecondaryMatch = secondarySubstituted && buffer.matchKeys(textToScan, secondarySubstituted.trim(), entry); | |
| 3682 | - console.debug(`WI UID:${entry.uid}: Filtering for secondary keyword - "${secondarySubstituted}".`); | |
| 3683 | - | |
| 3684 | - if (hasSecondaryMatch) { | |
| 3685 | - hasAnyMatch = true; | |
| 3686 | - } | |
| 3687 | - | |
| 3688 | - if (!hasSecondaryMatch) { | |
| 3689 | - hasAllMatch = false; | |
| 3690 | - } | |
| 3691 | - | |
| 3692 | - // Simplified AND ANY / NOT ALL if statement. (Proper fix for PR#1356 by Bronya) | |
| 3693 | - // If AND ANY logic and the main checks pass OR if NOT ALL logic and the main checks do not pass | |
| 3694 | - if ((selectiveLogic === world_info_logic.AND_ANY && hasSecondaryMatch) || (selectiveLogic === world_info_logic.NOT_ALL && !hasSecondaryMatch)) { | |
| 3695 | - // Differ both logic statements in the debugger | |
| 3696 | - if (selectiveLogic === world_info_logic.AND_ANY) { | |
| 3697 | - console.debug(`(AND ANY Check) Activating WI Entry ${entry.uid}. Found match for word: ${substituted} ${secondarySubstituted}`); | |
| 3698 | - } else { | |
| 3699 | - console.debug(`(NOT ALL Check) Activating WI Entry ${entry.uid}. Found match for word "${substituted}" without secondary keyword: ${secondarySubstituted}`); | |
| 3700 | - } | |
| 3701 | - activatedNow.add(entry); | |
| 3702 | - break secondary; | |
| 3703 | - } | |
| 3704 | - } | |
| 3705 | 3712 | |
| 3706 | - // Handle NOT ANY logic | |
| 3713 | + // Cache the text to scan before the loop, it won't change its content | |
| 3707 | - if (selectiveLogic === world_info_logic.NOT_ANY && !hasAnyMatch) { | |
| 3714 | + const textToScan = buffer.get(entry, scanState); | |
| 3708 | - console.debug(`(NOT ANY Check) Activating WI Entry ${entry.uid}, no secondary keywords found.`); | |
| 3709 | - activatedNow.add(entry); | |
| 3710 | - } | |
| 3711 | 3715 | |
| 3712 | - // Handle AND ALL logic | |
| 3716 | + // PRIMARY KEYWORDS | |
| 3713 | - if (selectiveLogic === world_info_logic.AND_ALL && hasAllMatch) { | |
| 3717 | + let primaryKeyMatch = entry.key.find(key => { | |
| 3714 | - console.debug(`(AND ALL Check) Activating WI Entry ${entry.uid}, all secondary keywords found.`); | |
| 3718 | + const substituted = substituteParams(key); | |
| 3715 | - activatedNow.add(entry); | |
| 3719 | + return substituted && buffer.matchKeys(textToScan, substituted.trim(), entry); | |
| 3716 | - } | |
| 3720 | + }); | |
| 3717 | - } else { | |
| 3721 | + | |
| 3718 | - // Handle cases where secondary is empty | |
| 3722 | + if (!primaryKeyMatch) { | |
| 3719 | - console.debug(`WI UID ${entry.uid}: Activated without filter logic.`); | |
| 3723 | + // Don't write logs for simple no-matches | |
| 3720 | - activatedNow.add(entry); | |
| 3724 | + continue; | |
| 3721 | - break primary; | |
| 3725 | + } | |
| 3722 | - } | |
| 3726 | + | |
| 3727 | + const hasSecondaryKeywords = ( | |
| 3728 | + entry.selective && //all entries are selective now | |
| 3729 | + Array.isArray(entry.keysecondary) && //always true | |
| 3730 | + entry.keysecondary.length //ignore empties | |
| 3731 | + ); | |
| 3732 | + | |
| 3733 | + if (!hasSecondaryKeywords) { | |
| 3734 | + // Handle cases where secondary is empty | |
| 3735 | + log('activated by primary key match', primaryKeyMatch); | |
| 3736 | + activatedNow.add(entry); | |
| 3737 | + continue; | |
| 3738 | + } | |
| 3739 | + | |
| 3740 | + | |
| 3741 | + // SECONDARY KEYWORDS | |
| 3742 | + const selectiveLogic = entry.selectiveLogic ?? 0; // If selectiveLogic isn't found, assume it's AND, only do this once per entry | |
| 3743 | + log('Entry with primary key match', primaryKeyMatch, 'has secondary keywords. Checking with logic logic', Object.entries(world_info_logic).find(x => x[1] === entry.selectiveLogic)); | |
| 3744 | + | |
| 3745 | + /** @type {() => boolean} */ | |
| 3746 | + function matchSecondaryKeys() { | |
| 3747 | + let hasAnyMatch = false; | |
| 3748 | + let hasAllMatch = true; | |
| 3749 | + for (let keysecondary of entry.keysecondary) { | |
| 3750 | + const secondarySubstituted = substituteParams(keysecondary); | |
| 3751 | + const hasSecondaryMatch = secondarySubstituted && buffer.matchKeys(textToScan, secondarySubstituted.trim(), entry); | |
| 3752 | + | |
| 3753 | + if (hasSecondaryMatch) hasAnyMatch = true; | |
| 3754 | + if (!hasSecondaryMatch) hasAllMatch = false; | |
| 3755 | + | |
| 3756 | + // Simplified AND ANY / NOT ALL if statement. (Proper fix for PR#1356 by Bronya) | |
| 3757 | + // If AND ANY logic and the main checks pass OR if NOT ALL logic and the main checks do not pass | |
| 3758 | + if (selectiveLogic === world_info_logic.AND_ANY && hasSecondaryMatch) { | |
| 3759 | + log('activated. (AND ANY) Found match secondary keyword', secondarySubstituted); | |
| 3760 | + return true; | |
| 3723 | 3761 | } |
| 3762 | + if (selectiveLogic === world_info_logic.NOT_ALL && !hasSecondaryMatch) { | |
| 3763 | + log('activated. (NOT ALL) Found not matching secondary keyword', secondarySubstituted); | |
| 3764 | + return true; | |
| 3765 | + } | |
| 3766 | + } | |
| 3767 | + | |
| 3768 | + // Handle NOT ANY logic | |
| 3769 | + if (selectiveLogic === world_info_logic.NOT_ANY && !hasAnyMatch) { | |
| 3770 | + log('activated. (NOT ANY) No secondary keywords found', entry.keysecondary); | |
| 3771 | + return true; | |
| 3772 | + } | |
| 3773 | + | |
| 3774 | + // Handle AND ALL logic | |
| 3775 | + if (selectiveLogic === world_info_logic.AND_ALL && hasAllMatch) { | |
| 3776 | + log('activated. (AND ALL) All secondary keywords found', entry.keysecondary); | |
| 3777 | + return true; | |
| 3724 | 3778 | } |
| 3779 | + | |
| 3780 | + return false; | |
| 3725 | 3781 | } |
| 3782 | + | |
| 3783 | + const matched = matchSecondaryKeys(); | |
| 3784 | + if (!matched) { | |
| 3785 | + log('skipped. Secondary keywords not satisfied', entry.keysecondary); | |
| 3786 | + continue; | |
| 3787 | + } | |
| 3788 | + | |
| 3789 | + // Success logging was already done inside the function, so just add the entry | |
| 3790 | + activatedNow.add(entry); | |
| 3791 | + continue; | |
| 3726 | 3792 | } |
| 3727 | 3793 | |
| 3728 | - scanState = world_info_recursive && activatedNow.size > 0 ? scan_state.RECURSION : scan_state.NONE; | |
| 3794 | + console.debug(`[WI] Search done. Found ${activatedNow.size} possible entries.`); | |
| 3729 | 3795 | const newEntries = [...activatedNow] |
| 3730 | 3796 | .sort((a, b) => sortedEntries.indexOf(a) - sortedEntries.indexOf(b)); |
| 3731 | 3797 | let newContent = ''; |
| 3732 | 3798 | const textToScanTokens = await getTokenCountAsync(allActivatedText); |
| 3733 | - const probabilityChecksBefore = failedProbabilityChecks.size; | |
| 3734 | 3799 | |
| 3735 | 3800 | filterByInclusionGroups(newEntries, allActivatedEntries, buffer, scanState); |
| 3736 | 3801 | |
| 3737 | 3802 | console.debug('[WI] --- PROBABILITY CHECKS BEGIN ---'); |
| 3738 | 3803 | for (const entry of newEntries) { |
| 3739 | - const rollValue = Math.random() * 100; | |
| 3804 | + function verifyProbability() { | |
| 3805 | + // If we don't need to roll, it's always true | |
| 3806 | + if (!entry.useProbability || entry.probability === 100) { | |
| 3807 | + console.debug(`WI entry ${entry.uid} does not use probability`); | |
| 3808 | + return true; | |
| 3809 | + } | |
| 3740 | 3810 | |
| 3741 | - if (entry.useProbability && rollValue > entry.probability) { | |
| 3742 | 3811 | const isSticky = timedEffects.isEffectActive('sticky', entry); |
| 3743 | 3812 | if (!isSticky) { |
| 3744 | 3813 | console.debug(`WI entry ${entry.uid} ${entry.key}is failedsticky, probabilitydoes check,not skippingneed to re-roll probability`); |
| 3745 | - failedProbabilityChecks.add(entry); | |
| 3814 | + return true; | |
| 3746 | - continue; | |
| 3815 | + } | |
| 3816 | + | |
| 3817 | + const rollValue = Math.random() * 100; | |
| 3818 | + if (rollValue <= entry.probability) { | |
| 3819 | + console.debug(`WI entry ${entry.uid} passed probability check of ${entry.probability}%`); | |
| 3820 | + return true; | |
| 3747 | 3821 | } |
| 3748 | - } else { console.debug(`uid:${entry.uid} passed probability check, inserting to prompt`); } | |
| 3822 | + | |
| 3823 | + failedProbabilityChecks.add(entry); | |
| 3824 | + return false; | |
| 3825 | + } | |
| 3826 | + | |
| 3827 | + const success = verifyProbability(); | |
| 3828 | + if (!success) { | |
| 3829 | + console.debug(`WI entry ${entry.uid} failed probability check, removing from activated entries`, entry); | |
| 3830 | + continue; | |
| 3831 | + } | |
| 3749 | 3832 | |
| 3750 | 3833 | // Substitute macros inline, for both this checking and also future processing |
| 3751 | 3834 | entry.content = substituteParams(entry.content); |
| 3752 | 3835 | newContent += `${entry.content}\n`; |
| 3753 | 3836 | |
| 3754 | 3837 | if ((textToScanTokens + (await getTokenCountAsync(newContent))) >= budget) { |
| 3755 | - console.debug('WI budget reached, stopping'); | |
| 3756 | 3838 | if (world_info_overflow_alert) { |
| 3757 | - console.log('Alerting'); | |
| 3839 | + console.warn(`[WI] budget of ${budget} reached, stopping after ${allActivatedEntries.size} entries`); | |
| 3758 | 3840 | toastr.warning(`World info budget reached after ${allActivatedEntries.size} entries.`, 'World Info'); |
| 3841 | + } else { | |
| 3842 | + console.debug(`[WI] budget of ${budget} reached, stopping after ${allActivatedEntries.size} entries`); | |
| 3759 | 3843 | } |
| 3760 | - scanState = scan_state.NONE; | |
| 3761 | 3844 | token_budget_overflowed = true; |
| 3762 | 3845 | break; |
| 3763 | 3846 | } |
| 3764 | 3847 | |
| 3765 | 3848 | allActivatedEntries.add(entry); |
| 3766 | 3849 | console.debug('`[WI] Entry ${entry.uid} activated:'activation successful, adding to prompt`, entry); |
| 3767 | 3850 | } |
| 3768 | 3851 | |
| 3769 | 3852 | const probabilityChecksAftersuccessfulNewEntries = newEntries.filter(x => !failedProbabilityChecks.sizehas(x)); |
| 3853 | + const successfulNewEntriesForRecursion = successfulNewEntries.filter(x => !x.preventRecursion); | |
| 3770 | 3854 | |
| 3771 | - if ((probabilityChecksAfter - probabilityChecksBefore) === activatedNow.size) { | |
| 3855 | + if (!newEntries.length) { | |
| 3772 | 3856 | console.debug('[WI probability checks] failedNo fornew allentries activated entries, stopping'); |
| 3773 | - scanState = scan_state.NONE; | |
| 3857 | + } else if (!successfulNewEntries.length) { | |
| 3858 | + console.debug('[WI] Probability checks failed for all activated entries, stopping'); | |
| 3859 | + } else { | |
| 3860 | + console.debug(`[WI] Successfully activated ${successfulNewEntries.length} new entries to prompt. ${allActivatedEntries.size} total entries activated.`, successfulNewEntries); | |
| 3774 | 3861 | } |
| 3775 | 3862 | |
| 3776 | - if (newEntries.length === 0) { | |
| 3863 | + // After processing and rolling entries is done, see if we should continue with normal recursion | |
| 3777 | - console.debug('No new entries activated, stopping'); | |
| 3864 | + if (world_info_recursive && !token_budget_overflowed && successfulNewEntriesForRecursion.length) { | |
| 3778 | 3865 | scanStatenextScanState = scan_state.NONERECURSION; |
| 3779 | 3866 | } |
| 3780 | 3867 | |
| 3781 | - if (scanState) { | |
| 3868 | + // If we are inside min activations scan, and we have recursive buffer, we should do a recursive scan before increasing the buffer again | |
| 3782 | - const text = newEntries | |
| 3869 | + // There might be recurse-trigger-able entries that match the buffer, so we need to check that | |
| 3783 | - .filter(x => !failedProbabilityChecks.has(x)) | |
| 3870 | + if (world_info_recursive && !token_budget_overflowed && scanState === scan_state.MIN_ACTIVATIONS && buffer.hasRecurse()) { | |
| 3784 | - .filter(x => !x.preventRecursion) | |
| 3871 | + nextScanState = scan_state.RECURSION; | |
| 3785 | - .map(x => x.content).join('\n'); | |
| 3786 | - buffer.addRecurse(text); | |
| 3787 | - allActivatedText = (text + '\n' + allActivatedText); | |
| 3788 | 3872 | } |
| 3789 | 3873 | |
| 3790 | - // world_info_min_activations | |
| 3874 | + // If scanning is planned to stop, but min activations is set and not satisfied, check if we should continue | |
| 3791 | - if (!scanState && !token_budget_overflowed) { | |
| 3875 | + const minActivationsNotSatisfied = world_info_min_activations > 0 && (allActivatedEntries.size < world_info_min_activations); | |
| 3792 | 3876 | if (world_info_min_activations > 0!nextScanState && (allActivatedEntries.size!token_budget_overflowed <&& world_info_min_activations)minActivationsNotSatisfied) { |
| 3793 | - let over_max = ( | |
| 3877 | + console.debug('[WI] --- MIN ACTIVATIONS CHECK ---'); | |
| 3794 | - world_info_min_activations_depth_max > 0 && | |
| 3795 | - buffer.getDepth() > world_info_min_activations_depth_max | |
| 3796 | - ) || (buffer.getDepth() > chat.length); | |
| 3797 | 3878 | |
| 3798 | - if (!over_max) { | |
| 3879 | + let over_max = ( | |
| 3799 | - scanState = scan_state.MIN_ACTIVATIONS; // loop | |
| 3880 | + world_info_min_activations_depth_max > 0 && | |
| 3800 | - buffer.advanceScanPosition(); | |
| 3881 | + buffer.getDepth() > world_info_min_activations_depth_max | |
| 3801 | - } | |
| 3882 | + ) || (buffer.getDepth() > chat.length); | |
| 3883 | + | |
| 3884 | + if (!over_max) { | |
| 3885 | + console.debug(`[WI] Min activations not reached (${allActivatedEntries.size}/${world_info_min_activations}), advancing depth to ${buffer.getDepth() + 1} and checking again`); | |
| 3886 | + nextScanState = scan_state.MIN_ACTIVATIONS; // loop | |
| 3887 | + buffer.advanceScan(); | |
| 3888 | + } else { | |
| 3889 | + console.debug(`[WI] Min activations not reached (${allActivatedEntries.size}/${world_info_min_activations}), but reached on of depth. Stopping`); | |
| 3802 | 3890 | } |
| 3803 | 3891 | } |
| 3892 | + | |
| 3893 | + // Final check if we should really continue scan, and extend the current WI recurse buffer | |
| 3894 | + scanState = nextScanState; | |
| 3895 | + if (scanState) { | |
| 3896 | + const text = successfulNewEntriesForRecursion | |
| 3897 | + .map(x => x.content).join('\n'); | |
| 3898 | + buffer.addRecurse(text); | |
| 3899 | + allActivatedText = (text + '\n' + allActivatedText); | |
| 3900 | + } | |
| 3804 | 3901 | } |
| 3805 | 3902 | |
| 3903 | + console.debug('[WI] --- BUILDING PROMPT ---'); | |
| 3904 | + | |
| 3806 | 3905 | // Forward-sorted list of entries for joining |
| 3807 | 3906 | const WIBeforeEntries = []; |
| 3808 | 3907 | const WIAfterEntries = []; |
| @@ -3818,7 +3917,7 @@ async function checkWorldInfo(chat, maxContext, isDryRun) { | ||
| 3818 | 3917 | const content = getRegexedString(entry.content, regex_placement.WORLD_INFO, { depth: regexDepth, isMarkdown: false, isPrompt: true }); |
| 3819 | 3918 | |
| 3820 | 3919 | if (!content) { |
| 3821 | 3920 | console.debug('Skipping adding `[WI] Entry ${entry.uid}`, 'skipped adding to prompt due to empty content:', entry); |
| 3822 | 3921 | return; |
| 3823 | 3922 | } |
| 3824 | 3923 | |
| @@ -3876,6 +3975,9 @@ async function checkWorldInfo(chat, maxContext, isDryRun) { | ||
| 3876 | 3975 | buffer.resetExternalEffects(); |
| 3877 | 3976 | timedEffects.cleanUp(); |
| 3878 | 3977 | |
| 3978 | + console.log(`[WI] Adding ${allActivatedEntries.size} entries to prompt`, Array.from(allActivatedEntries)); | |
| 3979 | + console.debug('[WI] --- DONE ---'); | |
| 3980 | + | |
| 3879 | 3981 | return { worldInfoBefore, worldInfoAfter, EMEntries, WIDepthEntries, allActivatedEntries }; |
| 3880 | 3982 | } |
| 3881 | 3983 | |
| @@ -3890,13 +3992,13 @@ function filterGroupsByScoring(groups, buffer, removeEntry, scanState) { | ||
| 3890 | 3992 | for (const [key, group] of Object.entries(groups)) { |
| 3891 | 3993 | // Group scoring is disabled both globally and for the group entries |
| 3892 | 3994 | if (!world_info_use_group_scoring && !group.some(x => x.useGroupScoring)) { |
| 3893 | 3995 | console.debug(`[WI] Skipping group scoring for group '${key}'`); |
| 3894 | 3996 | continue; |
| 3895 | 3997 | } |
| 3896 | 3998 | |
| 3897 | 3999 | const scores = group.map(entry => buffer.getScore(entry, scanState)); |
| 3898 | 4000 | const maxScore = Math.max(...scores); |
| 3899 | 4001 | console.debug(`[WI] Group '${key}' max score:`, ${maxScore}`); |
| 3900 | 4002 | //console.table(group.map((entry, i) => ({ uid: entry.uid, key: JSON.stringify(entry.key), score: scores[i] }))); |
| 3901 | 4003 | |
| 3902 | 4004 | for (let i = 0; i < group.length; i++) { |
| @@ -3907,7 +4009,7 @@ function filterGroupsByScoring(groups, buffer, removeEntry, scanState) { | ||
| 3907 | 4009 | } |
| 3908 | 4010 | |
| 3909 | 4011 | if (scores[i] < maxScore) { |
| 3910 | 4012 | console.debug(`Removing[WI] Entry ${group[i].uid}`, `removed as score loser from inclusion group '${key}' entry '${group[i].uid}'`, group[i]); |
| 3911 | 4013 | removeEntry(group[i]); |
| 3912 | 4014 | group.splice(i, 1); |
| 3913 | 4015 | scores.splice(i, 1); |
| @@ -3925,7 +4027,8 @@ function filterGroupsByScoring(groups, buffer, removeEntry, scanState) { | ||
| 3925 | 4027 | * @param {number} scanState The current scan state |
| 3926 | 4028 | */ |
| 3927 | 4029 | function filterByInclusionGroups(newEntries, allActivatedEntries, buffer, scanState) { |
| 3928 | 4030 | console.debug('[WI] --- INCLUSION GROUP CHECKS BEGIN ---'); |
| 4031 | + | |
| 3929 | 4032 | const grouped = newEntries.filter(x => x.group).reduce((acc, item) => { |
| 3930 | 4033 | item.group.split(/,\s*/).filter(x => x).forEach(group => { |
| 3931 | 4034 | if (!acc[group]) { |
| @@ -3937,7 +4040,7 @@ function filterByInclusionGroups(newEntries, allActivatedEntries, buffer, scanSt | ||
| 3937 | 4040 | }, {}); |
| 3938 | 4041 | |
| 3939 | 4042 | if (Object.keys(grouped).length === 0) { |
| 3940 | 4043 | console.debug('[WI] No inclusion groups found'); |
| 3941 | 4044 | return; |
| 3942 | 4045 | } |
| 3943 | 4046 | |
| @@ -3948,7 +4051,7 @@ function filterByInclusionGroups(newEntries, allActivatedEntries, buffer, scanSt | ||
| 3948 | 4051 | continue; |
| 3949 | 4052 | } |
| 3950 | 4053 | |
| 3951 | 4054 | if (logging) console.debug(`Removing[WI] Entry ${entry.uid}`, `removed as loser from inclusion group '${entry.group}' entry '${entry.uid}'`, entry); |
| 3952 | 4055 | removeEntry(entry); |
| 3953 | 4056 | } |
| 3954 | 4057 | } |
| @@ -3956,24 +4059,24 @@ function filterByInclusionGroups(newEntries, allActivatedEntries, buffer, scanSt | ||
| 3956 | 4059 | filterGroupsByScoring(grouped, buffer, removeEntry, scanState); |
| 3957 | 4060 | |
| 3958 | 4061 | for (const [key, group] of Object.entries(grouped)) { |
| 3959 | 4062 | console.debug(`[WI] Checking inclusion group '${key}' with ${group.length} entries`, group); |
| 3960 | 4063 | |
| 3961 | 4064 | if (Array.from(allActivatedEntries).some(x => x.group === key)) { |
| 3962 | 4065 | console.debug(`[WI] Skipping inclusion group check, group already activated '${key}' was already activated`); |
| 3963 | 4066 | // We need to forcefully deactivate all other entries in the group |
| 3964 | 4067 | removeAllBut(group, null, false); |
| 3965 | 4068 | continue; |
| 3966 | 4069 | } |
| 3967 | 4070 | |
| 3968 | 4071 | if (!Array.isArray(group) || group.length <= 1) { |
| 3969 | 4072 | console.debug('[WI] Skipping inclusion group check, only one entry'); |
| 3970 | 4073 | continue; |
| 3971 | 4074 | } |
| 3972 | 4075 | |
| 3973 | 4076 | // Check for group prio |
| 3974 | 4077 | const prios = group.filter(x => x.groupOverride).sort(sortFn); |
| 3975 | 4078 | if (prios.length) { |
| 3976 | 4079 | console.debug(`Activated inclusion[WI] groupEntry '${keyprios[0].uid}'`, with`activated byas prio winner entryfrom inclusion group '${prios[0].uidkey}'`, prios[0]); |
| 3977 | 4080 | removeAllBut(group, prios[0]); |
| 3978 | 4081 | continue; |
| 3979 | 4082 | } |
| @@ -3988,14 +4091,14 @@ function filterByInclusionGroups(newEntries, allActivatedEntries, buffer, scanSt | ||
| 3988 | 4091 | currentWeight += (entry.groupWeight ?? DEFAULT_WEIGHT); |
| 3989 | 4092 | |
| 3990 | 4093 | if (rollValue <= currentWeight) { |
| 3991 | 4094 | console.debug(`Activated inclusion[WI] groupEntry '${keyentry.uid}'`, with`activated as roll winner entryfrom inclusion group '${entry.uidkey}'`, entry); |
| 3992 | 4095 | winner = entry; |
| 3993 | 4096 | break; |
| 3994 | 4097 | } |
| 3995 | 4098 | } |
| 3996 | 4099 | |
| 3997 | 4100 | if (!winner) { |
| 3998 | 4101 | console.debug(`[WI] Failed to activate inclusion group '${key}', no winner found`); |
| 3999 | 4102 | continue; |
| 4000 | 4103 | } |
| 4001 | 4104 | |