Bulk extension field updates via merge-attributes with UNSET_VALUE sentinel (#5471) * feat: add bulk extension field updates with UNSET_VALUE sentinel for key deletion - Add `UNSET_VALUE` sentinel constant to signal complete field removal from character cards - Add `writeExtensionFieldBulk()` function to update extension fields across multiple characters in a single API call - Add `deleteValueByPath()` utility function to remove nested object keys by dot-path - Update `writeExtensionField()` to support `UNSET_VALUE` for deleting extension keys - Extend `/api/characters/merge-attributes * Revert package-lock.json changes * Allow null values in merge-attributes filter path validation Change filter.path existence check to only skip on undefined, not null. This allows merging attributes when the existing value is explicitly null, treating null as a valid value rather than absence of a value. * fix: share forbiddenRegExp between modules * feat: add writeExtensionFieldBulk and UNSET_VALUE constant to getContext * Update src/endpoints/characters.js Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * fix: validate for .png extension * Update public/scripts/extensions.js Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * refactor: extract shouldSkip logic as a function param to avoid double parsing --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

d720605be856e6f484e4b972d73d84f761ac4051

Wolfsblvt <wolfsblvt@gmail.com>

Signed

Parents: b436971a091d1512c3c3ebaf83e707d38dcfdf26

public/scripts/extensions.js+127 -4 Viewed

 import { eventSource, event_types, saveSettings, saveSettingsDebounced, getRequestHeaders, animation_duration, CLIENT_VERSION } from '../script.js';
 import { POPUP_RESULT, POPUP_TYPE, Popup } from './popup.js';
 import { renderTemplate, renderTemplateAsync } from './templates.js';
 import { delay, deleteValueByPath, equalsIgnoreCaseAndAccents, isSubsetOf, sanitizeSelector, setValueByPath, versionCompare } from './utils.js';
 import { getContext } from './st-context.js';
 import { isAdmin } from './user.js';
 import { addLocaleData, getCurrentLocale, t } from './i18n.js';

+}
 /**
 + * Sentinel value that signals a field should be completely removed (unset)
 + * from the character card rather than being set to any value. Pass this as
 + * the `value` argument to {@link writeExtensionField} or
 + * {@link writeExtensionFieldBulk} to delete the key entirely.
 + *
 + * Using `null` as a value will set the field to `null` (the key remains).
 + * Using this sentinel will delete the key from the character card.
 + * @type {string}
 + */
 +export const UNSET_VALUE = '__@@UNSET@@__';
++
 +/**
  * Writes a field to the character's data extensions object.
  * @param {number|string} characterId Index in the character array
  * @param {string} key Field name

         console.warn('Character not found', characterId);
         return;
+    }
     const pathextensionPath = `data.extensions.${key}`;
 -    setValueByPath(character, path, value);
 +    const isUnset = value === UNSET_VALUE;
++
 +    if (isUnset) {
 +        deleteValueByPath(character, extensionPath);
 +    } else {
 +        setValueByPath(character, extensionPath, value);
 +    }
     // Process JSON data
     if (character.json_data) {
         const jsonData = JSON.parse(character.json_data);
 -        setValueByPath(jsonData, path, value);
 +        if (isUnset) {
 +            deleteValueByPath(jsonData, extensionPath);
 +        } else {
 +            setValueByPath(jsonData, extensionPath, value);
 +        }
         character.json_data = JSON.stringify(jsonData);
         // Make sure the data doesn't get lost when saving the current character

+}
 /**
 + * @typedef {object} BulkExtensionFieldResult
 + * @property {string[]} updated  Avatar filenames that were successfully updated
 + * @property {string[]} skipped  Avatar filenames skipped (filter didn't match or unreadable)
 + * @property {string[]} failed   Avatar filenames where the update failed
 + */
++
 +/**
 + * Writes (or deletes) an extension field for multiple characters in a single
 + * bulk request. Unlike {@link writeExtensionField}, this sends one API call
 + * for all characters, and the server processes them in parallel.
 + *
 + * When `value` is {@link UNSET_VALUE} the extension key is **deleted** from
 + * each matching character card. Passing `null` sets the field to `null`
 + * (the key is preserved).
 + *
 + * @param {string[]|null} avatars Avatar filenames to update. Pass `null` or an
 + *   empty array to target **all** characters in the user's character directory.
 + * @param {string} key Extension field name (e.g. "greeting_tools")
 + * @param {any} value Field value, `null` to set null, or
 + *   {@link UNSET_VALUE} to delete the key entirely
 + * @param {object} [options={}] Optional settings
 + * @param {string} [options.filterPath] Dot-path filter — the server will only
 + *   update characters where this path is present and not `undefined`;
 + *   `null` still counts as a match. Useful when the frontend has shallow
 + *   character data and cannot pre-filter.
 + *   Defaults to `data.extensions.<key>` when unsetting, so deletion requests
 + *   automatically skip characters where the field is missing/`undefined`.
 + * @returns {Promise<BulkExtensionFieldResult>} Summary of the bulk operation
 + */
 +export async function writeExtensionFieldBulk(avatars, key, value, { filterPath } = {}) {
 +    const context = getContext();
 +    const extensionPath = `data.extensions.${key}`;
 +    const isUnset = value === UNSET_VALUE;
++
 +    // Build the server request
 +    const requestBody = {
 +        avatars: Array.isArray(avatars) && avatars.length > 0 ? avatars : [],
 +        data: {
 +            data: {
 +                extensions: {
 +                    [key]: value,
 +                },
 +            },
 +        },
 +    };
++
 +    // Default filter: when unsetting, only touch characters that have the field
 +    const resolvedFilterPath = filterPath ?? (isUnset ? extensionPath : undefined);
 +    if (resolvedFilterPath) {
 +        requestBody.filter = { path: resolvedFilterPath };
 +    }
++
 +    const mergeResponse = await fetch('/api/characters/merge-attributes', {
 +        method: 'POST',
 +        headers: getRequestHeaders(),
 +        body: JSON.stringify(requestBody),
 +    });
++
 +    if (!mergeResponse.ok) {
 +        console.error('Bulk extension field update failed', mergeResponse.statusText);
 +        return { updated: [], skipped: [], failed: [] };
 +    }
++
 +    /** @type {BulkExtensionFieldResult} */
 +    const result = await mergeResponse.json();
++
 +    // Sync in-memory character objects for successfully updated characters
 +    const updatedSet = new Set(result.updated);
 +    for (const character of context.characters) {
 +        if (!character || !updatedSet.has(character.avatar)) continue;
++
 +        if (isUnset) {
 +            deleteValueByPath(character, extensionPath);
 +        } else {
 +            setValueByPath(character, extensionPath, value);
 +        }
++
 +        // Keep json_data in sync
 +        if (character.json_data) {
 +            const jsonData = JSON.parse(character.json_data);
 +            if (isUnset) {
 +                deleteValueByPath(jsonData, extensionPath);
 +            } else {
 +                setValueByPath(jsonData, extensionPath, value);
 +            }
 +            character.json_data = JSON.stringify(jsonData);
 +        }
 +    }
++
 +    // If the currently active character was updated, sync the hidden input
 +    if (context.characterId !== undefined) {
 +        const activeChar = context.characters[context.characterId];
 +        if (activeChar && updatedSet.has(activeChar.avatar) && activeChar.json_data) {
 +            $('#character_json_data').val(activeChar.json_data);
 +        }
 +    }
++
 +    return result;
 +}
++
 +/**
  * Prompts the user to enter the Git URL of the extension to import.
  * After obtaining the Git URL, makes a POST request to '/api/extensions/install' to import the extension.
  * If the extension is imported successfully, a success message is displayed.

public/scripts/st-context.js+6 -0 Viewed

     renderExtensionTemplate,
     renderExtensionTemplateAsync,
     saveMetadataDebounced,
 +    UNSET_VALUE,
     writeExtensionField,
 +    writeExtensionFieldBulk,
 } from './extensions.js';
 import { groups, openGroupChat, selected_group, unshallowGroupMembers } from './group-chats.js';
 import { addLocaleData, getCurrentLocale, t, translate } from './i18n.js';

         generateRaw,
         generateRawData,
         writeExtensionField,
 +        writeExtensionFieldBulk,
         getThumbnailUrl,
         selectCharacterById,
         messageFormatting,

         symbols: {
             ignore: IGNORE_SYMBOL,
         },
 +        constants: {
 +            unset: UNSET_VALUE,
 +        },
     };
+}

public/scripts/utils.js+17 -0 Viewed

+}
 /**
 + * Deletes a value from a nested object at the given dot-separated path.
 + * @param {object} obj Object to delete from
 + * @param {string} path Dot-separated key path (e.g. "data.extensions.myKey")
 + */
 +export function deleteValueByPath(obj, path) {
 +    const keyParts = path.split('.');
 +    let current = obj;
 +    for (let i = 0; i < keyParts.length - 1; i++) {
 +        if (!current || typeof current !== 'object') return;
 +        current = current[keyParts[i]];
 +    }
 +    if (current && typeof current === 'object') {
 +        delete current[keyParts[keyParts.length - 1]];
 +    }
 +}
++
 +/**
  * Flashes the given HTML element via CSS flash animation for a defined period
  * @param {JQuery<HTMLElement>} element - The element to flash
  * @param {number} timespan - A number in milliseconds how the flash should last (default is 2000ms.  Multiples of 1000ms work best, as they end with the flash animation being at 100% opacity)

src/endpoints/characters.js+160 -24 Viewed

 import storage from 'node-persist';
 import { AVATAR_WIDTH, AVATAR_HEIGHT, DEFAULT_AVATAR_PATH } from '../constants.js';
 import { default as validateAvatarUrlMiddleware, getFileNameValidationFunction, forbiddenRegExp } from '../middleware/validateFileName.js';
 import { deepMerge, humanizedDateTime, tryParse, MemoryLimitedMap, getConfigValue, mutateJsonString, clientRelativePath, getUniqueName, sanitizeSafeCharacterReplacements } from '../util.js';
 import { TavernCardValidator } from '../validator/TavernCardValidator.js';
 import { parse, read, write } from '../character-card-parser.js';

 });
 /**
 + * Sentinel value that signals a field should be completely removed (unset)
 + * from the character card rather than being set to any value. Use this in
 + * the merge payload wherever a key should be deleted.
 + *
 + * Both the server and the frontend share this constant so that callers can
 + * explicitly opt into deletion without overloading `null`.
 + * @type {string}
 + */
 +const UNSET_SENTINEL = '__@@UNSET@@__';
++
 +/** Maximum number of characters processed in parallel during bulk merge */
 +const BULK_MERGE_CONCURRENCY = 10;
++
 +/**
 + * Recursively walks `source` and removes any key from `target` whose
 + * corresponding value in `source` equals the {@link UNSET_SENTINEL}.
 + * Called after {@link deepMerge} so that the sentinel gets replaced by
 + * an actual key deletion.
 + * @param {object} target The merged character object to clean up
 + * @param {object} source The original update payload (pre-merge clone)
 + */
 +function processUnsetSentinels(target, source) {
 +    for (const key of Object.keys(source)) {
 +        if (source[key] === UNSET_SENTINEL) {
 +            _.unset(target, key);
 +        } else if (_.isPlainObject(source[key]) && _.isPlainObject(target[key])) {
 +            processUnsetSentinels(target[key], source[key]);
 +        }
 +    }
 +}
++
 +/**
 + * Reads a character card, applies a merge update (with sentinel-based
 + * unsetting), validates the result, and writes it back.
 + * @param {string} avatarPath Full path to the character PNG
 + * @param {string} avatar     Avatar filename (e.g. "char.png")
 + * @param {object} updateData The merge payload to apply
 + * @param {import("express").Request} request Express request object
 + * @param {((data: any) => boolean) | null} [shouldSkip] Optional function to determine if a character should be skipped based on its original data (used for bulk merge filtering)
 + * @returns {Promise<{ok: boolean, error?: string, skipped?: boolean}>} Result of the merge operation, including any validation error
 + */
 +async function mergeCharacterUpdate(avatarPath, avatar, updateData, request, shouldSkip = null) {
 +    const pngStringData = await readCharacterData(avatarPath);
 +    if (!pngStringData) {
 +        return { ok: false, error: 'Invalid character file' };
 +    }
++
 +    let character = JSON.parse(pngStringData);
++
 +    if (typeof shouldSkip === 'function' && shouldSkip(character)) {
 +        return { ok: false, skipped: true };
 +    }
++
 +    const update = _.cloneDeep(updateData);
 +    _.unset(update, 'json_data');
 +    _.unset(character, 'json_data');
++
 +    character = deepMerge(character, update);
 +    processUnsetSentinels(character, update);
++
 +    const validator = new TavernCardValidator(character);
 +    //Accept either V1 or V2.
 +    if (!validator.validate()) {
 +        return { ok: false, error: validator.lastValidationError ?? 'Validation failed' };
 +    }
++
 +    const targetImg = avatar.replace('.png', '');
 +    await writeCharacterData(avatarPath, JSON.stringify(character), targetImg, request);
 +    return { ok: true };
 +}
++
 +/**
  * Handle a POST request to edit character properties.
+ *
  * MergesOperates thein requesttwo bodymodes withdepending theon selectedthe characterrequest andbody:
 - * validates the result against TavernCard V2 specification.
+ *
 - * @param {Object} request - The HTTP request object.
 + * **Single mode** (default behavior) — when `avatar` (string) is present:
 - * @param {Object} response - The HTTP response object.
 + *   Merges the request body with the selected character and validates the
 + *   result against TavernCard V2 specification.
 + *
 + * **Bulk mode** — when `avatars` (array) is present:
 + *   Applies the same merge to multiple characters in parallel. Supports:
 + *   - An explicit list of avatars, or all characters when the array is empty
 + *   - An optional server-side `filter` so only characters where a given
 + *     JSON path exists and is non-null are updated
 + *
 + * In both modes, any value equal to the sentinel `__@@UNSET@@__` will cause
 + * that key to be **deleted** from the character card instead of being set.
+ *
 + * @param {import("express").Request} request - The HTTP request object
 + * @param {import("express").Response} response - The HTTP response object
  * @returns {void}
  * */
 router.post('/merge-attributes', getFileNameValidationFunction('avatar'), async function (request, response) {
     try {
 -        const update = request.body;
 +        // ── Bulk mode: avatars array is present ──────────────────
 -        const avatarPath = path.join(request.user.directories.characters, update.avatar);
 +        if (Array.isArray(request.body.avatars)) {
 +            const { avatars, data, filter } = request.body;
 -        const pngStringData = await readCharacterData(avatarPath);
 +            if (!_.isPlainObject(data)) {
 +                return response.status(400).send({ message: 'No valid update data provided.' });
 +            }
 -        if (!pngStringData) {
 +            // Determine which avatar files to process
 -            console.error('Error: invalid character file.');
 +            let targetAvatars;
 -            return response.status(400).send('Error: invalid character file.');
 +            if (avatars.length > 0) {
 -        }
 +                for (const avatar of avatars) {
 +                    if (typeof avatar !== 'string' || forbiddenRegExp.test(avatar) || path.extname(avatar).toLowerCase() !== '.png') {
 +                        return response.status(400).send({ message: `Invalid avatar filename: ${avatar}` });
 +                    }
 +                }
 +                targetAvatars = avatars;
 +            } else {
 +                // Empty array → scan all characters in the directory
 +                const files = fs.readdirSync(request.user.directories.characters);
 +                targetAvatars = files.filter(file => path.extname(file).toLowerCase() === '.png');
 +            }
 -        let character = JSON.parse(pngStringData);
 +            const updated = [];
 +            const skipped = [];
 +            const failed = [];
 -        _.unset(update, 'json_data');
 +            /**
 -        _.unset(character, 'json_data');
 +             * Process a single character in bulk: read, filter, merge, validate, write.
 +             * @param {string} avatar Avatar filename
 +             */
 +            const processOne = async (avatar) => {
 +                const avatarPath = path.join(request.user.directories.characters, avatar);
 -        character = deepMerge(character, update);
 +                try {
 +                    /** @type {(character: object) => boolean} */
 +                    let shouldSkip = () => false;
++
 +                    // Apply optional server-side filter before updating the card
 +                    if (filter && typeof filter.path === 'string') {
 +                        shouldSkip = (character) => {
 +                            const value = _.get(character, filter.path);
 +                            return value === undefined;
 +                        };
 +                    }
++
 +                    const result = await mergeCharacterUpdate(avatarPath, avatar, data, request, shouldSkip);
 +                    if (result.ok) {
 +                        updated.push(avatar);
 +                    } else if (result.skipped) {
 +                        skipped.push(avatar);
 +                    } else {
 +                        console.warn(`Bulk merge failed for ${avatar}:`, result.error);
 +                        failed.push(avatar);
 +                    }
 +                } catch (error) {
 +                    console.error(`Bulk merge failed for ${avatar}:`, error);
 +                    failed.push(avatar);
 +                }
 +            };
 -        const validator = new TavernCardValidator(character);
 +            // Process in parallel with a concurrency limit
 -        const targetImg = (update.avatar).replace('.png', '');
 +            for (let i = 0; i < targetAvatars.length; i += BULK_MERGE_CONCURRENCY) {
 +                const batch = targetAvatars.slice(i, i + BULK_MERGE_CONCURRENCY);
 +                await Promise.allSettled(batch.map(processOne));
 +            }
++
 +            return response.send({ updated, skipped, failed });
 +        }
++
 +        // ── Single mode (default behavior) ───────────────────────
 +        const update = request.body;
 +        const avatarPath = path.join(request.user.directories.characters, update.avatar);
 -        //Accept either V1 or V2.
 +        const result = await mergeCharacterUpdate(avatarPath, update.avatar, update, request);
         if (validatorresult.validate()ok) {
 -            await writeCharacterData(avatarPath, JSON.stringify(character), targetImg, request);
             response.sendStatus(200);
         } else {
             console.warn(validatorresult.lastValidationErrorerror);
             response.status(400).send({ message: `Validation failed for ${characterupdate.nameavatar}`, error: validatorresult.lastValidationErrorerror });
+        }
     } catch (exception) {
         response.status(500).send({ message: 'Unexpected error while saving character.', error: exception.toString() });

src/middleware/validateFileName.js+2 -1 Viewed

 import path from 'node:path';
 +export const forbiddenRegExp = path.sep === '/' ? /[/\x00]/ : /[/\x00\\]/;
++
 /**
  * Checks if an object has a toString method.
  * @param {object} o Object to check

     */
     return function validateAvatarUrlMiddleware(req, res, next) {
         if (req.body && fieldName in req.body && (typeof req.body[fieldName] === 'string' || hasToString(req.body[fieldName]))) {
 -            const forbiddenRegExp = path.sep === '/' ? /[/\x00]/ : /[/\x00\\]/;
             if (forbiddenRegExp.test(req.body[fieldName])) {
                 console.error('An error occurred while validating the request body', {
                     handle: req.user.profile.handle,