Bulk extension field updates via merge-attributes with UNSET_VALUE sentinel (#5471) * feat: add bulk extension field updates with UNSET_VALUE sentinel for key deletion - Add `UNSET_VALUE` sentinel constant to signal complete field removal from character cards - Add `writeExtensionFieldBulk()` function to update extension fields across multiple characters in a single API call - Add `deleteValueByPath()` utility function to remove nested object keys by dot-path - Update `writeExtensionField()` to support `UNSET_VALUE` for deleting extension keys - Extend `/api/characters/merge-attributes * Revert package-lock.json changes * Allow null values in merge-attributes filter path validation Change filter.path existence check to only skip on undefined, not null. This allows merging attributes when the existing value is explicitly null, treating null as a valid value rather than absence of a value. * fix: share forbiddenRegExp between modules * feat: add writeExtensionFieldBulk and UNSET_VALUE constant to getContext * Update src/endpoints/characters.js Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * fix: validate for .png extension * Update public/scripts/extensions.js Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * refactor: extract shouldSkip logic as a function param to avoid double parsing --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

d720605be856e6f484e4b972d73d84f761ac4051

Wolfsblvt <wolfsblvt@gmail.com>

Signed

Parents: b436971a091d1512c3c3ebaf83e707d38dcfdf26

public/scripts/extensions.js+127 -4 Viewed

 import { eventSource, event_types, saveSettings, saveSettingsDebounced, getRequestHeaders, animation_duration, CLIENT_VERSION } from '../script.js';
 import { POPUP_RESULT, POPUP_TYPE, Popup } from './popup.js';
 import { renderTemplate, renderTemplateAsync } from './templates.js';
-import { delay, equalsIgnoreCaseAndAccents, isSubsetOf, sanitizeSelector, setValueByPath, versionCompare } from './utils.js';
+import { delay, deleteValueByPath, equalsIgnoreCaseAndAccents, isSubsetOf, sanitizeSelector, setValueByPath, versionCompare } from './utils.js';
 import { getContext } from './st-context.js';
 import { isAdmin } from './user.js';
 import { addLocaleData, getCurrentLocale, t } from './i18n.js';

+}
 /**
+ * Sentinel value that signals a field should be completely removed (unset)
+ * from the character card rather than being set to any value. Pass this as
+ * the `value` argument to {@link writeExtensionField} or
+ * {@link writeExtensionFieldBulk} to delete the key entirely.
+ *
+ * Using `null` as a value will set the field to `null` (the key remains).
+ * Using this sentinel will delete the key from the character card.
+ * @type {string}
+ */
+export const UNSET_VALUE = '__@@UNSET@@__';
+/**
  * Writes a field to the character's data extensions object.
  * @param {number|string} characterId Index in the character array
  * @param {string} key Field name

         console.warn('Character not found', characterId);
         return;
+    }
-    const path = `data.extensions.${key}`;
+    const extensionPath = `data.extensions.${key}`;
-    setValueByPath(character, path, value);
+    const isUnset = value === UNSET_VALUE;
+    if (isUnset) {
+        deleteValueByPath(character, extensionPath);
+    } else {
+        setValueByPath(character, extensionPath, value);
+    }
     // Process JSON data
     if (character.json_data) {
         const jsonData = JSON.parse(character.json_data);
-        setValueByPath(jsonData, path, value);
+        if (isUnset) {
+            deleteValueByPath(jsonData, extensionPath);
+        } else {
+            setValueByPath(jsonData, extensionPath, value);
+        }
         character.json_data = JSON.stringify(jsonData);
         // Make sure the data doesn't get lost when saving the current character

+}
 /**
+ * @typedef {object} BulkExtensionFieldResult
+ * @property {string[]} updated  Avatar filenames that were successfully updated
+ * @property {string[]} skipped  Avatar filenames skipped (filter didn't match or unreadable)
+ * @property {string[]} failed   Avatar filenames where the update failed
+ */
+/**
+ * Writes (or deletes) an extension field for multiple characters in a single
+ * bulk request. Unlike {@link writeExtensionField}, this sends one API call
+ * for all characters, and the server processes them in parallel.
+ *
+ * When `value` is {@link UNSET_VALUE} the extension key is **deleted** from
+ * each matching character card. Passing `null` sets the field to `null`
+ * (the key is preserved).
+ *
+ * @param {string[]|null} avatars Avatar filenames to update. Pass `null` or an
+ *   empty array to target **all** characters in the user's character directory.
+ * @param {string} key Extension field name (e.g. "greeting_tools")
+ * @param {any} value Field value, `null` to set null, or
+ *   {@link UNSET_VALUE} to delete the key entirely
+ * @param {object} [options={}] Optional settings
+ * @param {string} [options.filterPath] Dot-path filter — the server will only
+ *   update characters where this path is present and not `undefined`;
+ *   `null` still counts as a match. Useful when the frontend has shallow
+ *   character data and cannot pre-filter.
+ *   Defaults to `data.extensions.<key>` when unsetting, so deletion requests
+ *   automatically skip characters where the field is missing/`undefined`.
+ * @returns {Promise<BulkExtensionFieldResult>} Summary of the bulk operation
+ */
+export async function writeExtensionFieldBulk(avatars, key, value, { filterPath } = {}) {
+    const context = getContext();
+    const extensionPath = `data.extensions.${key}`;
+    const isUnset = value === UNSET_VALUE;
+    // Build the server request
+    const requestBody = {
+        avatars: Array.isArray(avatars) && avatars.length > 0 ? avatars : [],
+        data: {
+            data: {
+                extensions: {
+                    [key]: value,
+                },
+            },
+        },
+    };
+    // Default filter: when unsetting, only touch characters that have the field
+    const resolvedFilterPath = filterPath ?? (isUnset ? extensionPath : undefined);
+    if (resolvedFilterPath) {
+        requestBody.filter = { path: resolvedFilterPath };
+    }
+    const mergeResponse = await fetch('/api/characters/merge-attributes', {
+        method: 'POST',
+        headers: getRequestHeaders(),
+        body: JSON.stringify(requestBody),
+    });
+    if (!mergeResponse.ok) {
+        console.error('Bulk extension field update failed', mergeResponse.statusText);
+        return { updated: [], skipped: [], failed: [] };
+    }
+    /** @type {BulkExtensionFieldResult} */
+    const result = await mergeResponse.json();
+    // Sync in-memory character objects for successfully updated characters
+    const updatedSet = new Set(result.updated);
+    for (const character of context.characters) {
+        if (!character || !updatedSet.has(character.avatar)) continue;
+        if (isUnset) {
+            deleteValueByPath(character, extensionPath);
+        } else {
+            setValueByPath(character, extensionPath, value);
+        }
+        // Keep json_data in sync
+        if (character.json_data) {
+            const jsonData = JSON.parse(character.json_data);
+            if (isUnset) {
+                deleteValueByPath(jsonData, extensionPath);
+            } else {
+                setValueByPath(jsonData, extensionPath, value);
+            }
+            character.json_data = JSON.stringify(jsonData);
+        }
+    }
+    // If the currently active character was updated, sync the hidden input
+    if (context.characterId !== undefined) {
+        const activeChar = context.characters[context.characterId];
+        if (activeChar && updatedSet.has(activeChar.avatar) && activeChar.json_data) {
+            $('#character_json_data').val(activeChar.json_data);
+        }
+    }
+    return result;
+}
+/**
  * Prompts the user to enter the Git URL of the extension to import.
  * After obtaining the Git URL, makes a POST request to '/api/extensions/install' to import the extension.
  * If the extension is imported successfully, a success message is displayed.

public/scripts/st-context.js+6 -0 Viewed

     renderExtensionTemplate,
     renderExtensionTemplateAsync,
     saveMetadataDebounced,
+    UNSET_VALUE,
     writeExtensionField,
+    writeExtensionFieldBulk,
 } from './extensions.js';
 import { groups, openGroupChat, selected_group, unshallowGroupMembers } from './group-chats.js';
 import { addLocaleData, getCurrentLocale, t, translate } from './i18n.js';

         generateRaw,
         generateRawData,
         writeExtensionField,
+        writeExtensionFieldBulk,
         getThumbnailUrl,
         selectCharacterById,
         messageFormatting,

         symbols: {
             ignore: IGNORE_SYMBOL,
         },
+        constants: {
+            unset: UNSET_VALUE,
+        },
     };
+}

public/scripts/utils.js+17 -0 Viewed

+}
 /**
+ * Deletes a value from a nested object at the given dot-separated path.
+ * @param {object} obj Object to delete from
+ * @param {string} path Dot-separated key path (e.g. "data.extensions.myKey")
+ */
+export function deleteValueByPath(obj, path) {
+    const keyParts = path.split('.');
+    let current = obj;
+    for (let i = 0; i < keyParts.length - 1; i++) {
+        if (!current || typeof current !== 'object') return;
+        current = current[keyParts[i]];
+    }
+    if (current && typeof current === 'object') {
+        delete current[keyParts[keyParts.length - 1]];
+    }
+}
+/**
  * Flashes the given HTML element via CSS flash animation for a defined period
  * @param {JQuery<HTMLElement>} element - The element to flash
  * @param {number} timespan - A number in milliseconds how the flash should last (default is 2000ms.  Multiples of 1000ms work best, as they end with the flash animation being at 100% opacity)

src/endpoints/characters.js+158 -22 Viewed

 import storage from 'node-persist';
 import { AVATAR_WIDTH, AVATAR_HEIGHT, DEFAULT_AVATAR_PATH } from '../constants.js';
-import { default as validateAvatarUrlMiddleware, getFileNameValidationFunction } from '../middleware/validateFileName.js';
+import { default as validateAvatarUrlMiddleware, getFileNameValidationFunction, forbiddenRegExp } from '../middleware/validateFileName.js';
 import { deepMerge, humanizedDateTime, tryParse, MemoryLimitedMap, getConfigValue, mutateJsonString, clientRelativePath, getUniqueName, sanitizeSafeCharacterReplacements } from '../util.js';
 import { TavernCardValidator } from '../validator/TavernCardValidator.js';
 import { parse, read, write } from '../character-card-parser.js';

 });
 /**
- * Handle a POST request to edit character properties.
+ * Sentinel value that signals a field should be completely removed (unset)
- *
+ * from the character card rather than being set to any value. Use this in
- * Merges the request body with the selected character and
+ * the merge payload wherever a key should be deleted.
- * validates the result against TavernCard V2 specification.
- *
- * @param {Object} request - The HTTP request object.
- * @param {Object} response - The HTTP response object.
+ *
- * @returns {void}
+ * Both the server and the frontend share this constant so that callers can
- * */
+ * explicitly opt into deletion without overloading `null`.
-router.post('/merge-attributes', getFileNameValidationFunction('avatar'), async function (request, response) {
+ * @type {string}
-    try {
+ */
-        const update = request.body;
+const UNSET_SENTINEL = '__@@UNSET@@__';
-        const avatarPath = path.join(request.user.directories.characters, update.avatar);
-        const pngStringData = await readCharacterData(avatarPath);
+/** Maximum number of characters processed in parallel during bulk merge */
+const BULK_MERGE_CONCURRENCY = 10;
+/**
+ * Recursively walks `source` and removes any key from `target` whose
+ * corresponding value in `source` equals the {@link UNSET_SENTINEL}.
+ * Called after {@link deepMerge} so that the sentinel gets replaced by
+ * an actual key deletion.
+ * @param {object} target The merged character object to clean up
+ * @param {object} source The original update payload (pre-merge clone)
+ */
+function processUnsetSentinels(target, source) {
+    for (const key of Object.keys(source)) {
+        if (source[key] === UNSET_SENTINEL) {
+            _.unset(target, key);
+        } else if (_.isPlainObject(source[key]) && _.isPlainObject(target[key])) {
+            processUnsetSentinels(target[key], source[key]);
+        }
+    }
+}
+/**
+ * Reads a character card, applies a merge update (with sentinel-based
+ * unsetting), validates the result, and writes it back.
+ * @param {string} avatarPath Full path to the character PNG
+ * @param {string} avatar     Avatar filename (e.g. "char.png")
+ * @param {object} updateData The merge payload to apply
+ * @param {import("express").Request} request Express request object
+ * @param {((data: any) => boolean) | null} [shouldSkip] Optional function to determine if a character should be skipped based on its original data (used for bulk merge filtering)
+ * @returns {Promise<{ok: boolean, error?: string, skipped?: boolean}>} Result of the merge operation, including any validation error
+ */
+async function mergeCharacterUpdate(avatarPath, avatar, updateData, request, shouldSkip = null) {
+    const pngStringData = await readCharacterData(avatarPath);
     if (!pngStringData) {
-            console.error('Error: invalid character file.');
+        return { ok: false, error: 'Invalid character file' };
-            return response.status(400).send('Error: invalid character file.');
+    }
     let character = JSON.parse(pngStringData);
+    if (typeof shouldSkip === 'function' && shouldSkip(character)) {
+        return { ok: false, skipped: true };
+    }
+    const update = _.cloneDeep(updateData);
     _.unset(update, 'json_data');
     _.unset(character, 'json_data');
     character = deepMerge(character, update);
+    processUnsetSentinels(character, update);
     const validator = new TavernCardValidator(character);
-        const targetImg = (update.avatar).replace('.png', '');
     //Accept either V1 or V2.
-        if (validator.validate()) {
+    if (!validator.validate()) {
+        return { ok: false, error: validator.lastValidationError ?? 'Validation failed' };
+    }
+    const targetImg = avatar.replace('.png', '');
     await writeCharacterData(avatarPath, JSON.stringify(character), targetImg, request);
+    return { ok: true };
+}
+/**
+ * Handle a POST request to edit character properties.
+ *
+ * Operates in two modes depending on the request body:
+ *
+ * **Single mode** (default behavior) — when `avatar` (string) is present:
+ *   Merges the request body with the selected character and validates the
+ *   result against TavernCard V2 specification.
+ *
+ * **Bulk mode** — when `avatars` (array) is present:
+ *   Applies the same merge to multiple characters in parallel. Supports:
+ *   - An explicit list of avatars, or all characters when the array is empty
+ *   - An optional server-side `filter` so only characters where a given
+ *     JSON path exists and is non-null are updated
+ *
+ * In both modes, any value equal to the sentinel `__@@UNSET@@__` will cause
+ * that key to be **deleted** from the character card instead of being set.
+ *
+ * @param {import("express").Request} request - The HTTP request object
+ * @param {import("express").Response} response - The HTTP response object
+ * @returns {void}
+ */
+router.post('/merge-attributes', getFileNameValidationFunction('avatar'), async function (request, response) {
+    try {
+        // ── Bulk mode: avatars array is present ──────────────────
+        if (Array.isArray(request.body.avatars)) {
+            const { avatars, data, filter } = request.body;
+            if (!_.isPlainObject(data)) {
+                return response.status(400).send({ message: 'No valid update data provided.' });
+            }
+            // Determine which avatar files to process
+            let targetAvatars;
+            if (avatars.length > 0) {
+                for (const avatar of avatars) {
+                    if (typeof avatar !== 'string' || forbiddenRegExp.test(avatar) || path.extname(avatar).toLowerCase() !== '.png') {
+                        return response.status(400).send({ message: `Invalid avatar filename: ${avatar}` });
+                    }
+                }
+                targetAvatars = avatars;
+            } else {
+                // Empty array → scan all characters in the directory
+                const files = fs.readdirSync(request.user.directories.characters);
+                targetAvatars = files.filter(file => path.extname(file).toLowerCase() === '.png');
+            }
+            const updated = [];
+            const skipped = [];
+            const failed = [];
+            /**
+             * Process a single character in bulk: read, filter, merge, validate, write.
+             * @param {string} avatar Avatar filename
+             */
+            const processOne = async (avatar) => {
+                const avatarPath = path.join(request.user.directories.characters, avatar);
+                try {
+                    /** @type {(character: object) => boolean} */
+                    let shouldSkip = () => false;
+                    // Apply optional server-side filter before updating the card
+                    if (filter && typeof filter.path === 'string') {
+                        shouldSkip = (character) => {
+                            const value = _.get(character, filter.path);
+                            return value === undefined;
+                        };
+                    }
+                    const result = await mergeCharacterUpdate(avatarPath, avatar, data, request, shouldSkip);
+                    if (result.ok) {
+                        updated.push(avatar);
+                    } else if (result.skipped) {
+                        skipped.push(avatar);
+                    } else {
+                        console.warn(`Bulk merge failed for ${avatar}:`, result.error);
+                        failed.push(avatar);
+                    }
+                } catch (error) {
+                    console.error(`Bulk merge failed for ${avatar}:`, error);
+                    failed.push(avatar);
+                }
+            };
+            // Process in parallel with a concurrency limit
+            for (let i = 0; i < targetAvatars.length; i += BULK_MERGE_CONCURRENCY) {
+                const batch = targetAvatars.slice(i, i + BULK_MERGE_CONCURRENCY);
+                await Promise.allSettled(batch.map(processOne));
+            }
+            return response.send({ updated, skipped, failed });
+        }
+        // ── Single mode (default behavior) ───────────────────────
+        const update = request.body;
+        const avatarPath = path.join(request.user.directories.characters, update.avatar);
+        const result = await mergeCharacterUpdate(avatarPath, update.avatar, update, request);
+        if (result.ok) {
             response.sendStatus(200);
         } else {
-            console.warn(validator.lastValidationError);
+            console.warn(result.error);
-            response.status(400).send({ message: `Validation failed for ${character.name}`, error: validator.lastValidationError });
+            response.status(400).send({ message: `Validation failed for ${update.avatar}`, error: result.error });
+        }
     } catch (exception) {
         response.status(500).send({ message: 'Unexpected error while saving character.', error: exception.toString() });

src/middleware/validateFileName.js+2 -1 Viewed

 import path from 'node:path';
+export const forbiddenRegExp = path.sep === '/' ? /[/\x00]/ : /[/\x00\\]/;
 /**
  * Checks if an object has a toString method.
  * @param {object} o Object to check

     */
     return function validateAvatarUrlMiddleware(req, res, next) {
         if (req.body && fieldName in req.body && (typeof req.body[fieldName] === 'string' || hasToString(req.body[fieldName]))) {
-            const forbiddenRegExp = path.sep === '/' ? /[/\x00]/ : /[/\x00\\]/;
             if (forbiddenRegExp.test(req.body[fieldName])) {
                 console.error('An error occurred while validating the request body', {
                     handle: req.user.profile.handle,