Macros: refactor with a single replace point

8f373cf1dc2179d3c088fd8f80d3cfd9bdb575c7

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: b837c482fc4ddff38055c55b0807083768931d47

public/scripts/instruct-mode.js+11 -12 Viewed

 /**
  * Replaces instruct mode macros in the given input string.
- * @param {string} input Input string.
  * @param {Object<string, *>} env - Map of macro names to the values they'll be substituted with. If the param
  * values are functions, those functions will be called and their return values are used.
- * @returns {string} String with macros replaced.
+ * @returns {import('./macros.js').Macro[]} Macro objects.
  */
-export function replaceInstructMacros(input, env) {
+export function getInstructMacros(env) {
-    if (!input) {
-        return '';
-    }
     const syspromptMacros = {
         'systemPrompt': (power_user.prefer_character_prompt && env.charPrompt ? env.charPrompt : power_user.sysprompt.content),
         'defaultSystemPrompt|instructSystem|instructSystemPrompt': power_user.sysprompt.content,

         'instructLastInput|instructLastUserPrefix': power_user.instruct.last_input_sequence || power_user.instruct.input_sequence,
     };
+    const macros = [];
     for (const [placeholder, value] of Object.entries(instructMacros)) {
         const regex = new RegExp(`{{(${placeholder})}}`, 'gi');
-        input = input.replace(regex, power_user.instruct.enabled ? value : '');
+        const replace = () => power_user.instruct.enabled ? value : '';
+        macros.push({ regex, replace });
+    }
     for (const [placeholder, value] of Object.entries(syspromptMacros)) {
         const regex = new RegExp(`{{(${placeholder})}}`, 'gi');
-        input = input.replace(regex, power_user.sysprompt.enabled ? value : '');
+        const replace = () => power_user.sysprompt.enabled ? value : '';
+        macros.push({ regex, replace });
+    }
-    input = input.replace(/{{exampleSeparator}}/gi, power_user.context.example_separator);
+    macros.push({ regex: /{{exampleSeparator}}/gi, replace: () => power_user.context.example_separator });
-    input = input.replace(/{{chatStart}}/gi, power_user.context.chat_start);
+    macros.push({ regex: /{{chatStart}}/gi, replace: () => power_user.context.chat_start });
-    return input;
+    return macros;
+}
 jQuery(() => {

public/scripts/macros.js+125 -98 Viewed

 import { chat, chat_metadata, main_api, getMaxContextSize, getCurrentChatId, substituteParams } from '../script.js';
 import { timestampToMoment, isDigitsOnly, getStringHash, escapeRegex, uuidv4 } from './utils.js';
 import { textgenerationwebui_banned_in_macros } from './textgen-settings.js';
-import { replaceInstructMacros } from './instruct-mode.js';
+import { getInstructMacros } from './instruct-mode.js';
-import { replaceVariableMacros } from './variables.js';
+import { getVariableMacros } from './variables.js';
+/**
+ * @typedef Macro
+ * @property {RegExp} regex - Regular expression to match the macro
+ * @property {(substring: string, ...args: any[]) => string} replace - Function to replace the macro
+ */
 // Register any macro that you want to leave in the compiled story string
 Handlebars.registerHelper('trim', () => '{{trim}}');

 /**
  * Replaces banned words in macros with an empty string.
  * Adds them to textgenerationwebui ban list.
- * @param {string} inText Text to replace banned words in
+ * @returns {Macro}
- * @returns {string} Text without the "banned" macro
  */
-function bannedWordsReplace(inText) {
+function getBannedWordsMacro() {
-    if (!inText) {
-        return '';
-    }
     const banPattern = /{{banned "(.*)"}}/gi;
+    const banReplace = (match, bannedWord) => {
-    if (main_api == 'textgenerationwebui') {
+        if (main_api == 'textgenerationwebui') {
-        const bans = inText.matchAll(banPattern);
+            console.log('Found banned word in macros: ' + bannedWord);
-        if (bans) {
+            textgenerationwebui_banned_in_macros.push(bannedWord);
-            for (const banCase of bans) {
-                console.log('Found banned words in macros: ' + banCase[1]);
-                textgenerationwebui_banned_in_macros.push(banCase[1]);
-            }
+        }
-    }
+        return '';
+    };
-    inText = inText.replaceAll(banPattern, '');
+    return { regex: banPattern, replace: banReplace };
-    return inText;
+}
 function getTimeSinceLastMessage() {

     return 'just now';
+}
-function randomReplace(input, emptyListPlaceholder = '') {
+/**
+ * Returns a macro that picks a random item from a list.
+ * @returns {Macro} The random replace macro
+ */
+function getRandomReplaceMacro() {
     const randomPattern = /{{random\s?::?([^}]+)}}/gi;
+    const randomReplace = (match, listString) => {
-    input = input.replace(randomPattern, (match, listString) => {
         // Split on either double colons or comma. If comma is the separator, we are also trimming all items.
         const list = listString.includes('::')
             ? listString.split('::')

             : listString.replace(/\\,/g, '##�COMMA�##').split(',').map(item => item.trim().replace(/##�COMMA�##/g, ','));
         if (list.length === 0) {
-            return emptyListPlaceholder;
+            return '';
+        }
         const rng = seedrandom('added entropy.', { entropy: true });
         const randomIndex = Math.floor(rng() * list.length);
         return list[randomIndex];
-    });
+    };
-    return input;
-}
-function pickReplace(input, rawContent, emptyListPlaceholder = '') {
+    return { regex: randomPattern, replace: randomReplace };
-    const pickPattern = /{{pick\s?::?([^}]+)}}/gi;
+}
+/**
+ * Returns a macro that picks a random item from a list with a consistent seed.
+ * @param {string} rawContent The raw content of the string
+ * @returns {Macro} The pick replace macro
+ */
+function getPickReplaceMacro(rawContent) {
     // We need to have a consistent chat hash, otherwise we'll lose rolls on chat file rename or branch switches
     // No need to save metadata here - branching and renaming will implicitly do the save for us, and until then loading it like this is consistent
     const chatIdHash = getChatIdHash();
     const rawContentHash = getStringHash(rawContent);
-    return input.replace(pickPattern, (match, listString, offset) => {
+    const pickPattern = /{{pick\s?::?([^}]+)}}/gi;
+    const pickReplace = (match, listString, offset) => {
         // Split on either double colons or comma. If comma is the separator, we are also trimming all items.
         const list = listString.includes('::')
             ? listString.split('::')

             : listString.replace(/\\,/g, '##�COMMA�##').split(',').map(item => item.trim().replace(/##�COMMA�##/g, ','));
         if (list.length === 0) {
-            return emptyListPlaceholder;
+            return '';
+        }
         // We build a hash seed based on: unique chat file, raw content, and the placement inside this content

         const rng = seedrandom(finalSeed);
         const randomIndex = Math.floor(rng() * list.length);
         return list[randomIndex];
-    });
+    };
+    return { regex: pickPattern, replace: pickReplace };
+}
-function diceRollReplace(input, invalidRollPlaceholder = '') {
+/**
+ * @returns {Macro} The dire roll macro
+ */
+function getDiceRollMacro() {
     const rollPattern = /{{roll[ : ]([^}]+)}}/gi;
+    const rollReplace = (match, matchValue) => {
-    return input.replace(rollPattern, (match, matchValue) => {
         let formula = matchValue.trim();
         if (isDigitsOnly(formula)) {

         if (!isValid) {
             console.debug(`Invalid roll formula: ${formula}`);
-            return invalidRollPlaceholder;
+            return '';
+        }
         const result = droll.roll(formula);
-        return new String(result.total);
+        if (result === false) return '';
-    });
+        return String(result.total);
+    };
+    return { regex: rollPattern, replace: rollReplace };
+}
 /**
  * Returns the difference between two times. Works with any time format acceptable by moment().
  * Can work with {{date}} {{time}} macros
- * @param {string} input - The string to replace time difference macros in.
+ * @returns {Macro} The time difference macro
- * @returns {string} The string with replaced time difference macros.
  */
-function timeDiffReplace(input) {
+function getTimeDiffMacro() {
     const timeDiffPattern = /{{timeDiff::(.*?)::(.*?)}}/gi;
+    const timeDiffReplace = (_match, matchPart1, matchPart2) => {
-    const output = input.replace(timeDiffPattern, (_match, matchPart1, matchPart2) => {
         const time1 = moment(matchPart1);
         const time2 = moment(matchPart2);
         const timeDifference = moment.duration(time1.diff(time2));
         return timeDifference.humanize(true);
-    });
+    };
-    return output;
+    return { regex: timeDiffPattern, replace: timeDiffReplace };
+}
 /**

     const rawContent = content;
-    // Legacy non-macro substitutions
+    /**
-    content = content.replace(/<USER>/gi, typeof env.user === 'function' ? env.user() : env.user);
+     * Built-ins running before the env variables
-    content = content.replace(/<BOT>/gi, typeof env.char === 'function' ? env.char() : env.char);
+     * @type {Macro[]}
-    content = content.replace(/<CHAR>/gi, typeof env.char === 'function' ? env.char() : env.char);
+     * */
-    content = content.replace(/<CHARIFNOTGROUP>/gi, typeof env.group === 'function' ? env.group() : env.group);
+    const preEnvMacros = [
-    content = content.replace(/<GROUP>/gi, typeof env.group === 'function' ? env.group() : env.group);
+        // Legacy non-curly macros
+        { regex: /<USER>/gi, replace: () => typeof env.user === 'function' ? env.user() : env.user },
-    // Short circuit if there are no macros
+        { regex: /<BOT>/gi, replace: () => typeof env.char === 'function' ? env.char() : env.char },
-    if (!content.includes('{{')) {
+        { regex: /<CHAR>/gi, replace: () => typeof env.char === 'function' ? env.char() : env.char },
-        return content;
+        { regex: /<CHARIFNOTGROUP>/gi, replace: () => typeof env.group === 'function' ? env.group() : env.group },
-    }
+        { regex: /<GROUP>/gi, replace: () => typeof env.group === 'function' ? env.group() : env.group },
+        getDiceRollMacro(),
+        ...getInstructMacros(env),
+        ...getVariableMacros(),
+        { regex: /{{newline}}/gi, replace: () => '\n' },
+        { regex: /(?:\r?\n)*{{trim}}(?:\r?\n)*/gi, replace: () => '' },
+        { regex: /{{noop}}/gi, replace: () => '' },
+        { regex: /{{input}}/gi, replace: () => String($('#send_textarea').val()) },
+    ];
-    content = diceRollReplace(content);
+    /**
-    content = replaceInstructMacros(content, env);
+     * Built-ins running after the env variables
-    content = replaceVariableMacros(content);
+     * @type {Macro[]}
-    content = content.replace(/{{newline}}/gi, '\n');
+    */
-    content = content.replace(/(?:\r?\n)*{{trim}}(?:\r?\n)*/gi, '');
+    const postEnvMacros = [
-    content = content.replace(/{{noop}}/gi, '');
+        { regex: /{{maxPrompt}}/gi, replace: () => String(getMaxContextSize()) },
-    content = content.replace(/{{input}}/gi, () => String($('#send_textarea').val()));
+        { regex: /{{lastMessage}}/gi, replace: () => getLastMessage() },
+        { regex: /{{lastMessageId}}/gi, replace: () => String(getLastMessageId() ?? '') },
+        { regex: /{{lastUserMessage}}/gi, replace: () => getLastUserMessage() },
+        { regex: /{{lastCharMessage}}/gi, replace: () => getLastCharMessage() },
+        { regex: /{{firstIncludedMessageId}}/gi, replace: () => String(getFirstIncludedMessageId() ?? '') },
+        { regex: /{{lastSwipeId}}/gi, replace: () => String(getLastSwipeId() ?? '') },
+        { regex: /{{currentSwipeId}}/gi, replace: () => String(getCurrentSwipeId() ?? '') },
+        { regex: /{{reverse:(.+?)}}/gi, replace: (_, str) => Array.from(str).reverse().join('') },
+        { regex: /\{\{\/\/([\s\S]*?)\}\}/gm, replace: () => '' },
+        { regex: /{{time}}/gi, replace: () => moment().format('LT') },
+        { regex: /{{date}}/gi, replace: () => moment().format('LL') },
+        { regex: /{{weekday}}/gi, replace: () => moment().format('dddd') },
+        { regex: /{{isotime}}/gi, replace: () => moment().format('HH:mm') },
+        { regex: /{{isodate}}/gi, replace: () => moment().format('YYYY-MM-DD') },
+        { regex: /{{datetimeformat +([^}]*)}}/gi, replace: (_, format) => moment().format(format) },
+        { regex: /{{idle_duration}}/gi, replace: () => getTimeSinceLastMessage() },
+        { regex: /{{time_UTC([-+]\d+)}}/gi, replace: (_, offset) => moment().utc().utcOffset(parseInt(offset, 10)).format('LT') },
+        getTimeDiffMacro(),
+        getBannedWordsMacro(),
+        getRandomReplaceMacro(),
+        getPickReplaceMacro(rawContent),
+    ];
     // Add all registered macros to the env object
-    const nonce = uuidv4();
     MacrosParser.populateEnv(env);
+    const nonce = uuidv4();
+    const envMacros = [];
     // Substitute passed-in variables
     for (const varName in env) {
         if (!Object.hasOwn(env, varName)) continue;
-        content = content.replace(new RegExp(`{{${escapeRegex(varName)}}}`, 'gi'), () => {
+        const envRegex = new RegExp(`{{${escapeRegex(varName)}}}`, 'gi');
+        const envReplace = () => {
             const param = env[varName];
             const value = MacrosParser.sanitizeMacroValue(typeof param === 'function' ? param(nonce) : param);
             return value;
-        });
+        };
+        envMacros.push({ regex: envRegex, replace: envReplace });
+    }
+    const macros = [...preEnvMacros, ...envMacros, ...postEnvMacros];
+    for (const macro of macros) {
+        // Stop if the content is empty
+        if (!content) {
+            break;
+        }
+        // Short-circuit if no curly braces are found
+        if (!macro.regex.source.startsWith('<') && !content.includes('{{')) {
+            break;
+        }
+        content = content.replace(macro.regex, macro.replace);
+    }
-    content = content.replace(/{{maxPrompt}}/gi, () => String(getMaxContextSize()));
-    content = content.replace(/{{lastMessage}}/gi, () => getLastMessage());
-    content = content.replace(/{{lastMessageId}}/gi, () => String(getLastMessageId() ?? ''));
-    content = content.replace(/{{lastUserMessage}}/gi, () => getLastUserMessage());
-    content = content.replace(/{{lastCharMessage}}/gi, () => getLastCharMessage());
-    content = content.replace(/{{firstIncludedMessageId}}/gi, () => String(getFirstIncludedMessageId() ?? ''));
-    content = content.replace(/{{lastSwipeId}}/gi, () => String(getLastSwipeId() ?? ''));
-    content = content.replace(/{{currentSwipeId}}/gi, () => String(getCurrentSwipeId() ?? ''));
-    content = content.replace(/{{reverse:(.+?)}}/gi, (_, str) => Array.from(str).reverse().join(''));
-    content = content.replace(/\{\{\/\/([\s\S]*?)\}\}/gm, '');
-    content = content.replace(/{{time}}/gi, () => moment().format('LT'));
-    content = content.replace(/{{date}}/gi, () => moment().format('LL'));
-    content = content.replace(/{{weekday}}/gi, () => moment().format('dddd'));
-    content = content.replace(/{{isotime}}/gi, () => moment().format('HH:mm'));
-    content = content.replace(/{{isodate}}/gi, () => moment().format('YYYY-MM-DD'));
-    content = content.replace(/{{datetimeformat +([^}]*)}}/gi, (_, format) => {
-        const formattedTime = moment().format(format);
-        return formattedTime;
-    });
-    content = content.replace(/{{idle_duration}}/gi, () => getTimeSinceLastMessage());
-    content = content.replace(/{{time_UTC([-+]\d+)}}/gi, (_, offset) => {
-        const utcOffset = parseInt(offset, 10);
-        const utcTime = moment().utc().utcOffset(utcOffset).format('LT');
-        return utcTime;
-    });
-    content = timeDiffReplace(content);
-    content = bannedWordsReplace(content);
-    content = randomReplace(content);
-    content = pickReplace(content, rawContent);
     return content;
+}

public/scripts/variables.js+19 -69 Viewed

     return name;
+}
-export function replaceVariableMacros(input) {
+/**
-    const lines = input.split('\n');
+ * @returns {import('./macros.js').Macro[]}
+ */
-    for (let i = 0; i < lines.length; i++) {
+export function getVariableMacros() {
-        let line = lines[i];
+    const macros = [
-        // Skip lines without macros
-        if (!line || !line.includes('{{')) {
-            continue;
-        }
         // Replace {{getvar::name}} with the value of the variable name
-        line = line.replace(/{{getvar::([^}]+)}}/gi, (_, name) => {
+        { regex: /{{getvar::([^}]+)}}/gi, replace: (_, name) => getLocalVariable(name.trim()) },
-            name = name.trim();
-            return getLocalVariable(name);
-        });
         // Replace {{setvar::name::value}} with empty string and set the variable name to value
-        line = line.replace(/{{setvar::([^:]+)::([^}]+)}}/gi, (_, name, value) => {
+        { regex: /{{setvar::([^:]+)::([^}]+)}}/gi, replace: (_, name, value) => { setLocalVariable(name.trim(), value); return ''; } },
-            name = name.trim();
-            setLocalVariable(name, value);
-            return '';
-        });
         // Replace {{addvar::name::value}} with empty string and add value to the variable value
-        line = line.replace(/{{addvar::([^:]+)::([^}]+)}}/gi, (_, name, value) => {
+        { regex: /{{addvar::([^:]+)::([^}]+)}}/gi, replace: (_, name, value) => { addLocalVariable(name.trim(), value); return ''; } },
-            name = name.trim();
-            addLocalVariable(name, value);
-            return '';
-        });
         // Replace {{incvar::name}} with empty string and increment the variable name by 1
-        line = line.replace(/{{incvar::([^}]+)}}/gi, (_, name) => {
+        { regex: /{{incvar::([^}]+)}}/gi, replace: (_, name) => incrementLocalVariable(name.trim()) },
-            name = name.trim();
-            return incrementLocalVariable(name);
-        });
         // Replace {{decvar::name}} with empty string and decrement the variable name by 1
-        line = line.replace(/{{decvar::([^}]+)}}/gi, (_, name) => {
+        { regex: /{{decvar::([^}]+)}}/gi, replace: (_, name) => decrementLocalVariable(name.trim()) },
-            name = name.trim();
-            return decrementLocalVariable(name);
-        });
         // Replace {{getglobalvar::name}} with the value of the global variable name
-        line = line.replace(/{{getglobalvar::([^}]+)}}/gi, (_, name) => {
+        { regex: /{{getglobalvar::([^}]+)}}/gi, replace: (_, name) => getGlobalVariable(name.trim()) },
-            name = name.trim();
-            return getGlobalVariable(name);
-        });
         // Replace {{setglobalvar::name::value}} with empty string and set the global variable name to value
-        line = line.replace(/{{setglobalvar::([^:]+)::([^}]+)}}/gi, (_, name, value) => {
+        { regex: /{{setglobalvar::([^:]+)::([^}]+)}}/gi, replace: (_, name, value) => { setGlobalVariable(name.trim(), value); return ''; } },
-            name = name.trim();
-            setGlobalVariable(name, value);
-            return '';
-        });
         // Replace {{addglobalvar::name::value}} with empty string and add value to the global variable value
-        line = line.replace(/{{addglobalvar::([^:]+)::([^}]+)}}/gi, (_, name, value) => {
+        { regex: /{{addglobalvar::([^:]+)::([^}]+)}}/gi, replace: (_, name, value) => { addGlobalVariable(name.trim(), value); return ''; } },
-            name = name.trim();
-            addGlobalVariable(name, value);
-            return '';
-        });
         // Replace {{incglobalvar::name}} with empty string and increment the global variable name by 1
-        line = line.replace(/{{incglobalvar::([^}]+)}}/gi, (_, name) => {
+        { regex: /{{incglobalvar::([^}]+)}}/gi, replace: (_, name) => incrementGlobalVariable(name.trim()) },
-            name = name.trim();
-            return incrementGlobalVariable(name);
-        });
         // Replace {{decglobalvar::name}} with empty string and decrement the global variable name by 1
-        line = line.replace(/{{decglobalvar::([^}]+)}}/gi, (_, name) => {
+        { regex: /{{decglobalvar::([^}]+)}}/gi, replace: (_, name) => decrementGlobalVariable(name.trim()) },
-            name = name.trim();
+    ];
-            return decrementGlobalVariable(name);
-        });
-        lines[i] = line;
-    }
-    return lines.join('\n');
+    return macros;
+}
 async function listVariablesCallback(args) {

         callback: sortArrayObjectCallback,
         returns: 'the sorted list or dictionary keys',
         namedArgumentList: [
-            SlashCommandNamedArgument.fromProps({ name: 'keysort',
+            SlashCommandNamedArgument.fromProps({
+                name: 'keysort',
                 description: 'whether to sort by key or value; ignored for lists',
                 typeList: [ARGUMENT_TYPE.BOOLEAN],
                 enumList: ['true', 'false'],