Macros: refactor with a single replace point

8f373cf1dc2179d3c088fd8f80d3cfd9bdb575c7

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: b837c482fc4ddff38055c55b0807083768931d47
3 files changed, +151 -175Showing whitespace changes
public/scripts/instruct-mode.js+11 -12
@@ -565,16 +565,11 @@ function selectMatchingContextTemplate(name) {
565565
566566/**
567567 * Replaces instruct mode macros in the given input string.
568- * @param {string} input Input string.
569568 * @param {Object<string, *>} env - Map of macro names to the values they'll be substituted with. If the param
570569 * values are functions, those functions will be called and their return values are used.
571- * @returns {string} String with macros replaced.
570+ * @returns {import('./macros.js').Macro[]} Macro objects.
572571 */
573572export function replaceInstructMacrosgetInstructMacros(input, env) {
574- if (!input) {
575- return '';
576- }
577-
578573 const syspromptMacros = {
579574 'systemPrompt': (power_user.prefer_character_prompt && env.charPrompt ? env.charPrompt : power_user.sysprompt.content),
580575 'defaultSystemPrompt|instructSystem|instructSystemPrompt': power_user.sysprompt.content,
@@ -598,20 +593,24 @@ export function replaceInstructMacros(input, env) {
598593 'instructLastInput|instructLastUserPrefix': power_user.instruct.last_input_sequence || power_user.instruct.input_sequence,
599594 };
600595
596+ const macros = [];
597+
601598 for (const [placeholder, value] of Object.entries(instructMacros)) {
602599 const regex = new RegExp(`{{(${placeholder})}}`, 'gi');
603600 inputconst replace = input.replace(regex,) => power_user.instruct.enabled ? value : '');
601+ macros.push({ regex, replace });
604602 }
605603
606604 for (const [placeholder, value] of Object.entries(syspromptMacros)) {
607605 const regex = new RegExp(`{{(${placeholder})}}`, 'gi');
608606 inputconst replace = input.replace(regex,) => power_user.sysprompt.enabled ? value : '');
607+ macros.push({ regex, replace });
609608 }
610609
611610 input = inputmacros.replacepush({ regex: /{{exampleSeparator}}/gi, replace: () => power_user.context.example_separator });
612611 input = inputmacros.replacepush({ regex: /{{chatStart}}/gi, replace: () => power_user.context.chat_start });
613612
614613 return inputmacros;
615614}
616615
617616jQuery(() => {
public/scripts/macros.js+121 -94
@@ -2,8 +2,14 @@ import { Handlebars, moment, seedrandom, droll } from '../lib.js';
22import { chat, chat_metadata, main_api, getMaxContextSize, getCurrentChatId, substituteParams } from '../script.js';
33import { timestampToMoment, isDigitsOnly, getStringHash, escapeRegex, uuidv4 } from './utils.js';
44import { textgenerationwebui_banned_in_macros } from './textgen-settings.js';
55import { replaceInstructMacrosgetInstructMacros } from './instruct-mode.js';
66import { replaceVariableMacrosgetVariableMacros } from './variables.js';
7+
8+/**
9+ * @typedef Macro
10+ * @property {RegExp} regex - Regular expression to match the macro
11+ * @property {(substring: string, ...args: any[]) => string} replace - Function to replace the macro
12+ */
713
814// Register any macro that you want to leave in the compiled story string
915Handlebars.registerHelper('trim', () => '{{trim}}');
@@ -261,28 +267,19 @@ function getCurrentSwipeId() {
261267/**
262268 * Replaces banned words in macros with an empty string.
263269 * Adds them to textgenerationwebui ban list.
264- * @param {string} inText Text to replace banned words in
270+ * @returns {Macro}
265- * @returns {string} Text without the "banned" macro
266271 */
267272function bannedWordsReplacegetBannedWordsMacro(inText) {
268- if (!inText) {
269- return '';
270- }
271-
272273 const banPattern = /{{banned "(.*)"}}/gi;
273-
274+ const banReplace = (match, bannedWord) => {
274275 if (main_api == 'textgenerationwebui') {
275- const bans = inText.matchAll(banPattern);
276+ console.log('Found banned word in macros: ' + bannedWord);
276- if (bans) {
277+ textgenerationwebui_banned_in_macros.push(bannedWord);
277- for (const banCase of bans) {
278- console.log('Found banned words in macros: ' + banCase[1]);
279- textgenerationwebui_banned_in_macros.push(banCase[1]);
280- }
281- }
282278 }
279+ return '';
280+ };
283281
284- inText = inText.replaceAll(banPattern, '');
282+ return { regex: banPattern, replace: banReplace };
285- return inText;
286283}
287284
288285function getTimeSinceLastMessage() {
@@ -317,10 +314,13 @@ function getTimeSinceLastMessage() {
317314 return 'just now';
318315}
319316
320-function randomReplace(input, emptyListPlaceholder = '') {
317+/**
318+ * Returns a macro that picks a random item from a list.
319+ * @returns {Macro} The random replace macro
320+ */
321+function getRandomReplaceMacro() {
321322 const randomPattern = /{{random\s?::?([^}]+)}}/gi;
322-
323+ const randomReplace = (match, listString) => {
323- input = input.replace(randomPattern, (match, listString) => {
324324 // Split on either double colons or comma. If comma is the separator, we are also trimming all items.
325325 const list = listString.includes('::')
326326 ? listString.split('::')
@@ -328,24 +328,29 @@ function randomReplace(input, emptyListPlaceholder = '') {
328328 : listString.replace(/\\,/g, '##�COMMA�##').split(',').map(item => item.trim().replace(/##�COMMA�##/g, ','));
329329
330330 if (list.length === 0) {
331331 return emptyListPlaceholder'';
332332 }
333333 const rng = seedrandom('added entropy.', { entropy: true });
334334 const randomIndex = Math.floor(rng() * list.length);
335335 return list[randomIndex];
336336 });
337- return input;
338-}
339337
340-function pickReplace(input, rawContent, emptyListPlaceholder = '') {
338+ return { regex: randomPattern, replace: randomReplace };
341- const pickPattern = /{{pick\s?::?([^}]+)}}/gi;
339+}
342340
341+/**
342+ * Returns a macro that picks a random item from a list with a consistent seed.
343+ * @param {string} rawContent The raw content of the string
344+ * @returns {Macro} The pick replace macro
345+ */
346+function getPickReplaceMacro(rawContent) {
343347 // We need to have a consistent chat hash, otherwise we'll lose rolls on chat file rename or branch switches
344348 // No need to save metadata here - branching and renaming will implicitly do the save for us, and until then loading it like this is consistent
345349 const chatIdHash = getChatIdHash();
346350 const rawContentHash = getStringHash(rawContent);
347351
348- return input.replace(pickPattern, (match, listString, offset) => {
352+ const pickPattern = /{{pick\s?::?([^}]+)}}/gi;
353+ const pickReplace = (match, listString, offset) => {
349354 // Split on either double colons or comma. If comma is the separator, we are also trimming all items.
350355 const list = listString.includes('::')
351356 ? listString.split('::')
@@ -353,7 +358,7 @@ function pickReplace(input, rawContent, emptyListPlaceholder = '') {
353358 : listString.replace(/\\,/g, '##�COMMA�##').split(',').map(item => item.trim().replace(/##�COMMA�##/g, ','));
354359
355360 if (list.length === 0) {
356361 return emptyListPlaceholder'';
357362 }
358363
359364 // We build a hash seed based on: unique chat file, raw content, and the placement inside this content
@@ -364,13 +369,17 @@ function pickReplace(input, rawContent, emptyListPlaceholder = '') {
364369 const rng = seedrandom(finalSeed);
365370 const randomIndex = Math.floor(rng() * list.length);
366371 return list[randomIndex];
367372 });
373+
374+ return { regex: pickPattern, replace: pickReplace };
368375}
369376
370-function diceRollReplace(input, invalidRollPlaceholder = '') {
377+/**
378+ * @returns {Macro} The dire roll macro
379+ */
380+function getDiceRollMacro() {
371381 const rollPattern = /{{roll[ : ]([^}]+)}}/gi;
372-
382+ const rollReplace = (match, matchValue) => {
373- return input.replace(rollPattern, (match, matchValue) => {
374383 let formula = matchValue.trim();
375384
376385 if (isDigitsOnly(formula)) {
@@ -381,32 +390,33 @@ function diceRollReplace(input, invalidRollPlaceholder = '') {
381390
382391 if (!isValid) {
383392 console.debug(`Invalid roll formula: ${formula}`);
384393 return invalidRollPlaceholder'';
385394 }
386395
387396 const result = droll.roll(formula);
388- return new String(result.total);
397+ if (result === false) return '';
389- });
398+ return String(result.total);
399+ };
400+
401+ return { regex: rollPattern, replace: rollReplace };
390402}
391403
392404/**
393405 * Returns the difference between two times. Works with any time format acceptable by moment().
394406 * Can work with {{date}} {{time}} macros
395407 * @paramreturns {stringMacro} input - The string to replace time difference macros in.macro
396- * @returns {string} The string with replaced time difference macros.
397408 */
398409function timeDiffReplacegetTimeDiffMacro(input) {
399410 const timeDiffPattern = /{{timeDiff::(.*?)::(.*?)}}/gi;
400-
411+ const timeDiffReplace = (_match, matchPart1, matchPart2) => {
401- const output = input.replace(timeDiffPattern, (_match, matchPart1, matchPart2) => {
402412 const time1 = moment(matchPart1);
403413 const time2 = moment(matchPart2);
404414
405415 const timeDifference = moment.duration(time1.diff(time2));
406416 return timeDifference.humanize(true);
407417 });
408418
409- return output;
419+ return { regex: timeDiffPattern, replace: timeDiffReplace };
410420}
411421
412422/**
@@ -423,72 +433,89 @@ export function evaluateMacros(content, env) {
423433
424434 const rawContent = content;
425435
426- // Legacy non-macro substitutions
436+ /**
427- content = content.replace(/<USER>/gi, typeof env.user === 'function' ? env.user() : env.user);
437+ * Built-ins running before the env variables
428- content = content.replace(/<BOT>/gi, typeof env.char === 'function' ? env.char() : env.char);
438+ * @type {Macro[]}
429- content = content.replace(/<CHAR>/gi, typeof env.char === 'function' ? env.char() : env.char);
439+ * */
430- content = content.replace(/<CHARIFNOTGROUP>/gi, typeof env.group === 'function' ? env.group() : env.group);
440+ const preEnvMacros = [
431- content = content.replace(/<GROUP>/gi, typeof env.group === 'function' ? env.group() : env.group);
441+ // Legacy non-curly macros
432-
442+ { regex: /<USER>/gi, replace: () => typeof env.user === 'function' ? env.user() : env.user },
433- // Short circuit if there are no macros
443+ { regex: /<BOT>/gi, replace: () => typeof env.char === 'function' ? env.char() : env.char },
434- if (!content.includes('{{')) {
444+ { regex: /<CHAR>/gi, replace: () => typeof env.char === 'function' ? env.char() : env.char },
435- return content;
445+ { regex: /<CHARIFNOTGROUP>/gi, replace: () => typeof env.group === 'function' ? env.group() : env.group },
436- }
446+ { regex: /<GROUP>/gi, replace: () => typeof env.group === 'function' ? env.group() : env.group },
447+ getDiceRollMacro(),
448+ ...getInstructMacros(env),
449+ ...getVariableMacros(),
450+ { regex: /{{newline}}/gi, replace: () => '\n' },
451+ { regex: /(?:\r?\n)*{{trim}}(?:\r?\n)*/gi, replace: () => '' },
452+ { regex: /{{noop}}/gi, replace: () => '' },
453+ { regex: /{{input}}/gi, replace: () => String($('#send_textarea').val()) },
454+ ];
437455
438- content = diceRollReplace(content);
456+ /**
439- content = replaceInstructMacros(content, env);
457+ * Built-ins running after the env variables
440- content = replaceVariableMacros(content);
458+ * @type {Macro[]}
441- content = content.replace(/{{newline}}/gi, '\n');
459+ */
442- content = content.replace(/(?:\r?\n)*{{trim}}(?:\r?\n)*/gi, '');
460+ const postEnvMacros = [
443- content = content.replace(/{{noop}}/gi, '');
461+ { regex: /{{maxPrompt}}/gi, replace: () => String(getMaxContextSize()) },
444462 content { =regex: content.replace(/{{inputlastMessage}}/gi, replace: () => String($('#send_textarea').valgetLastMessage())); },
463+ { regex: /{{lastMessageId}}/gi, replace: () => String(getLastMessageId() ?? '') },
464+ { regex: /{{lastUserMessage}}/gi, replace: () => getLastUserMessage() },
465+ { regex: /{{lastCharMessage}}/gi, replace: () => getLastCharMessage() },
466+ { regex: /{{firstIncludedMessageId}}/gi, replace: () => String(getFirstIncludedMessageId() ?? '') },
467+ { regex: /{{lastSwipeId}}/gi, replace: () => String(getLastSwipeId() ?? '') },
468+ { regex: /{{currentSwipeId}}/gi, replace: () => String(getCurrentSwipeId() ?? '') },
469+ { regex: /{{reverse:(.+?)}}/gi, replace: (_, str) => Array.from(str).reverse().join('') },
470+ { regex: /\{\{\/\/([\s\S]*?)\}\}/gm, replace: () => '' },
471+ { regex: /{{time}}/gi, replace: () => moment().format('LT') },
472+ { regex: /{{date}}/gi, replace: () => moment().format('LL') },
473+ { regex: /{{weekday}}/gi, replace: () => moment().format('dddd') },
474+ { regex: /{{isotime}}/gi, replace: () => moment().format('HH:mm') },
475+ { regex: /{{isodate}}/gi, replace: () => moment().format('YYYY-MM-DD') },
476+ { regex: /{{datetimeformat +([^}]*)}}/gi, replace: (_, format) => moment().format(format) },
477+ { regex: /{{idle_duration}}/gi, replace: () => getTimeSinceLastMessage() },
478+ { regex: /{{time_UTC([-+]\d+)}}/gi, replace: (_, offset) => moment().utc().utcOffset(parseInt(offset, 10)).format('LT') },
479+ getTimeDiffMacro(),
480+ getBannedWordsMacro(),
481+ getRandomReplaceMacro(),
482+ getPickReplaceMacro(rawContent),
483+ ];
445484
446485 // Add all registered macros to the env object
447- const nonce = uuidv4();
448486 MacrosParser.populateEnv(env);
487+ const nonce = uuidv4();
488+ const envMacros = [];
449489
450490 // Substitute passed-in variables
451491 for (const varName in env) {
452492 if (!Object.hasOwn(env, varName)) continue;
453493
454494 contentconst envRegex = content.replace(new RegExp(`{{${escapeRegex(varName)}}}`, 'gi'), () => {;
495+ const envReplace = () => {
455496 const param = env[varName];
456497 const value = MacrosParser.sanitizeMacroValue(typeof param === 'function' ? param(nonce) : param);
457498 return value;
458499 });
500+
501+ envMacros.push({ regex: envRegex, replace: envReplace });
459502 }
460503
461- content = content.replace(/{{maxPrompt}}/gi, () => String(getMaxContextSize()));
504+ const macros = [...preEnvMacros, ...envMacros, ...postEnvMacros];
462- content = content.replace(/{{lastMessage}}/gi, () => getLastMessage());
505+
463- content = content.replace(/{{lastMessageId}}/gi, () => String(getLastMessageId() ?? ''));
506+ for (const macro of macros) {
464- content = content.replace(/{{lastUserMessage}}/gi, () => getLastUserMessage());
507+ // Stop if the content is empty
465- content = content.replace(/{{lastCharMessage}}/gi, () => getLastCharMessage());
508+ if (!content) {
466- content = content.replace(/{{firstIncludedMessageId}}/gi, () => String(getFirstIncludedMessageId() ?? ''));
509+ break;
467- content = content.replace(/{{lastSwipeId}}/gi, () => String(getLastSwipeId() ?? ''));
510+ }
468- content = content.replace(/{{currentSwipeId}}/gi, () => String(getCurrentSwipeId() ?? ''));
469- content = content.replace(/{{reverse:(.+?)}}/gi, (_, str) => Array.from(str).reverse().join(''));
470511
471- content = content.replace(/\{\{\/\/([\s\S]*?)\}\}/gm, '');
512+ // Short-circuit if no curly braces are found
513+ if (!macro.regex.source.startsWith('<') && !content.includes('{{')) {
514+ break;
515+ }
472516
473- content = content.replace(/{{time}}/gi, () => moment().format('LT'));
517+ content = content.replace(macro.regex, macro.replace);
474- content = content.replace(/{{date}}/gi, () => moment().format('LL'));
518+ }
475- content = content.replace(/{{weekday}}/gi, () => moment().format('dddd'));
476- content = content.replace(/{{isotime}}/gi, () => moment().format('HH:mm'));
477- content = content.replace(/{{isodate}}/gi, () => moment().format('YYYY-MM-DD'));
478519
479- content = content.replace(/{{datetimeformat +([^}]*)}}/gi, (_, format) => {
480- const formattedTime = moment().format(format);
481- return formattedTime;
482- });
483- content = content.replace(/{{idle_duration}}/gi, () => getTimeSinceLastMessage());
484- content = content.replace(/{{time_UTC([-+]\d+)}}/gi, (_, offset) => {
485- const utcOffset = parseInt(offset, 10);
486- const utcTime = moment().utc().utcOffset(utcOffset).format('LT');
487- return utcTime;
488- });
489- content = timeDiffReplace(content);
490- content = bannedWordsReplace(content);
491- content = randomReplace(content);
492- content = pickReplace(content, rawContent);
493520 return content;
494521}
public/scripts/variables.js+19 -69
@@ -223,85 +223,34 @@ export function resolveVariable(name, scope = null) {
223223 return name;
224224}
225225
226-export function replaceVariableMacros(input) {
226+/**
227- const lines = input.split('\n');
227+ * @returns {import('./macros.js').Macro[]}
228-
228+ */
229- for (let i = 0; i < lines.length; i++) {
229+export function getVariableMacros() {
230230 let const linemacros = lines[i];
231-
232- // Skip lines without macros
233- if (!line || !line.includes('{{')) {
234- continue;
235- }
236-
237231 // Replace {{getvar::name}} with the value of the variable name
238232 line{ =regex: line.replace(/{{getvar::([^}]+)}}/gi, replace: (_, name) => {getLocalVariable(name.trim()) },
239- name = name.trim();
240- return getLocalVariable(name);
241- });
242-
243233 // Replace {{setvar::name::value}} with empty string and set the variable name to value
244234 line{ =regex: line.replace(/{{setvar::([^:]+)::([^}]+)}}/gi, replace: (_, name, value) => { setLocalVariable(name.trim(), value); return ''; } },
245- name = name.trim();
246- setLocalVariable(name, value);
247- return '';
248- });
249-
250235 // Replace {{addvar::name::value}} with empty string and add value to the variable value
251236 line{ =regex: line.replace(/{{addvar::([^:]+)::([^}]+)}}/gi, replace: (_, name, value) => { addLocalVariable(name.trim(), value); return ''; } },
252- name = name.trim();
253- addLocalVariable(name, value);
254- return '';
255- });
256-
257237 // Replace {{incvar::name}} with empty string and increment the variable name by 1
258238 line{ =regex: line.replace(/{{incvar::([^}]+)}}/gi, replace: (_, name) => {incrementLocalVariable(name.trim()) },
259- name = name.trim();
260- return incrementLocalVariable(name);
261- });
262-
263239 // Replace {{decvar::name}} with empty string and decrement the variable name by 1
264240 line{ =regex: line.replace(/{{decvar::([^}]+)}}/gi, replace: (_, name) => {decrementLocalVariable(name.trim()) },
265- name = name.trim();
266- return decrementLocalVariable(name);
267- });
268-
269241 // Replace {{getglobalvar::name}} with the value of the global variable name
270242 line{ =regex: line.replace(/{{getglobalvar::([^}]+)}}/gi, replace: (_, name) => {getGlobalVariable(name.trim()) },
271- name = name.trim();
272- return getGlobalVariable(name);
273- });
274-
275243 // Replace {{setglobalvar::name::value}} with empty string and set the global variable name to value
276244 line{ =regex: line.replace(/{{setglobalvar::([^:]+)::([^}]+)}}/gi, replace: (_, name, value) => { setGlobalVariable(name.trim(), value); return ''; } },
277- name = name.trim();
278- setGlobalVariable(name, value);
279- return '';
280- });
281-
282245 // Replace {{addglobalvar::name::value}} with empty string and add value to the global variable value
283246 line{ =regex: line.replace(/{{addglobalvar::([^:]+)::([^}]+)}}/gi, replace: (_, name, value) => { addGlobalVariable(name.trim(), value); return ''; } },
284- name = name.trim();
285- addGlobalVariable(name, value);
286- return '';
287- });
288-
289247 // Replace {{incglobalvar::name}} with empty string and increment the global variable name by 1
290248 line{ =regex: line.replace(/{{incglobalvar::([^}]+)}}/gi, replace: (_, name) => {incrementGlobalVariable(name.trim()) },
291- name = name.trim();
292- return incrementGlobalVariable(name);
293- });
294-
295249 // Replace {{decglobalvar::name}} with empty string and decrement the global variable name by 1
296250 line{ =regex: line.replace(/{{decglobalvar::([^}]+)}}/gi, replace: (_, name) => {decrementGlobalVariable(name.trim()) },
297- name = name.trim();
251+ ];
298- return decrementGlobalVariable(name);
299- });
300-
301- lines[i] = line;
302- }
303252
304- return lines.join('\n');
253+ return macros;
305254}
306255
307256async function listVariablesCallback(args) {
@@ -2148,7 +2097,8 @@ export function registerVariableCommands() {
21482097 callback: sortArrayObjectCallback,
21492098 returns: 'the sorted list or dictionary keys',
21502099 namedArgumentList: [
21512100 SlashCommandNamedArgument.fromProps({ name: 'keysort',
2101+ name: 'keysort',
21522102 description: 'whether to sort by key or value; ignored for lists',
21532103 typeList: [ARGUMENT_TYPE.BOOLEAN],
21542104 enumList: ['true', 'false'],