1
import { chevrotain } from '../../../lib.js';
2
const { createToken, Lexer } = chevrotain;
3

4
/** @typedef {import('chevrotain').TokenType} TokenType */
5

6

7
/** Regex for lexer token matching (no anchors). */
8
const IDENTIFIER_LEXER_PATTERN = /[a-zA-Z][\w-_]*/;
9

10
/**
11
 * Pattern for valid macro identifiers.
12
 * Must start with a letter, followed by word chars (letters, digits, underscore) or hyphens.
13
 * Used by both the lexer token and the validation regex.
14
 *
15
 * Regex for full-string validation (with anchors). Exported for macro registration.
16
 */
17
export const MACRO_IDENTIFIER_PATTERN = /^[a-zA-Z][\w-_]*$/;
18

19
/**
20
 * Pattern for valid variable shorthand identifiers.
21
 * Must start with a letter, followed by word chars (letters, digits, underscore) or hyphens,
22
 * but must end with a word character (not a hyphen).
23
 *
24
 * Used for variable shorthand syntax like .varName or $varName.
25
 */
26
export const MACRO_VARIABLE_SHORTHAND_PATTERN = /[a-zA-Z](?:[\w\-_]*[\w])?/;
27

28
/** @enum {string} */
29
const modes = Object.freeze({
30
    plaintext: 'plaintext_mode',
31
    macro_def: 'macro_def_mode',
32
    macro_identifier_end: 'macro_identifier_end_mode',
33
    macro_args: 'macro_args_mode',
34
    macro_filter_modifer: 'macro_filter_modifer_mode',
35
    macro_filter_modifier_end: 'macro_filter_modifier_end_mode',
36
    // Variable shorthand modes
37
    var_identifier: 'var_identifier_mode',
38
    var_after_identifier: 'var_after_identifier_mode',
39
    var_value: 'var_value_mode',
40
});
41

42
/**
43
 * All lexer tokens used by the macro parser.
44
 * @readonly
45
 */
46
const Tokens = Object.freeze({
47
/** General capture-all plaintext without macros. Consumes any character that is not the first '{' of a macro opener '{{'. */
48
    Plaintext: createToken({ name: 'Plaintext', pattern: /(?:[^{]|\{(?!\{))+/u, line_breaks: true }),
49
    /** Single literal '{' that appears immediately before a macro opener '{{' */
50
    PlaintextOpenBrace: createToken({ name: 'Plaintext.OpenBrace', pattern: /\{(?=\{\{)/ }),
51

52
    /** General macro capture */
53
    Macro: {
54
        Start: createToken({ name: 'Macro.Start', pattern: /\{\{/ }),
55
        /**
56
         * Macro execution flags - special symbols that modify macro resolution behavior.
57
         * - `!` = immediate resolve (TBD)
58
         * - `?` = delayed resolve (TBD)
59
         * - `~` = re-evaluate (TBD)
60
         * - `/` = closing block marker for scoped macros
61
         * - `#` = preserve whitespace (don't auto-trim scoped content), also legacy handlebars compatibility
62
         */
63
        Flags: createToken({ name: 'Macro.Flag', pattern: /[!?~#/]/ }),
64
        /**
65
         * Filter flag (`>`) - separate token because it changes parsing behavior.
66
         * When present, `|` characters inside the macro are treated as filter/pipe operators.
67
         */
68
        FilterFlag: createToken({ name: 'Macro.FilterFlag', pattern: />/ }),
69
        DoubleSlash: createToken({ name: 'Macro.DoubleSlash', pattern: /\/\// }),
70
        /**
71
         * Separate macro identifier needed, that is similar to the global indentifier, but captures the actual macro "name"
72
         * We need this, because this token is going to switch lexer mode, while the general identifier does not.
73
         */
74
        Identifier: createToken({ name: 'Macro.Identifier', pattern: IDENTIFIER_LEXER_PATTERN }),
75
        /** At the end of an identifier, there has to be whitspace, or must be directly followed by colon/double-colon separator, output modifier or closing braces */
76
        EndOfIdentifier: createToken({ name: 'Macro.EndOfIdentifier', pattern: /(?:\s+|(?=:{1,2})|(?=[|}]))/, group: Lexer.SKIPPED }),
77
        BeforeEnd: createToken({ name: 'Macro.BeforeEnd', pattern: /(?=\}\})/, group: Lexer.SKIPPED }),
78
        End: createToken({ name: 'Macro.End', pattern: /\}\}/ }),
79
    },
80

81
    /** Captures that only appear inside arguments */
82
    Args: {
83
        DoubleColon: createToken({ name: 'Args.DoubleColon', pattern: /::/ }),
84
        Colon: createToken({ name: 'Args.Colon', pattern: /:/ }),
85
        Equals: createToken({ name: 'Args.Equals', pattern: /=/ }),
86
        Quote: createToken({ name: 'Args.Quote', pattern: /"/ }),
87
    },
88

89
    Filter: {
90
        EscapedPipe: createToken({ name: 'Filter.EscapedPipe', pattern: /\\\|/ }),
91
        Pipe: createToken({ name: 'Filter.Pipe', pattern: /\|/ }),
92
        Identifier: createToken({ name: 'Filter.Identifier', pattern: IDENTIFIER_LEXER_PATTERN }),
93
        /** At the end of an identifier, there has to be whitspace, or must be directly followed by colon/double-colon separator, output modifier or closing braces */
94
        EndOfIdentifier: createToken({ name: 'Filter.EndOfIdentifier', pattern: /(?:\s+|(?=:{1,2})|(?=[|}]))/, group: Lexer.SKIPPED }),
95
    },
96

97
    // All tokens that can be captured inside a macro
98
    Identifier: createToken({ name: 'Identifier', pattern: IDENTIFIER_LEXER_PATTERN }),
99
    WhiteSpace: createToken({ name: 'WhiteSpace', pattern: /\s+/, group: Lexer.SKIPPED }),
100

101
    /** Variable shorthand tokens */
102
    Var: {
103
        /** Local variable prefix (`.`) - triggers variable shorthand for local variables */
104
        LocalPrefix: createToken({ name: 'Var.LocalPrefix', pattern: /\./ }),
105
        /** Global variable prefix (`$`) - triggers variable shorthand for global variables */
106
        GlobalPrefix: createToken({ name: 'Var.GlobalPrefix', pattern: /\$/ }),
107
        /**
108
         * Variable identifier - allows hyphens inside but not at the end to avoid conflict with -- operator.
109
         * Pattern: starts with letter, optionally followed by word chars/hyphens, but must end with word char.
110
         * Examples: myVar, my-var, my_var, myVar123, my-long-var-name
111
         * Invalid: my-, my--, -var
112
         */
113
        Identifier: createToken({ name: 'Var.Identifier', pattern: MACRO_VARIABLE_SHORTHAND_PATTERN }),
114

115
        /** All tokens that are valid operators inside a variable shorthand expression */
116
        Operators: {
117
            /** Increment operator (`++`) */
118
            Increment: createToken({ name: 'Var.Increment', pattern: /\+\+/ }),
119
            /** Decrement operator (`--`) */
120
            Decrement: createToken({ name: 'Var.Decrement', pattern: /--/ }),
121
            /** Nullish coalescing assignment operator (`??=`) - sets var if undefined, must come before NullishCoalescing */
122
            NullishCoalescingEquals: createToken({ name: 'Var.NullishCoalescingEquals', pattern: /\?\?=/ }),
123
            /** Nullish coalescing operator (`??`) - returns default if var undefined */
124
            NullishCoalescing: createToken({ name: 'Var.NullishCoalescing', pattern: /\?\?/ }),
125
            /** Logical OR assignment operator (`||=`) - sets var if falsy, must come before LogicalOr */
126
            LogicalOrEquals: createToken({ name: 'Var.LogicalOrEquals', pattern: /\|\|=/ }),
127
            /** Logical OR operator (`||`) - returns default if var falsy */
128
            LogicalOr: createToken({ name: 'Var.LogicalOr', pattern: /\|\|/ }),
129
            /** Subtract operator (`-=`) - subtracts value from variable */
130
            MinusEquals: createToken({ name: 'Var.MinusEquals', pattern: /-=/ }),
131
            /** Equality comparison operator (`==`) - compares variable to value */
132
            DoubleEquals: createToken({ name: 'Var.DoubleEquals', pattern: /==/ }),
133
            /** Not equals comparison operator (`!=`) - compares variable to value, returns inverted result */
134
            NotEquals: createToken({ name: 'Var.NotEquals', pattern: /!=/ }),
135
            /** Greater than or equal comparison operator (`>=`) - must come before GreaterThan */
136
            GreaterThanOrEqual: createToken({ name: 'Var.GreaterThanOrEqual', pattern: />=/ }),
137
            /** Greater than comparison operator (`>`) */
138
            GreaterThan: createToken({ name: 'Var.GreaterThan', pattern: />/ }),
139
            /** Less than or equal comparison operator (`<=`) - must come before LessThan */
140
            LessThanOrEqual: createToken({ name: 'Var.LessThanOrEqual', pattern: /<=/ }),
141
            /** Less than comparison operator (`<`) */
142
            LessThan: createToken({ name: 'Var.LessThan', pattern: /</ }),
143
            /** Add/append operator (`+=`) - must come before Equals to avoid conflict */
144
            PlusEquals: createToken({ name: 'Var.PlusEquals', pattern: /\+=/ }),
145
            /** Set operator (`=`) */
146
            Equals: createToken({ name: 'Var.Equals', pattern: /=/ }),
147
        },
148
    },
149

150
    /**
151
     * Capture unknown characters one by one, to still allow other tokens being matched once they are there.
152
     * This includes any possible braces that is not the double closing braces as MacroEnd.
153
     */
154
    Unknown: createToken({ name: 'Unknown', pattern: /([^}]|\}(?!\}))/ }),
155

156
    /** TODO: Capture-all rest for now, that is not the macro end or opening of a new macro. Might be replaced later down the line. */
157
    Text: createToken({ name: 'Text', pattern: /.+(?=\}\}|\{\{)/, line_breaks: true }),
158

159
    /**
160
     * DANGER ZONE: Careful with this token. This is used as a way to pop the current mode, if no other token matches.
161
     * Can be used in modes that don't have a "defined" end really, like when capturing a single argument, argument list, etc.
162
     * Has to ALWAYS be the last token.
163
     */
164
    ModePopper: createToken({ name: 'ModePopper', pattern: () => [''], line_breaks: false, group: Lexer.SKIPPED }),
165
});
166

167
/** @type {Map<string,string>} Saves all token definitions that are marked as entering modes */
168
const enterModesMap = new Map();
169

170
/**
171
 * Lexer definition object that maps states/modes to their token rules.
172
 * Each mode defines which tokens are valid in that context and how to transition between modes.
173
 * @readonly
174
 */
175
const Def = {
176
    modes: {
177
        [modes.plaintext]: [
178
            using(Tokens.Plaintext),
179
            using(Tokens.PlaintextOpenBrace),
180
            enter(Tokens.Macro.Start, modes.macro_def),
181
        ],
182
        [modes.macro_def]: [
183
            exits(Tokens.Macro.End, modes.macro_def),
184

185
            // An explicit double-slash will be treated above flags to consume, as it'll introduce a comment macro. Directly following is the args then.
186
            enter(Tokens.Macro.DoubleSlash, modes.macro_args),
187

188
            // Variable shorthand prefixes - must come before flags to take precedence
189
            // These enter the variable identifier mode to parse variable expressions
190
            enter(Tokens.Var.LocalPrefix, modes.var_identifier),
191
            enter(Tokens.Var.GlobalPrefix, modes.var_identifier),
192

193
            using(Tokens.Macro.Flags),
194
            // Filter flag is separate because it affects parsing behavior for pipes
195
            using(Tokens.Macro.FilterFlag),
196

197
            // We allow whitspaces inbetween flags or in front of the modifier
198
            using(Tokens.WhiteSpace),
199

200
            // Inside a macro, we will match the identifier
201
            // Enter 'macro_identifier_end' mode automatically at the end of the identifier, so we don't match more than one identifier
202
            enter(Tokens.Macro.Identifier, modes.macro_identifier_end),
203

204
            // If none of the tokens above are found, this is an invalid macro at runtime.
205
            // We still need to exit the mode to prevent lexer errors
206
            exits(Tokens.ModePopper, modes.macro_def),
207
        ],
208
        [modes.macro_identifier_end]: [
209
            // Valid options after a macro identifier: whitespace, colon/double-colon (captured), macro end braces, or output modifier pipe.
210
            exits(Tokens.Macro.BeforeEnd, modes.macro_identifier_end),
211
            enter(Tokens.Macro.EndOfIdentifier, modes.macro_args, { andExits: modes.macro_identifier_end }),
212
        ],
213
        [modes.macro_args]: [
214
            // Macro args allow nested macros
215
            enter(Tokens.Macro.Start, modes.macro_def),
216

217
            // We allow escaped pipes to not start output modifiers. We need to capture this first, before the pipe
218
            using(Tokens.Filter.EscapedPipe),
219

220
            // If at any place during args writing there is a pipe, we lex it as an output identifier, and then continue with lex its args
221
            enter(Tokens.Filter.Pipe, modes.macro_filter_modifer),
222

223
            using(Tokens.Args.DoubleColon),
224
            using(Tokens.Args.Colon),
225
            using(Tokens.Args.Equals),
226
            using(Tokens.Args.Quote),
227
            using(Tokens.Identifier),
228

229
            using(Tokens.WhiteSpace),
230

231
            // Last fallback, before we need to exit the mode, as we might have characters we (wrongly) haven't defined yet
232
            using(Tokens.Unknown),
233

234
            // Args are optional, and we don't know how long, so exit the mode to be able to capture the actual macro end
235
            exits(Tokens.ModePopper, modes.macro_args),
236
        ],
237
        [modes.macro_filter_modifer]: [
238
            using(Tokens.WhiteSpace),
239

240
            enter(Tokens.Filter.Identifier, modes.macro_filter_modifier_end, { andExits: modes.macro_filter_modifer }),
241
        ],
242
        [modes.macro_filter_modifier_end]: [
243
            // Valid options after a filter itenfier: whitespace, colon/double-colon (captured), macro end braces, or output modifier pipe.
244
            exits(Tokens.Macro.BeforeEnd, modes.macro_identifier_end),
245
            exits(Tokens.Filter.EndOfIdentifier, modes.macro_filter_modifer),
246
        ],
247

248
        // After seeing `.` or `$`, expect a variable identifier
249
        [modes.var_identifier]: [
250
            using(Tokens.WhiteSpace),
251
            // Consume the variable identifier and move to operator detection
252
            enter(Tokens.Var.Identifier, modes.var_after_identifier, { andExits: modes.var_identifier }),
253
            // If no valid identifier found, exit back (will result in parser error)
254
            exits(Tokens.ModePopper, modes.var_identifier),
255
        ],
256
        // After the variable identifier, look for operators or end
257
        [modes.var_after_identifier]: [
258
            using(Tokens.WhiteSpace),
259
            // Check for operators - order matters: longer patterns first
260
            using(Tokens.Var.Operators.Increment),
261
            using(Tokens.Var.Operators.Decrement),
262
            enter(Tokens.Var.Operators.NullishCoalescingEquals, modes.var_value, { andExits: modes.var_after_identifier }),
263
            enter(Tokens.Var.Operators.NullishCoalescing, modes.var_value, { andExits: modes.var_after_identifier }),
264
            enter(Tokens.Var.Operators.LogicalOrEquals, modes.var_value, { andExits: modes.var_after_identifier }),
265
            enter(Tokens.Var.Operators.LogicalOr, modes.var_value, { andExits: modes.var_after_identifier }),
266
            enter(Tokens.Var.Operators.MinusEquals, modes.var_value, { andExits: modes.var_after_identifier }),
267
            enter(Tokens.Var.Operators.DoubleEquals, modes.var_value, { andExits: modes.var_after_identifier }),
268
            enter(Tokens.Var.Operators.NotEquals, modes.var_value, { andExits: modes.var_after_identifier }),
269
            enter(Tokens.Var.Operators.GreaterThanOrEqual, modes.var_value, { andExits: modes.var_after_identifier }),
270
            enter(Tokens.Var.Operators.GreaterThan, modes.var_value, { andExits: modes.var_after_identifier }),
271
            enter(Tokens.Var.Operators.LessThanOrEqual, modes.var_value, { andExits: modes.var_after_identifier }),
272
            enter(Tokens.Var.Operators.LessThan, modes.var_value, { andExits: modes.var_after_identifier }),
273
            enter(Tokens.Var.Operators.PlusEquals, modes.var_value, { andExits: modes.var_after_identifier }),
274
            enter(Tokens.Var.Operators.Equals, modes.var_value, { andExits: modes.var_after_identifier }),
275
            // If we see the end, exit
276
            exits(Tokens.Macro.BeforeEnd, modes.var_after_identifier),
277
            // Fallback exit
278
            exits(Tokens.ModePopper, modes.var_after_identifier),
279
        ],
280
        // After `=` or `+=`, capture the value (can contain nested macros)
281
        [modes.var_value]: [
282
            // Nested macros in value
283
            enter(Tokens.Macro.Start, modes.macro_def),
284

285
            using(Tokens.Identifier),
286
            using(Tokens.WhiteSpace),
287
            using(Tokens.Unknown),
288

289
            // Exit when we're about to see the end
290
            exits(Tokens.ModePopper, modes.var_value),
291
        ],
292
    },
293
    defaultMode: modes.plaintext,
294
};
295

296
/**
297
 * The singleton instance of the MacroLexer.
298
 *
299
 * @type {MacroLexer}
300
 */
301
let instance;
302
export { instance as MacroLexer };
303

304
class MacroLexer extends Lexer {
305
    /** @type {MacroLexer} */ static #instance;
306
    /** @type {MacroLexer} */ static get instance() { return MacroLexer.#instance ?? (MacroLexer.#instance = new MacroLexer()); }
307

308
    // Define the tokens
309
    /** @readonly */ static tokens = Tokens;
310
    /** @readonly */ static def = Def;
311
    /** @readonly */ tokens = Tokens;
312
    /** @readonly */ def = MacroLexer.def;
313

314
    /** @private */
315
    constructor() {
316
        super(MacroLexer.def, {
317
            traceInitPerf: false,
318
        });
319
    }
320

321
    test(input) {
322
        const result = this.tokenize(input);
323
        return {
324
            errors: result.errors,
325
            groups: result.groups,
326
            tokens: result.tokens.map(({ tokenType, ...rest }) => ({ type: tokenType.name, ...rest, tokenType: tokenType })),
327
        };
328
    }
329
}
330

331
instance = MacroLexer.instance;
332

333
/**
334
 * [Utility]
335
 * Set push mode on the token definition.
336
 * Can be used inside the token mode definition block.
337
 *
338
 * Marks the token to **enter** the following lexer mode.
339
 *
340
 * Optionally, you can specify the modes to exit when entering this mode.
341
 *
342
 * @param {TokenType} token - The token to modify
343
 * @param {string} mode - The mode to set
344
 * @param {object} [options={}] - Additional options
345
 * @param {string} [options.andExits] - The modes to exit when entering this mode
346
 * @returns {TokenType} The token again
347
 */
348
function enter(token, mode, { andExits = undefined } = {}) {
349
    if (!token) throw new Error('Token must not be undefined');
350
    if (enterModesMap.has(token.name) && enterModesMap.get(token.name) !== mode) {
351
        throw new Error(`Token ${token.name} already is set to enter mode ${enterModesMap.get(token.name)}. The token definition are global, so they cannot be used to lead to different modes.`);
352
    }
353

354
    if (andExits) exits(token, andExits);
355

356
    token.PUSH_MODE = mode;
357
    enterModesMap.set(token.name, mode);
358
    return token;
359
}
360

361
/**
362
 * [Utility]
363
 * Set pop mode on the token definition.
364
 * Can be used inside the token mode definition block.
365
 *
366
 * Marks the token to **exit** the following lexer mode.
367
 *
368
 * @param {TokenType} token - The token to modify
369
 * @param {string} mode - The mode to leave
370
 * @returns {TokenType} The token again
371
 */
372
function exits(token, mode) {
373
    if (!token) throw new Error('Token must not be undefined');
374
    token.POP_MODE = !!mode; // Always set to true. We just use the mode here, so the linter thinks it was used. We just pass it in for clarity in the definition
375
    return token;
376
}
377

378
/**
379
 * [Utility]
380
 * Can be used inside the token mode definition block.
381
 *
382
 * Marks the token to to just be used/consumed, and not exit or enter a mode.
383
 *
384
 * @param {TokenType} token - The token to modify
385
 * @returns {TokenType} The token again
386
 */
387
function using(token) {
388
    if (!token) throw new Error('Token must not be undefined');
389
    if (enterModesMap.has(token.name)) {
390
        throw new Error(`Token ${token.name} is already marked to enter a mode (${enterModesMap.get(token.name)}). The token definition are global, so they cannot be used to lead or stay differently.`);
391
    }
392
    return token;
393
}