1
import { test, expect } from '@playwright/test';
2
import { testSetup } from './frontent-test-utils.js';
3

4
/** @typedef {import('chevrotain').ILexingResult} ILexingResult */
5
/** @typedef {import('chevrotain').ILexingError} ILexingError */
6
/** @typedef {{type: string, text: string}} TestableToken */
7

8
test.describe('MacroLexer', () => {
9
    // Currently this test suits runs without ST context. Enable, if ever needed
10
    test.beforeEach(testSetup.goST);
11

12
    test.describe('General Macro', () => {
13
        // {{user}}
14
        test('should handle macro only', async ({ page }) => {
15
            const input = '{{user}}';
16
            const tokens = await runLexerGetTokens(page, input);
17

18
            const expectedTokens = [
19
                { type: 'Macro.Start', text: '{{' },
20
                { type: 'Macro.Identifier', text: 'user' },
21
                { type: 'Macro.End', text: '}}' },
22
            ];
23

24
            expect(tokens).toEqual(expectedTokens);
25
        });
26
        // {{}}
27
        test('should handle empty macro', async ({ page }) => {
28
            const input = '{{}}';
29
            const tokens = await runLexerGetTokens(page, input);
30

31
            const expectedTokens = [
32
                { type: 'Macro.Start', text: '{{' },
33
                { type: 'Macro.End', text: '}}' },
34
            ];
35

36
            expect(tokens).toEqual(expectedTokens);
37
        });
38
        // {{   user   }}
39
        test('should handle macro with leading and trailing whitespace inside', async ({ page }) => {
40
            const input = '{{   user   }}';
41
            const tokens = await runLexerGetTokens(page, input);
42

43
            const expectedTokens = [
44
                { type: 'Macro.Start', text: '{{' },
45
                { type: 'Macro.Identifier', text: 'user' },
46
                { type: 'Macro.End', text: '}}' },
47
            ];
48

49
            expect(tokens).toEqual(expectedTokens);
50
        });
51
        // {{macro1}}{{macro2}}
52
        test('should handle multiple sequential macros', async ({ page }) => {
53
            const input = '{{macro1}}{{macro2}}';
54
            const tokens = await runLexerGetTokens(page, input);
55

56
            const expectedTokens = [
57
                { type: 'Macro.Start', text: '{{' },
58
                { type: 'Macro.Identifier', text: 'macro1' },
59
                { type: 'Macro.End', text: '}}' },
60
                { type: 'Macro.Start', text: '{{' },
61
                { type: 'Macro.Identifier', text: 'macro2' },
62
                { type: 'Macro.End', text: '}}' },
63
            ];
64

65
            expect(tokens).toEqual(expectedTokens);
66
        });
67
    });
68

69
    test.describe('Macro Nesting', () => {
70
        // {{outerMacro {{innerMacro}}}}
71
        test('should handle nested macros', async ({ page }) => {
72
            const input = '{{outerMacro {{innerMacro}}}}';
73
            const tokens = await runLexerGetTokens(page, input);
74

75
            const expectedTokens = [
76
                { type: 'Macro.Start', text: '{{' },
77
                { type: 'Macro.Identifier', text: 'outerMacro' },
78
                { type: 'Macro.Start', text: '{{' },
79
                { type: 'Macro.Identifier', text: 'innerMacro' },
80
                { type: 'Macro.End', text: '}}' },
81
                { type: 'Macro.End', text: '}}' },
82
            ];
83

84
            expect(tokens).toEqual(expectedTokens);
85
        });
86
        // {{doStuff "inner {{nested}} string"}}
87
        test('should handle macros with nested quotation marks', async ({ page }) => {
88
            const input = '{{doStuff "inner {{nested}} string"}}';
89
            const tokens = await runLexerGetTokens(page, input);
90

91
            const expectedTokens = [
92
                { type: 'Macro.Start', text: '{{' },
93
                { type: 'Macro.Identifier', text: 'doStuff' },
94
                { type: 'Args.Quote', text: '"' },
95
                { type: 'Identifier', text: 'inner' },
96
                { type: 'Macro.Start', text: '{{' },
97
                { type: 'Macro.Identifier', text: 'nested' },
98
                { type: 'Macro.End', text: '}}' },
99
                { type: 'Identifier', text: 'string' },
100
                { type: 'Args.Quote', text: '"' },
101
                { type: 'Macro.End', text: '}}' },
102
            ];
103

104
            expect(tokens).toEqual(expectedTokens);
105
        });
106
    });
107

108
    test.describe('Macro Identifier', () => {
109
        // {{ a }}
110
        test('should allow one-character macro identifiers', async ({ page }) => {
111
            const input = '{{ a }}';
112
            const tokens = await runLexerGetTokens(page, input);
113

114
            const expectedTokens = [
115
                { type: 'Macro.Start', text: '{{' },
116
                { type: 'Macro.Identifier', text: 'a' },
117
                { type: 'Macro.End', text: '}}' },
118
            ];
119

120
            expect(tokens).toEqual(expectedTokens);
121
        });
122
        // {{ some macro }}
123
        test('should only capture the first identifier as macro identifier when there are whitespaces between two valid identifiers', async ({ page }) => {
124
            const input = '{{ some macro }}';
125
            const tokens = await runLexerGetTokens(page, input);
126

127
            const expectedTokens = [
128
                { type: 'Macro.Start', text: '{{' },
129
                { type: 'Macro.Identifier', text: 'some' },
130
                { type: 'Identifier', text: 'macro' },
131
                { type: 'Macro.End', text: '}}' },
132
            ];
133

134
            expect(tokens).toEqual(expectedTokens);
135
        });
136
        // {{my2cents}}
137
        test('should allow numerics inside the macro identifier', async ({ page }) => {
138
            const input = '{{my2cents}}';
139
            const tokens = await runLexerGetTokens(page, input);
140

141
            const expectedTokens = [
142
                { type: 'Macro.Start', text: '{{' },
143
                { type: 'Macro.Identifier', text: 'my2cents' },
144
                { type: 'Macro.End', text: '}}' },
145
            ];
146

147
            expect(tokens).toEqual(expectedTokens);
148
        });
149
        // {{SCREAM}}
150
        test('should allow capslock macro', async ({ page }) => {
151
            const input = '{{SCREAM}}';
152
            const tokens = await runLexerGetTokens(page, input);
153

154
            const expectedTokens = [
155
                { type: 'Macro.Start', text: '{{' },
156
                { type: 'Macro.Identifier', text: 'SCREAM' },
157
                { type: 'Macro.End', text: '}}' },
158
            ];
159

160
            expect(tokens).toEqual(expectedTokens);
161
        });
162
        // {{some-longer-macro}}
163
        test('should allow dashes in macro identifiers', async ({ page }) => {
164
            const input = '{{some-longer-macro}}';
165
            const tokens = await runLexerGetTokens(page, input);
166

167
            const expectedTokens = [
168
                { type: 'Macro.Start', text: '{{' },
169
                { type: 'Macro.Identifier', text: 'some-longer-macro' },
170
                { type: 'Macro.End', text: '}}' },
171
            ];
172

173
            expect(tokens).toEqual(expectedTokens);
174
        });
175
        // {{legacy_macro}}
176
        test('should allow underscores as legacy in macro identifiers', async ({ page }) => {
177
            const input = '{{legacy_macro}}';
178
            const tokens = await runLexerGetTokens(page, input);
179

180
            const expectedTokens = [
181
                { type: 'Macro.Start', text: '{{' },
182
                { type: 'Macro.Identifier', text: 'legacy_macro' },
183
                { type: 'Macro.End', text: '}}' },
184
            ];
185

186
            expect(tokens).toEqual(expectedTokens);
187
        });
188

189
        test.describe('Error Cases (Macro Identifier)', () => {
190
            // {{macro!@#%}}
191
            test('[Error] should not lex special characters as part of the macro identifier', async ({ page }) => {
192
                const input = '{{macro!@#%}}';
193
                const { tokens, errors } = await runLexerGetTokensAndErrors(page, input);
194

195
                const expectedErrors = [
196
                    { message: 'unexpected character: ->!<- at offset: 7, skipped 4 characters.' },
197
                ];
198

199
                expect(errors).toMatchObject(expectedErrors);
200

201
                const expectedTokens = [
202
                    { type: 'Macro.Start', text: '{{' },
203
                    { type: 'Macro.Identifier', text: 'macro' },
204
                    // Do not lex the wrong characters
205
                    { type: 'Macro.End', text: '}}' },
206
                ];
207

208
                expect(tokens).toEqual(expectedTokens);
209
            });
210
            // {{ma!@#%ro}}
211
            test('[Error] should not parse invalid chars in macro identifier as valid macro identifier', async ({ page }) => {
212
                const input = '{{ma!@#%ro}}';
213
                const { tokens, errors } = await runLexerGetTokensAndErrors(page, input);
214

215
                const expectedErrors = [
216
                    { message: 'unexpected character: ->!<- at offset: 4, skipped 6 characters.' },
217
                ];
218

219
                expect(errors).toMatchObject(expectedErrors);
220

221
                const expectedTokens = [
222
                    { type: 'Macro.Start', text: '{{' },
223
                    { type: 'Macro.Identifier', text: 'ma' },
224
                    // Do not lex the wrong characters
225
                    { type: 'Macro.End', text: '}}' },
226
                ];
227

228
                expect(tokens).toEqual(expectedTokens);
229
            });
230
        });
231
    });
232

233
    test.describe('Macro Arguments', () => {
234
        // {{setvar::myVar::This is Sparta!}}
235
        test('should tokenize macros with double colons arguments correctly', async ({ page }) => {
236
            const input = '{{setvar::myVar::This is Sparta!}}';
237
            const tokens = await runLexerGetTokens(page, input);
238

239
            const expectedTokens = [
240
                { type: 'Macro.Start', text: '{{' },
241
                { type: 'Macro.Identifier', text: 'setvar' },
242
                { type: 'Args.DoubleColon', text: '::' },
243
                { type: 'Identifier', text: 'myVar' },
244
                { type: 'Args.DoubleColon', text: '::' },
245
                { type: 'Identifier', text: 'This' },
246
                { type: 'Identifier', text: 'is' },
247
                { type: 'Identifier', text: 'Sparta' },
248
                { type: 'Unknown', text: '!' },
249
                { type: 'Macro.End', text: '}}' },
250
            ];
251

252
            expect(tokens).toEqual(expectedTokens);
253
        });
254
        // {{doStuff key=MyValue another=AnotherValue}}
255
        test('should handle named arguments with key=value syntax', async ({ page }) => {
256
            const input = '{{doStuff key=MyValue another=AnotherValue}}';
257
            const tokens = await runLexerGetTokens(page, input);
258

259
            const expectedTokens = [
260
                { type: 'Macro.Start', text: '{{' },
261
                { type: 'Macro.Identifier', text: 'doStuff' },
262
                { type: 'Identifier', text: 'key' },
263
                { type: 'Args.Equals', text: '=' },
264
                { type: 'Identifier', text: 'MyValue' },
265
                { type: 'Identifier', text: 'another' },
266
                { type: 'Args.Equals', text: '=' },
267
                { type: 'Identifier', text: 'AnotherValue' },
268
                { type: 'Macro.End', text: '}}' },
269
            ];
270

271
            expect(tokens).toEqual(expectedTokens);
272
        });
273
        // {{getvar key="My variable"}}
274
        test('should handle named arguments with quotation marks', async ({ page }) => {
275
            const input = '{{getvar key="My variable"}}';
276
            const tokens = await runLexerGetTokens(page, input);
277

278
            const expectedTokens = [
279
                { type: 'Macro.Start', text: '{{' },
280
                { type: 'Macro.Identifier', text: 'getvar' },
281
                { type: 'Identifier', text: 'key' },
282
                { type: 'Args.Equals', text: '=' },
283
                { type: 'Args.Quote', text: '"' },
284
                { type: 'Identifier', text: 'My' },
285
                { type: 'Identifier', text: 'variable' },
286
                { type: 'Args.Quote', text: '"' },
287
                { type: 'Macro.End', text: '}}' },
288
            ];
289

290
            expect(tokens).toEqual(expectedTokens);
291
        });
292
        // {{getvar KEY=big}}
293
        test('should handle capslock argument name identifiers', async ({ page }) => {
294
            const input = '{{getvar KEY=big}}';
295
            const tokens = await runLexerGetTokens(page, input);
296

297
            const expectedTokens = [
298
                { type: 'Macro.Start', text: '{{' },
299
                { type: 'Macro.Identifier', text: 'getvar' },
300
                { type: 'Identifier', text: 'KEY' },
301
                { type: 'Args.Equals', text: '=' },
302
                { type: 'Identifier', text: 'big' },
303
                { type: 'Macro.End', text: '}}' },
304
            ];
305

306
            expect(tokens).toEqual(expectedTokens);
307
        });
308
        // {{dostuff longer-key=value}}
309
        test('should handle argument name identifiers with dashes', async ({ page }) => {
310
            const input = '{{dostuff longer-key=value}}';
311
            const tokens = await runLexerGetTokens(page, input);
312

313
            const expectedTokens = [
314
                { type: 'Macro.Start', text: '{{' },
315
                { type: 'Macro.Identifier', text: 'dostuff' },
316
                { type: 'Identifier', text: 'longer-key' },
317
                { type: 'Args.Equals', text: '=' },
318
                { type: 'Identifier', text: 'value' },
319
                { type: 'Macro.End', text: '}}' },
320
            ];
321

322
            expect(tokens).toEqual(expectedTokens);
323
        });
324
        // {{macro legacy_key=blah}}
325
        test('should handle legacy argument name identifiers', async ({ page }) => {
326
            const input = '{{macro legacy_key=blah}}';
327
            const tokens = await runLexerGetTokens(page, input);
328

329
            const expectedTokens = [
330
                { type: 'Macro.Start', text: '{{' },
331
                { type: 'Macro.Identifier', text: 'macro' },
332
                { type: 'Identifier', text: 'legacy_key' },
333
                { type: 'Args.Equals', text: '=' },
334
                { type: 'Identifier', text: 'blah' },
335
                { type: 'Macro.End', text: '}}' },
336
            ];
337

338
            expect(tokens).toEqual(expectedTokens);
339
        });
340
        // {{roll:1d4}}
341
        test('should handle argument with legacy one colon syntax to start the arguments', async ({ page }) => {
342
            const input = '{{roll:1d4}}';
343
            const tokens = await runLexerGetTokens(page, input);
344

345
            const expectedTokens = [
346
                { type: 'Macro.Start', text: '{{' },
347
                { type: 'Macro.Identifier', text: 'roll' },
348
                { type: 'Args.Colon', text: ':' },
349
                { type: 'Unknown', text: '1' },
350
                { type: 'Identifier', text: 'd4' },
351
                { type: 'Macro.End', text: '}}' },
352
            ];
353

354
            expect(tokens).toEqual(expectedTokens);
355
        });
356
        // {{random "this" "and that" "and some more"}}
357
        test('should handle multiple unnamed arguments in quotation marks', async ({ page }) => {
358
            const input = '{{random "this" "and that" "and some more"}}';
359
            const tokens = await runLexerGetTokens(page, input);
360

361
            const expectedTokens = [
362
                { type: 'Macro.Start', text: '{{' },
363
                { type: 'Macro.Identifier', text: 'random' },
364
                { type: 'Args.Quote', text: '"' },
365
                { type: 'Identifier', text: 'this' },
366
                { type: 'Args.Quote', text: '"' },
367
                { type: 'Args.Quote', text: '"' },
368
                { type: 'Identifier', text: 'and' },
369
                { type: 'Identifier', text: 'that' },
370
                { type: 'Args.Quote', text: '"' },
371
                { type: 'Args.Quote', text: '"' },
372
                { type: 'Identifier', text: 'and' },
373
                { type: 'Identifier', text: 'some' },
374
                { type: 'Identifier', text: 'more' },
375
                { type: 'Args.Quote', text: '"' },
376
                { type: 'Macro.End', text: '}}' },
377
            ];
378

379
            expect(tokens).toEqual(expectedTokens);
380
        });
381
        // {{doStuff key="My Spaced Value" otherKey=SingleKey}}
382
        test('should handle named arguments with mixed style', async ({ page }) => {
383
            const input = '{{doStuff key="My Spaced Value" otherKey=SingleKey}}';
384
            const tokens = await runLexerGetTokens(page, input);
385

386
            const expectedTokens = [
387
                { type: 'Macro.Start', text: '{{' },
388
                { type: 'Macro.Identifier', text: 'doStuff' },
389
                { type: 'Identifier', text: 'key' },
390
                { type: 'Args.Equals', text: '=' },
391
                { type: 'Args.Quote', text: '"' },
392
                { type: 'Identifier', text: 'My' },
393
                { type: 'Identifier', text: 'Spaced' },
394
                { type: 'Identifier', text: 'Value' },
395
                { type: 'Args.Quote', text: '"' },
396
                { type: 'Identifier', text: 'otherKey' },
397
                { type: 'Args.Equals', text: '=' },
398
                { type: 'Identifier', text: 'SingleKey' },
399
                { type: 'Macro.End', text: '}}' },
400
            ];
401

402
            expect(tokens).toEqual(expectedTokens);
403
        });
404
        // {{doStuff key=}}
405
        test('should handle macros with empty named arguments', async ({ page }) => {
406
            const input = '{{doStuff key=}}';
407
            const tokens = await runLexerGetTokens(page, input);
408

409
            const expectedTokens = [
410
                { type: 'Macro.Start', text: '{{' },
411
                { type: 'Macro.Identifier', text: 'doStuff' },
412
                { type: 'Identifier', text: 'key' },
413
                { type: 'Args.Equals', text: '=' },
414
                { type: 'Macro.End', text: '}}' },
415
            ];
416

417
            expect(tokens).toEqual(expectedTokens);
418
        });
419
        // {{random "" ""}}
420
        test('should handle empty unnamed arguments if quoted', async ({ page }) => {
421
            const input = '{{random "" ""}}';
422
            const tokens = await runLexerGetTokens(page, input);
423

424
            const expectedTokens = [
425
                { type: 'Macro.Start', text: '{{' },
426
                { type: 'Macro.Identifier', text: 'random' },
427
                { type: 'Args.Quote', text: '"' },
428
                { type: 'Args.Quote', text: '"' },
429
                { type: 'Args.Quote', text: '"' },
430
                { type: 'Args.Quote', text: '"' },
431
                { type: 'Macro.End', text: '}}' },
432
            ];
433

434
            expect(tokens).toEqual(expectedTokens);
435
        });
436
        // {{doStuff special chars #!@&*()}}
437
        test('should handle macros with special characters in arguments', async ({ page }) => {
438
            const input = '{{doStuff special chars #!@&*()}}';
439
            const tokens = await runLexerGetTokens(page, input);
440

441
            const expectedTokens = [
442
                { type: 'Macro.Start', text: '{{' },
443
                { type: 'Macro.Identifier', text: 'doStuff' },
444
                { type: 'Identifier', text: 'special' },
445
                { type: 'Identifier', text: 'chars' },
446
                { type: 'Unknown', text: '#' },
447
                { type: 'Unknown', text: '!' },
448
                { type: 'Unknown', text: '@' },
449
                { type: 'Unknown', text: '&' },
450
                { type: 'Unknown', text: '*' },
451
                { type: 'Unknown', text: '(' },
452
                { type: 'Unknown', text: ')' },
453
                { type: 'Macro.End', text: '}}' },
454
            ];
455

456
            expect(tokens).toEqual(expectedTokens);
457
        });
458
        // {{longMacro arg1="value1" arg2="value2" arg3="value3"}}
459
        test('should handle long macros with multiple arguments', async ({ page }) => {
460
            const input = '{{longMacro arg1="value1" arg2="value2" arg3="value3"}}';
461
            const tokens = await runLexerGetTokens(page, input);
462

463
            const expectedTokens = [
464
                { type: 'Macro.Start', text: '{{' },
465
                { type: 'Macro.Identifier', text: 'longMacro' },
466
                { type: 'Identifier', text: 'arg1' },
467
                { type: 'Args.Equals', text: '=' },
468
                { type: 'Args.Quote', text: '"' },
469
                { type: 'Identifier', text: 'value1' },
470
                { type: 'Args.Quote', text: '"' },
471
                { type: 'Identifier', text: 'arg2' },
472
                { type: 'Args.Equals', text: '=' },
473
                { type: 'Args.Quote', text: '"' },
474
                { type: 'Identifier', text: 'value2' },
475
                { type: 'Args.Quote', text: '"' },
476
                { type: 'Identifier', text: 'arg3' },
477
                { type: 'Args.Equals', text: '=' },
478
                { type: 'Args.Quote', text: '"' },
479
                { type: 'Identifier', text: 'value3' },
480
                { type: 'Args.Quote', text: '"' },
481
                { type: 'Macro.End', text: '}}' },
482
            ];
483

484
            expect(tokens).toEqual(expectedTokens);
485
        });
486
        // {{complexMacro "text with {{nested}} content" key=val}}
487
        test('should handle macros with complex argument patterns', async ({ page }) => {
488
            const input = '{{complexMacro "text with {{nested}} content" key=val}}';
489
            const tokens = await runLexerGetTokens(page, input);
490

491
            const expectedTokens = [
492
                { type: 'Macro.Start', text: '{{' },
493
                { type: 'Macro.Identifier', text: 'complexMacro' },
494
                { type: 'Args.Quote', text: '"' },
495
                { type: 'Identifier', text: 'text' },
496
                { type: 'Identifier', text: 'with' },
497
                { type: 'Macro.Start', text: '{{' },
498
                { type: 'Macro.Identifier', text: 'nested' },
499
                { type: 'Macro.End', text: '}}' },
500
                { type: 'Identifier', text: 'content' },
501
                { type: 'Args.Quote', text: '"' },
502
                { type: 'Identifier', text: 'key' },
503
                { type: 'Args.Equals', text: '=' },
504
                { type: 'Identifier', text: 'val' },
505
                { type: 'Macro.End', text: '}}' },
506
            ];
507

508
            expect(tokens).toEqual(expectedTokens);
509
        });
510
        // TODO: test invalid argument name identifiers
511
    });
512

513
    test.describe('Macro Execution Modifiers', () => {
514
        // {{!immediate}}
515
        test('should support ! flag', async ({ page }) => {
516
            const input = '{{!immediate}}';
517
            const tokens = await runLexerGetTokens(page, input);
518

519
            const expectedTokens = [
520
                { type: 'Macro.Start', text: '{{' },
521
                { type: 'Macro.Flag', text: '!' },
522
                { type: 'Macro.Identifier', text: 'immediate' },
523
                { type: 'Macro.End', text: '}}' },
524
            ];
525

526
            expect(tokens).toEqual(expectedTokens);
527
        });
528
        // {{?lazy}}
529
        test('should support ? flag', async ({ page }) => {
530
            const input = '{{?lazy}}';
531
            const tokens = await runLexerGetTokens(page, input);
532

533
            const expectedTokens = [
534
                { type: 'Macro.Start', text: '{{' },
535
                { type: 'Macro.Flag', text: '?' },
536
                { type: 'Macro.Identifier', text: 'lazy' },
537
                { type: 'Macro.End', text: '}}' },
538
            ];
539

540
            expect(tokens).toEqual(expectedTokens);
541
        });
542
        // {{~reevaluate}}
543
        test('should support ~ flag', async ({ page }) => {
544
            const input = '{{~reevaluate}}';
545
            const tokens = await runLexerGetTokens(page, input);
546

547
            const expectedTokens = [
548
                { type: 'Macro.Start', text: '{{' },
549
                { type: 'Macro.Flag', text: '~' },
550
                { type: 'Macro.Identifier', text: 'reevaluate' },
551
                { type: 'Macro.End', text: '}}' },
552
            ];
553

554
            expect(tokens).toEqual(expectedTokens);
555
        });
556
        // {{/if}}
557
        test('should support / flag', async ({ page }) => {
558
            const input = '{{/if}}';
559
            const tokens = await runLexerGetTokens(page, input);
560

561
            const expectedTokens = [
562
                { type: 'Macro.Start', text: '{{' },
563
                { type: 'Macro.Flag', text: '/' },
564
                { type: 'Macro.Identifier', text: 'if' },
565
                { type: 'Macro.End', text: '}}' },
566
            ];
567

568
            expect(tokens).toEqual(expectedTokens);
569
        });
570
        // {{#legacy}}
571
        test('should support legacy # flag', async ({ page }) => {
572
            const input = '{{#legacy}}';
573
            const tokens = await runLexerGetTokens(page, input);
574

575
            const expectedTokens = [
576
                { type: 'Macro.Start', text: '{{' },
577
                { type: 'Macro.Flag', text: '#' },
578
                { type: 'Macro.Identifier', text: 'legacy' },
579
                { type: 'Macro.End', text: '}}' },
580
            ];
581

582
            expect(tokens).toEqual(expectedTokens);
583
        });
584
        // {{  !  identifier  }}
585
        test('should allow whitespaces around flags', async ({ page }) => {
586
            const input = '{{  !  identifier  }}';
587
            const tokens = await runLexerGetTokens(page, input);
588

589
            const expectedTokens = [
590
                { type: 'Macro.Start', text: '{{' },
591
                { type: 'Macro.Flag', text: '!' },
592
                { type: 'Macro.Identifier', text: 'identifier' },
593
                { type: 'Macro.End', text: '}}' },
594
            ];
595

596
            expect(tokens).toEqual(expectedTokens);
597
        });
598
        // {{ ?~lateragain }}
599
        test('should support multiple flags', async ({ page }) => {
600
            const input = '{{ ?~lateragain }}';
601
            const tokens = await runLexerGetTokens(page, input);
602

603
            const expectedTokens = [
604
                { type: 'Macro.Start', text: '{{' },
605
                { type: 'Macro.Flag', text: '?' },
606
                { type: 'Macro.Flag', text: '~' },
607
                { type: 'Macro.Identifier', text: 'lateragain' },
608
                { type: 'Macro.End', text: '}}' },
609
            ];
610

611
            expect(tokens).toEqual(expectedTokens);
612
        });
613
        // {{ ! .importantvariable }}
614
        test('should support multiple flags with whitespace', async ({ page }) => {
615
            const input = '{{ !#importantvariable }}';
616
            const tokens = await runLexerGetTokens(page, input);
617

618
            const expectedTokens = [
619
                { type: 'Macro.Start', text: '{{' },
620
                { type: 'Macro.Flag', text: '!' },
621
                { type: 'Macro.Flag', text: '#' },
622
                { type: 'Macro.Identifier', text: 'importantvariable' },
623
                { type: 'Macro.End', text: '}}' },
624
            ];
625

626
            expect(tokens).toEqual(expectedTokens);
627
        });
628
        // {{>filtered}}
629
        test('should support > filter flag as separate token', async ({ page }) => {
630
            const input = '{{>filtered}}';
631
            const tokens = await runLexerGetTokens(page, input);
632

633
            const expectedTokens = [
634
                { type: 'Macro.Start', text: '{{' },
635
                { type: 'Macro.FilterFlag', text: '>' },
636
                { type: 'Macro.Identifier', text: 'filtered' },
637
                { type: 'Macro.End', text: '}}' },
638
            ];
639

640
            expect(tokens).toEqual(expectedTokens);
641
        });
642
        // {{ ! > user }}
643
        test('should support filter flag combined with other flags', async ({ page }) => {
644
            const input = '{{ ! > user }}';
645
            const tokens = await runLexerGetTokens(page, input);
646

647
            const expectedTokens = [
648
                { type: 'Macro.Start', text: '{{' },
649
                { type: 'Macro.Flag', text: '!' },
650
                { type: 'Macro.FilterFlag', text: '>' },
651
                { type: 'Macro.Identifier', text: 'user' },
652
                { type: 'Macro.End', text: '}}' },
653
            ];
654

655
            expect(tokens).toEqual(expectedTokens);
656
        });
657
        // {{ a shaaark }}
658
        test('should not capture single letter as flag, but as macro identifiers', async ({ page }) => {
659
            const input = '{{ a shaaark }}';
660
            const tokens = await runLexerGetTokens(page, input);
661

662
            const expectedTokens = [
663
                { type: 'Macro.Start', text: '{{' },
664
                { type: 'Macro.Identifier', text: 'a' },
665
                { type: 'Identifier', text: 'shaaark' },
666
                { type: 'Macro.End', text: '}}' },
667
            ];
668

669
            expect(tokens).toEqual(expectedTokens);
670
        });
671

672
        test.describe('"Error" Cases (Macro Execution Modifiers)', () => {
673
            // {{ @unknown }}
674
            test('should not capture unknown special characters as flag', async ({ page }) => {
675
                const input = '{{ @unknown }}';
676
                const { tokens, errors } = await runLexerGetTokensAndErrors(page, input);
677

678
                // No errors expected, as lexer should not error out even on invalid macros
679
                expect(errors).toMatchObject([]);
680

681
                const expectedTokens = [
682
                    { type: 'Macro.Start', text: '{{' },
683
                    // Because '@' is invalid in lexer, it'll "pop out" and be captured as plaintext
684
                    { type: 'Plaintext', text: '@unknown }}' },
685
                ];
686

687
                expect(tokens).toEqual(expectedTokens);
688
            });
689
            // {{ 2 cents }}
690
            test('should not capture numbers as flag - they are also invalid macro identifiers', async ({ page }) => {
691
                const input = '{{ 2 cents }}';
692
                const { tokens, errors } = await runLexerGetTokensAndErrors(page, input);
693

694
                // No errors expected, as lexer should not error out even on invalid macros
695
                expect(errors).toMatchObject([]);
696

697
                const expectedTokens = [
698
                    { type: 'Macro.Start', text: '{{' },
699
                    // Because '2' is invalid in lexer, it'll "pop out" and be captured as plaintext
700
                    { type: 'Plaintext', text: '2 cents }}' },
701
                ];
702

703
                expect(tokens).toEqual(expectedTokens);
704
            });
705
        });
706
    });
707

708
    test.describe('Variable Shorthand Syntax', () => {
709
        // {{.variable}} - Local variable get
710
        test('should tokenize local variable shorthand', async ({ page }) => {
711
            const input = '{{.myvar}}';
712
            const tokens = await runLexerGetTokens(page, input);
713

714
            const expectedTokens = [
715
                { type: 'Macro.Start', text: '{{' },
716
                { type: 'Var.LocalPrefix', text: '.' },
717
                { type: 'Var.Identifier', text: 'myvar' },
718
                { type: 'Macro.End', text: '}}' },
719
            ];
720

721
            expect(tokens).toEqual(expectedTokens);
722
        });
723

724
        // {{$variable}} - Global variable get
725
        test('should tokenize global variable shorthand', async ({ page }) => {
726
            const input = '{{$myvar}}';
727
            const tokens = await runLexerGetTokens(page, input);
728

729
            const expectedTokens = [
730
                { type: 'Macro.Start', text: '{{' },
731
                { type: 'Var.GlobalPrefix', text: '$' },
732
                { type: 'Var.Identifier', text: 'myvar' },
733
                { type: 'Macro.End', text: '}}' },
734
            ];
735

736
            expect(tokens).toEqual(expectedTokens);
737
        });
738

739
        // {{.my-var}} - Variable with hyphen in name
740
        test('should tokenize variable with hyphen in name', async ({ page }) => {
741
            const input = '{{.my-var}}';
742
            const tokens = await runLexerGetTokens(page, input);
743

744
            const expectedTokens = [
745
                { type: 'Macro.Start', text: '{{' },
746
                { type: 'Var.LocalPrefix', text: '.' },
747
                { type: 'Var.Identifier', text: 'my-var' },
748
                { type: 'Macro.End', text: '}}' },
749
            ];
750

751
            expect(tokens).toEqual(expectedTokens);
752
        });
753

754
        // {{.counter++}} - Increment operator
755
        test('should tokenize increment operator', async ({ page }) => {
756
            const input = '{{.counter++}}';
757
            const tokens = await runLexerGetTokens(page, input);
758

759
            const expectedTokens = [
760
                { type: 'Macro.Start', text: '{{' },
761
                { type: 'Var.LocalPrefix', text: '.' },
762
                { type: 'Var.Identifier', text: 'counter' },
763
                { type: 'Var.Increment', text: '++' },
764
                { type: 'Macro.End', text: '}}' },
765
            ];
766

767
            expect(tokens).toEqual(expectedTokens);
768
        });
769

770
        // {{$counter--}} - Decrement operator
771
        test('should tokenize decrement operator', async ({ page }) => {
772
            const input = '{{$counter--}}';
773
            const tokens = await runLexerGetTokens(page, input);
774

775
            const expectedTokens = [
776
                { type: 'Macro.Start', text: '{{' },
777
                { type: 'Var.GlobalPrefix', text: '$' },
778
                { type: 'Var.Identifier', text: 'counter' },
779
                { type: 'Var.Decrement', text: '--' },
780
                { type: 'Macro.End', text: '}}' },
781
            ];
782

783
            expect(tokens).toEqual(expectedTokens);
784
        });
785

786
        // {{.myvar = value}} - Set operator
787
        test('should tokenize set operator with value', async ({ page }) => {
788
            const input = '{{.myvar = hello}}';
789
            const tokens = await runLexerGetTokens(page, input);
790

791
            const expectedTokens = [
792
                { type: 'Macro.Start', text: '{{' },
793
                { type: 'Var.LocalPrefix', text: '.' },
794
                { type: 'Var.Identifier', text: 'myvar' },
795
                { type: 'Var.Equals', text: '=' },
796
                { type: 'Identifier', text: 'hello' },
797
                { type: 'Macro.End', text: '}}' },
798
            ];
799

800
            expect(tokens).toEqual(expectedTokens);
801
        });
802

803
        // {{.myvar += 5}} - Add operator
804
        test('should tokenize add operator with value', async ({ page }) => {
805
            const input = '{{.myvar += 5}}';
806
            const tokens = await runLexerGetTokens(page, input);
807

808
            const expectedTokens = [
809
                { type: 'Macro.Start', text: '{{' },
810
                { type: 'Var.LocalPrefix', text: '.' },
811
                { type: 'Var.Identifier', text: 'myvar' },
812
                { type: 'Var.PlusEquals', text: '+=' },
813
                { type: 'Unknown', text: '5' },
814
                { type: 'Macro.End', text: '}}' },
815
            ];
816

817
            expect(tokens).toEqual(expectedTokens);
818
        });
819

820
        // {{ !.importantvariable }} - Variable prefix after flags
821
        test('should tokenize variable prefix after flags', async ({ page }) => {
822
            const input = '{{ !.importantvariable }}';
823
            const tokens = await runLexerGetTokens(page, input);
824

825
            // When . is encountered, it triggers variable mode regardless of previous flags
826
            const expectedTokens = [
827
                { type: 'Macro.Start', text: '{{' },
828
                { type: 'Macro.Flag', text: '!' },
829
                { type: 'Var.LocalPrefix', text: '.' },
830
                { type: 'Var.Identifier', text: 'importantvariable' },
831
                { type: 'Macro.End', text: '}}' },
832
            ];
833

834
            expect(tokens).toEqual(expectedTokens);
835
        });
836

837
        // {{.my-long-var-name}} - Variable with multiple hyphens
838
        test('should tokenize variable with multiple hyphens', async ({ page }) => {
839
            const input = '{{.my-long-var-name}}';
840
            const tokens = await runLexerGetTokens(page, input);
841

842
            const expectedTokens = [
843
                { type: 'Macro.Start', text: '{{' },
844
                { type: 'Var.LocalPrefix', text: '.' },
845
                { type: 'Var.Identifier', text: 'my-long-var-name' },
846
                { type: 'Macro.End', text: '}}' },
847
            ];
848

849
            expect(tokens).toEqual(expectedTokens);
850
        });
851

852
        // {{.myvar = Hello {{user}}}} - Nested macro in value
853
        test('should tokenize nested macro in variable value', async ({ page }) => {
854
            const input = '{{.myvar = Hello {{user}}}}';
855
            const tokens = await runLexerGetTokens(page, input);
856

857
            const expectedTokens = [
858
                { type: 'Macro.Start', text: '{{' },
859
                { type: 'Var.LocalPrefix', text: '.' },
860
                { type: 'Var.Identifier', text: 'myvar' },
861
                { type: 'Var.Equals', text: '=' },
862
                { type: 'Identifier', text: 'Hello' },
863
                { type: 'Macro.Start', text: '{{' },
864
                { type: 'Macro.Identifier', text: 'user' },
865
                { type: 'Macro.End', text: '}}' },
866
                { type: 'Macro.End', text: '}}' },
867
            ];
868

869
            expect(tokens).toEqual(expectedTokens);
870
        });
871

872
        // {{ .myvar }} - Whitespace around variable shorthand
873
        test('should tokenize variable shorthand with surrounding whitespace', async ({ page }) => {
874
            const input = '{{ .myvar }}';
875
            const tokens = await runLexerGetTokens(page, input);
876

877
            const expectedTokens = [
878
                { type: 'Macro.Start', text: '{{' },
879
                { type: 'Var.LocalPrefix', text: '.' },
880
                { type: 'Var.Identifier', text: 'myvar' },
881
                { type: 'Macro.End', text: '}}' },
882
            ];
883

884
            expect(tokens).toEqual(expectedTokens);
885
        });
886

887
        // {{$myVar123}} - Variable with numbers
888
        test('should tokenize variable with numbers in name', async ({ page }) => {
889
            const input = '{{$myVar123}}';
890
            const tokens = await runLexerGetTokens(page, input);
891

892
            const expectedTokens = [
893
                { type: 'Macro.Start', text: '{{' },
894
                { type: 'Var.GlobalPrefix', text: '$' },
895
                { type: 'Var.Identifier', text: 'myVar123' },
896
                { type: 'Macro.End', text: '}}' },
897
            ];
898

899
            expect(tokens).toEqual(expectedTokens);
900
        });
901

902
        // {{.my_var}} - Variable with underscore
903
        test('should tokenize variable with underscore in name', async ({ page }) => {
904
            const input = '{{.my_var}}';
905
            const tokens = await runLexerGetTokens(page, input);
906

907
            const expectedTokens = [
908
                { type: 'Macro.Start', text: '{{' },
909
                { type: 'Var.LocalPrefix', text: '.' },
910
                { type: 'Var.Identifier', text: 'my_var' },
911
                { type: 'Macro.End', text: '}}' },
912
            ];
913

914
            expect(tokens).toEqual(expectedTokens);
915
        });
916

917
        // {{.counter ++ }} - Increment with whitespace
918
        test('should tokenize increment operator with surrounding whitespace', async ({ page }) => {
919
            const input = '{{.counter ++ }}';
920
            const tokens = await runLexerGetTokens(page, input);
921

922
            const expectedTokens = [
923
                { type: 'Macro.Start', text: '{{' },
924
                { type: 'Var.LocalPrefix', text: '.' },
925
                { type: 'Var.Identifier', text: 'counter' },
926
                { type: 'Var.Increment', text: '++' },
927
                { type: 'Macro.End', text: '}}' },
928
            ];
929

930
            expect(tokens).toEqual(expectedTokens);
931
        });
932
    });
933

934
    test.describe('Macro Output Modifiers', () => {
935
        // {{macro | outputModifier}}
936
        test('should support output modifier without arguments', async ({ page }) => {
937
            const input = '{{macro | outputModifier}}';
938
            const tokens = await runLexerGetTokens(page, input);
939

940
            const expectedTokens = [
941
                { type: 'Macro.Start', text: '{{' },
942
                { type: 'Macro.Identifier', text: 'macro' },
943
                { type: 'Filter.Pipe', text: '|' },
944
                { type: 'Filter.Identifier', text: 'outputModifier' },
945
                { type: 'Macro.End', text: '}}' },
946
            ];
947

948
            expect(tokens).toEqual(expectedTokens);
949
        });
950
        // {{macro | outputModifier arg1=val1 arg2=val2}}
951
        test('should support output modifier with named arguments', async ({ page }) => {
952
            const input = '{{macro | outputModifier arg1=val1 arg2=val2}}';
953
            const tokens = await runLexerGetTokens(page, input);
954

955
            const expectedTokens = [
956
                { type: 'Macro.Start', text: '{{' },
957
                { type: 'Macro.Identifier', text: 'macro' },
958
                { type: 'Filter.Pipe', text: '|' },
959
                { type: 'Filter.Identifier', text: 'outputModifier' },
960
                { type: 'Identifier', text: 'arg1' },
961
                { type: 'Args.Equals', text: '=' },
962
                { type: 'Identifier', text: 'val1' },
963
                { type: 'Identifier', text: 'arg2' },
964
                { type: 'Args.Equals', text: '=' },
965
                { type: 'Identifier', text: 'val2' },
966
                { type: 'Macro.End', text: '}}' },
967
            ];
968

969
            expect(tokens).toEqual(expectedTokens);
970
        });
971
        // {{macro | outputModifier "unnamed1" "unnamed2"}}
972
        test('should support output modifier with unnamed arguments', async ({ page }) => {
973
            const input = '{{macro | outputModifier "unnamed1" "unnamed2"}}';
974
            const tokens = await runLexerGetTokens(page, input);
975

976
            const expectedTokens = [
977
                { type: 'Macro.Start', text: '{{' },
978
                { type: 'Macro.Identifier', text: 'macro' },
979
                { type: 'Filter.Pipe', text: '|' },
980
                { type: 'Filter.Identifier', text: 'outputModifier' },
981
                { type: 'Args.Quote', text: '"' },
982
                { type: 'Identifier', text: 'unnamed1' },
983
                { type: 'Args.Quote', text: '"' },
984
                { type: 'Args.Quote', text: '"' },
985
                { type: 'Identifier', text: 'unnamed2' },
986
                { type: 'Args.Quote', text: '"' },
987
                { type: 'Macro.End', text: '}}' },
988
            ];
989

990
            expect(tokens).toEqual(expectedTokens);
991
        });
992
        // {{macro arg1=val1 | outputModifier arg2=val2 "unnamed1"}}
993
        test('should support macro arguments before output modifier', async ({ page }) => {
994
            const input = '{{macro arg1=val1 | outputModifier arg2=val2 "unnamed1"}}';
995
            const tokens = await runLexerGetTokens(page, input);
996

997
            const expectedTokens = [
998
                { type: 'Macro.Start', text: '{{' },
999
                { type: 'Macro.Identifier', text: 'macro' },
1000
                { type: 'Identifier', text: 'arg1' },
1001
                { type: 'Args.Equals', text: '=' },
1002
                { type: 'Identifier', text: 'val1' },
1003
                { type: 'Filter.Pipe', text: '|' },
1004
                { type: 'Filter.Identifier', text: 'outputModifier' },
1005
                { type: 'Identifier', text: 'arg2' },
1006
                { type: 'Args.Equals', text: '=' },
1007
                { type: 'Identifier', text: 'val2' },
1008
                { type: 'Args.Quote', text: '"' },
1009
                { type: 'Identifier', text: 'unnamed1' },
1010
                { type: 'Args.Quote', text: '"' },
1011
                { type: 'Macro.End', text: '}}' },
1012
            ];
1013

1014
            expect(tokens).toEqual(expectedTokens);
1015
        });
1016
        // {{macro | outputModifier1 | outputModifier2}}
1017
        test('should support chaining multiple output modifiers', async ({ page }) => {
1018
            const input = '{{macro | outputModifier1 | outputModifier2}}';
1019
            const tokens = await runLexerGetTokens(page, input);
1020

1021
            const expectedTokens = [
1022
                { type: 'Macro.Start', text: '{{' },
1023
                { type: 'Macro.Identifier', text: 'macro' },
1024
                { type: 'Filter.Pipe', text: '|' },
1025
                { type: 'Filter.Identifier', text: 'outputModifier1' },
1026
                { type: 'Filter.Pipe', text: '|' },
1027
                { type: 'Filter.Identifier', text: 'outputModifier2' },
1028
                { type: 'Macro.End', text: '}}' },
1029
            ];
1030

1031
            expect(tokens).toEqual(expectedTokens);
1032
        });
1033
        // {{macro | outputModifier1 arg1=val1 | outputModifier2 arg2=val2}}
1034
        test('should support chaining multiple output modifiers with arguments', async ({ page }) => {
1035
            const input = '{{macro | outputModifier1 arg1=val1 | outputModifier2 arg2=val2}}';
1036
            const tokens = await runLexerGetTokens(page, input);
1037

1038
            const expectedTokens = [
1039
                { type: 'Macro.Start', text: '{{' },
1040
                { type: 'Macro.Identifier', text: 'macro' },
1041
                { type: 'Filter.Pipe', text: '|' },
1042
                { type: 'Filter.Identifier', text: 'outputModifier1' },
1043
                { type: 'Identifier', text: 'arg1' },
1044
                { type: 'Args.Equals', text: '=' },
1045
                { type: 'Identifier', text: 'val1' },
1046
                { type: 'Filter.Pipe', text: '|' },
1047
                { type: 'Filter.Identifier', text: 'outputModifier2' },
1048
                { type: 'Identifier', text: 'arg2' },
1049
                { type: 'Args.Equals', text: '=' },
1050
                { type: 'Identifier', text: 'val2' },
1051
                { type: 'Macro.End', text: '}}' },
1052
            ];
1053

1054
            expect(tokens).toEqual(expectedTokens);
1055
        });
1056
        // {{macro|outputModifier}}
1057
        test('should support output modifiers without whitespace', async ({ page }) => {
1058
            const input = '{{macro|outputModifier}}';
1059
            const tokens = await runLexerGetTokens(page, input);
1060

1061
            const expectedTokens = [
1062
                { type: 'Macro.Start', text: '{{' },
1063
                { type: 'Macro.Identifier', text: 'macro' },
1064
                { type: 'Filter.Pipe', text: '|' },
1065
                { type: 'Filter.Identifier', text: 'outputModifier' },
1066
                { type: 'Macro.End', text: '}}' },
1067
            ];
1068

1069
            expect(tokens).toEqual(expectedTokens);
1070
        });
1071
        // {{ macro test escaped \| pipe }}
1072
        test('should support escaped pipes, not treating them as output modifiers', async ({ page }) => {
1073
            const input = '{{ macro test escaped \\| pipe }}';
1074
            const tokens = await runLexerGetTokens(page, input);
1075

1076
            const expectedTokens = [
1077
                { type: 'Macro.Start', text: '{{' },
1078
                { type: 'Macro.Identifier', text: 'macro' },
1079
                { type: 'Identifier', text: 'test' },
1080
                { type: 'Identifier', text: 'escaped' },
1081
                { type: 'Filter.EscapedPipe', text: '\\|' },
1082
                { type: 'Identifier', text: 'pipe' },
1083
                { type: 'Macro.End', text: '}}' },
1084
            ];
1085

1086
            expect(tokens).toEqual(expectedTokens);
1087
        });
1088

1089
        test.describe('Error Cases (Macro Output Modifiers)', () => {
1090
            // {{|macro}}
1091
            test('should not capture when starting the macro with a pipe', async ({ page }) => {
1092
                const input = '{{|macro}}';
1093
                const { tokens, errors } = await runLexerGetTokensAndErrors(page, input);
1094

1095
                // No errors expected, as lexer should not error out even on invalid macros
1096
                expect(errors).toMatchObject([]);
1097

1098
                const expectedTokens = [
1099
                    { type: 'Macro.Start', text: '{{' },
1100
                    { type: 'Plaintext', text: '|macro}}' },
1101
                ];
1102

1103
                expect(tokens).toEqual(expectedTokens);
1104
            });
1105
            // {{macro | Iam$peci@l}}
1106
            test('[Error] should not allow special characters inside output modifier identifier', async ({ page }) => {
1107
                const input = '{{macro | Iam$peci@l}}';
1108
                const { tokens, errors } = await runLexerGetTokensAndErrors(page, input);
1109

1110
                const expectedErrors = [
1111
                    { message: 'unexpected character: ->$<- at offset: 13, skipped 7 characters.' },
1112
                ];
1113

1114
                expect(errors).toMatchObject(expectedErrors);
1115

1116
                const expectedTokens = [
1117
                    { type: 'Macro.Start', text: '{{' },
1118
                    { type: 'Macro.Identifier', text: 'macro' },
1119
                    { type: 'Filter.Pipe', text: '|' },
1120
                    { type: 'Filter.Identifier', text: 'Iam' },
1121
                    { type: 'Macro.End', text: '}}' },
1122
                ];
1123

1124
                expect(tokens).toEqual(expectedTokens);
1125
            });
1126
            // {{macro | !cannotBeImportant }}
1127
            test('[Error] should not allow output modifiers to have execution modifiers', async ({ page }) => {
1128
                const input = '{{macro | !cannotBeImportant }}';
1129
                const { tokens, errors } = await runLexerGetTokensAndErrors(page, input);
1130

1131
                const expectedErrors = [
1132
                    { message: 'unexpected character: ->!<- at offset: 10, skipped 1 characters.' },
1133
                ];
1134

1135
                expect(errors).toMatchObject(expectedErrors);
1136

1137
                const expectedTokens = [
1138
                    { type: 'Macro.Start', text: '{{' },
1139
                    { type: 'Macro.Identifier', text: 'macro' },
1140
                    { type: 'Filter.Pipe', text: '|' },
1141
                    { type: 'Filter.Identifier', text: 'cannotBeImportant' },
1142
                    { type: 'Macro.End', text: '}}' },
1143
                ];
1144

1145
                expect(tokens).toEqual(expectedTokens);
1146
            });
1147
            // {{macro | 2invalidIdentifier}}
1148
            test('[Error] should not allow invalid identifier starting with a number', async ({ page }) => {
1149
                const input = '{{macro | 2invalidIdentifier}}';
1150
                const { tokens, errors } = await runLexerGetTokensAndErrors(page, input);
1151

1152
                const expectedErrors = [
1153
                    { message: 'unexpected character: ->2<- at offset: 10, skipped 1 characters.' },
1154
                ];
1155

1156
                expect(errors).toMatchObject(expectedErrors);
1157

1158
                const expectedTokens = [
1159
                    { type: 'Macro.Start', text: '{{' },
1160
                    { type: 'Macro.Identifier', text: 'macro' },
1161
                    { type: 'Filter.Pipe', text: '|' },
1162
                    { type: 'Filter.Identifier', text: 'invalidIdentifier' },
1163
                    { type: 'Macro.End', text: '}}' },
1164
                ];
1165

1166
                expect(tokens).toEqual(expectedTokens);
1167
            });
1168
            // {{macro || outputModifier}}
1169
            test('[Error] should not allow double pipe used without an identifier', async ({ page }) => {
1170
                const input = '{{macro || outputModifier}}';
1171
                const { tokens, errors } = await runLexerGetTokensAndErrors(page, input);
1172

1173
                const expectedErrors = [
1174
                    { message: 'unexpected character: ->|<- at offset: 9, skipped 1 characters.' },
1175
                ];
1176

1177
                expect(errors).toMatchObject(expectedErrors);
1178

1179
                const expectedTokens = [
1180
                    { type: 'Macro.Start', text: '{{' },
1181
                    { type: 'Macro.Identifier', text: 'macro' },
1182
                    { type: 'Filter.Pipe', text: '|' },
1183
                    { type: 'Filter.Identifier', text: 'outputModifier' },
1184
                    { type: 'Macro.End', text: '}}' },
1185
                ];
1186

1187
                expect(tokens).toEqual(expectedTokens);
1188
            });
1189
        });
1190
    });
1191

1192
    test.describe('Macro While Typing..', () => {
1193
    // {{unclosed_macro word and more. Done.
1194
        test('should allow unclosed macros, but tries to parse it as a macro', async ({ page }) => {
1195
            const input = '{{unclosed_macro word and more. Done.';
1196
            const tokens = await runLexerGetTokens(page, input);
1197

1198
            const expectedTokens = [
1199
                { type: 'Macro.Start', text: '{{' },
1200
                { type: 'Macro.Identifier', text: 'unclosed_macro' },
1201
                { type: 'Identifier', text: 'word' },
1202
                { type: 'Identifier', text: 'and' },
1203
                { type: 'Identifier', text: 'more' },
1204
                { type: 'Unknown', text: '.' },
1205
                { type: 'Identifier', text: 'Done' },
1206
                { type: 'Unknown', text: '.' },
1207
            ];
1208

1209
            expect(tokens).toEqual(expectedTokens);
1210
        });
1211
    });
1212

1213
    test.describe('Macro and Plaintext', () => {
1214
        // Hello, {{user}}!
1215
        test('should handle basic macro tokenization', async ({ page }) => {
1216
            const input = 'Hello, {{user}}!';
1217
            const tokens = await runLexerGetTokens(page, input);
1218

1219
            const expectedTokens = [
1220
                { type: 'Plaintext', text: 'Hello, ' },
1221
                { type: 'Macro.Start', text: '{{' },
1222
                { type: 'Macro.Identifier', text: 'user' },
1223
                { type: 'Macro.End', text: '}}' },
1224
                { type: 'Plaintext', text: '!' },
1225
            ];
1226

1227
            // Compare the actual result with expected tokens
1228
            expect(tokens).toEqual(expectedTokens);
1229
        });
1230
        // Just some text here.
1231
        test('should tokenize plaintext only', async ({ page }) => {
1232
            const input = 'Just some text here.';
1233
            const tokens = await runLexerGetTokens(page, input);
1234

1235
            const expectedTokens = [
1236
                { type: 'Plaintext', text: 'Just some text here.' },
1237
            ];
1238

1239
            expect(tokens).toEqual(expectedTokens);
1240
        });
1241
    });
1242

1243
    test.describe('"Error" Cases in Macro Lexing', () => {
1244
        // this is an unopened_macro}} and will be done
1245
        test('should treat unopened macros as simple plaintext', async ({ page }) => {
1246
            const input = 'this is an unopened_macro}} and will be done';
1247
            const tokens = await runLexerGetTokens(page, input);
1248

1249
            const expectedTokens = [
1250
                { type: 'Plaintext', text: 'this is an unopened_macro}} and will be done' },
1251
            ];
1252

1253
            expect(tokens).toEqual(expectedTokens);
1254
        });
1255
        // { { not a macro } }
1256
        test('should treat opening/closing with whitspaces between brackets not as macros', async ({ page }) => {
1257
            const input = '{ { not a macro } }';
1258
            const tokens = await runLexerGetTokens(page, input);
1259

1260
            const expectedTokens = [
1261
                { type: 'Plaintext', text: '{ { not a macro } }' },
1262
            ];
1263

1264
            expect(tokens).toEqual(expectedTokens);
1265
        });
1266
        // invalid {{ 000 }} followed by correct {{ macro }}
1267
        test('should handle valid macro correctly after an invalid macro', async ({ page }) => {
1268
            const input = 'invalid {{ 000 }} followed by correct {{ macro }}';
1269
            const { tokens, errors } = await runLexerGetTokensAndErrors(page, input);
1270

1271
            // No errors expected, as lexer should not error out even on invalid macros
1272
            expect(errors).toMatchObject([]);
1273

1274
            const expectedTokens = [
1275
                { type: 'Plaintext', text: 'invalid ' },
1276
                { type: 'Macro.Start', text: '{{' },
1277
                // '000' is invalid vor the lexer, so it is captured as plaintext
1278
                { type: 'Plaintext', text: '000 }} followed by correct ' },
1279
                { type: 'Macro.Start', text: '{{' },
1280
                { type: 'Macro.Identifier', text: 'macro' },
1281
                { type: 'Macro.End', text: '}}' },
1282
            ];
1283

1284
            expect(tokens).toEqual(expectedTokens);
1285
        });
1286
    });
1287
});
1288

1289
/**
1290
 * Asynchronously runs the MacroLexer on the given input and returns the tokens.
1291
 *
1292
 * Lexer errors will throw an Error. To test and validate lexer errors, use `runLexerGetTokensAndErrors`.
1293
 *
1294
 * @param {import('@playwright/test').Page} page - The Playwright page object.
1295
 * @param {string} input - The input string to be tokenized.
1296
 * @returns {Promise<TestableToken[]>} A promise that resolves to an array of tokens.
1297
 */
1298
async function runLexerGetTokens(page, input) {
1299
    const { tokens, errors } = await runLexerGetTokensAndErrors(page, input);
1300

1301
    // Make sure that lexer errors get correctly marked as errors during testing, even if the resulting tokens might work.
1302
    // If we don't test for errors, the test should fail.
1303
    if (errors.length > 0) {
1304
        throw new Error('Lexer errors found\n' + errors.map(x => x.message).join('\n'));
1305
    }
1306

1307
    return tokens;
1308
}
1309

1310

1311
/**
1312
 * Asynchronously runs the MacroLexer on the given input and returns the tokens and errors.
1313
 *
1314
 * Use `runLexerGetTokens` if you don't want to explicitly test against lexer errors.
1315
 *
1316
 * @param {import('@playwright/test').Page} page - The Playwright page object.
1317
 * @param {string} input - The input string to be tokenized.
1318
 * @returns {Promise<{tokens: TestableToken[], errors: LexerError[]}>} A promise that resolves to an object containing an array of tokens and an array of lexer errors.
1319
 */
1320
async function runLexerGetTokensAndErrors(page, input) {
1321
    const result = await page.evaluate(async (input) => {
1322
        /** @type {import('../../public/scripts/macros/engine/MacroLexer.js')} */
1323
        const { MacroLexer } = await import('./scripts/macros/engine/MacroLexer.js');
1324

1325
        const result = MacroLexer.tokenize(input);
1326
        return result;
1327
    }, input);
1328

1329
    return simplifyTokens(result);
1330
}
1331

1332
/**
1333
 * Simplify the lexer tokens result into an easily testable format.
1334
 *
1335
 * @param {ILexingResult} result The result from the lexer
1336
 * @returns {{tokens: TestableToken[], errors: ILexingError[]}} The tokens
1337
 */
1338
function simplifyTokens(result) {
1339
    const errors = result.errors;
1340
    const tokens = result.tokens
1341
        // Extract relevant properties from tokens for comparison
1342
        .map(token => ({
1343
            type: token.tokenType.name,
1344
            text: token.image,
1345
        }));
1346

1347
    return { tokens, errors };
1348
}