| 519 | 519 | |
| 520 | 520 | const tokenizerId = getTokenizerBestMatch(api); |
| 521 | 521 | |
| 522 | 522 | //** @type {any[]} Flatten unique token IDs across all logprobs */ |
| 523 | 523 | const tokenIds = Array.from(new Set(input.flatMap(logprobs => |
| 524 | 524 | logprobs.topLogprobs.map(([token]) => token).concat(logprobs.token), |
| 525 | 525 | ))); |
| 526 | 526 | |
| 527 | 527 | // Submit token IDs to tokenizer to get token text, then build ID->text map |
| 528 | 528 | // noinspection JSCheckFunctionSignatures - mutates input in-place |
| 529 | 529 | const { chunks } = decodeTextTokens(tokenizerId, tokenIds.map(parseInt)); |
| 530 | 530 | const tokenIdText = new Map(tokenIds.map((id, i) => [id, chunks[i]])); |
| 531 | 531 | |
| 532 | 532 | // Fixup logprobs data with token text |