| 519 | | 519 | |
| 520 | const tokenizerId = getTokenizerBestMatch(api); | 520 | const tokenizerId = getTokenizerBestMatch(api); |
| 521 | | 521 | |
| 522 | // Flatten unique token IDs across all logprobs | 522 | /** @type {any[]} Flatten unique token IDs across all logprobs */ |
| 523 | const tokenIds = Array.from(new Set(input.flatMap(logprobs => | 523 | const tokenIds = Array.from(new Set(input.flatMap(logprobs => |
| 524 | logprobs.topLogprobs.map(([token]) => token).concat(logprobs.token), | 524 | logprobs.topLogprobs.map(([token]) => token).concat(logprobs.token), |
| 525 | ))); | 525 | ))); |
| 526 | | 526 | |
| 527 | // Submit token IDs to tokenizer to get token text, then build ID->text map | 527 | // Submit token IDs to tokenizer to get token text, then build ID->text map |
| 528 | // noinspection JSCheckFunctionSignatures - mutates input in-place | 528 | // noinspection JSCheckFunctionSignatures - mutates input in-place |
| 529 | const { chunks } = decodeTextTokens(tokenizerId, tokenIds.map(parseInt)); | 529 | const { chunks } = decodeTextTokens(tokenizerId, tokenIds); |
| 530 | const tokenIdText = new Map(tokenIds.map((id, i) => [id, chunks[i]])); | 530 | const tokenIdText = new Map(tokenIds.map((id, i) => [id, chunks[i]])); |
| 531 | | 531 | |
| 532 | // Fixup logprobs data with token text | 532 | // Fixup logprobs data with token text |