| 1042 | 1042 | if (!logprobs?.length) { |
| 1043 | 1043 | return null; |
| 1044 | 1044 | } |
| 1045 | + |
| 1046 | + // 3 cases: |
| 1047 | + // 1. Before commit 6c5bc06, "probs" key with "tok_str"/"prob", and probs are [0, 1] so use them directly. |
| 1048 | + // 2. After commit 6c5bc06 but before commit 89d604f broke logprobs (they all return the first token's logprobs) |
| 1049 | + // We don't know the llama.cpp version so we can't do much about this. |
| 1050 | + // 3. After commit 89d604f uses OpenAI-compatible format with "completion_probabilities" and "token"/"logprob" keys. |
| 1051 | + // Note that it is also the *actual* logprob (negative number), so we need to convert to [0, 1]. |
| 1052 | + if (logprobs?.[0]?.probs) { |
| 1045 | 1053 | const candidates = logprobs?.[0]?.probs?.map(x => [x.tok_str, x.prob]); |
| 1046 | 1054 | if (!candidates) { |
| 1047 | 1055 | return null; |
| 1048 | 1056 | } |
| 1049 | 1057 | return { token, topLogprobs: candidates }; |
| 1058 | + } else if (logprobs?.[0].top_logprobs) { |
| 1059 | + const candidates = logprobs?.[0]?.top_logprobs?.map(x => [x.token, Math.exp(x.logprob)]); |
| 1060 | + if (!candidates) { |
| 1061 | + return null; |
| 1062 | + } |
| 1063 | + return { token, topLogprobs: candidates }; |
| 1064 | + } |
| 1065 | + return null; |
| 1050 | 1066 | } |
| 1051 | 1067 | default: |
| 1052 | 1068 | return null; |