| 1042 | 1042 | if (!logprobs?.length) { |
| 1043 | 1043 | return null; |
| 1044 | 1044 | } |
| 1045 | | - const candidates = logprobs?.[0]?.probs?.map(x => [x.tok_str, x.prob]); |
| 1045 | + |
| 1046 | | - if (!candidates) { |
| 1046 | + // 3 cases: |
| 1047 | | - return null; |
| 1047 | + // 1. Before commit 6c5bc06, "probs" key with "tok_str"/"prob", and probs are [0, 1] so use them directly. |
| 1048 | + // 2. After commit 6c5bc06 but before commit 89d604f broke logprobs (they all return the first token's logprobs) |
| 1049 | + // We don't know the llama.cpp version so we can't do much about this. |
| 1050 | + // 3. After commit 89d604f uses OpenAI-compatible format with "completion_probabilities" and "token"/"logprob" keys. |
| 1051 | + // Note that it is also the *actual* logprob (negative number), so we need to convert to [0, 1]. |
| 1052 | + if (logprobs?.[0]?.probs) { |
| 1053 | + const candidates = logprobs?.[0]?.probs?.map(x => [x.tok_str, x.prob]); |
| 1054 | + if (!candidates) { |
| 1055 | + return null; |
| 1056 | + } |
| 1057 | + return { token, topLogprobs: candidates }; |
| 1058 | + } else if (logprobs?.[0].top_logprobs) { |
| 1059 | + const candidates = logprobs?.[0]?.top_logprobs?.map(x => [x.token, Math.exp(x.logprob)]); |
| 1060 | + if (!candidates) { |
| 1061 | + return null; |
| 1062 | + } |
| 1063 | + return { token, topLogprobs: candidates }; |
| 1048 | 1064 | } |
| 1049 | | - return { token, topLogprobs: candidates }; |
| 1065 | + return null; |
| 1050 | 1066 | } |
| 1051 | 1067 | default: |
| 1052 | 1068 | return null; |