correctly interpret some alternate whitespaces in token names

5d5e552cbdaf46bd123a3ce5a90057e93d545e09

50h100a <all_awful@proton.me>

Parents: 1ac6780e9c7f8a24a91ddbd8efd0f638e480cbd3
2 files changed, +5 -5Showing whitespace changes
public/scripts/extensions/token-counter/index.js+1 -1
@@ -89,7 +89,7 @@ function drawChunks(chunks, ids) {
89 $('#tokenized_chunks_display').empty();89 $('#tokenized_chunks_display').empty();
9090
91 for (let i = 0; i < chunks.length; i++) {91 for (let i = 0; i < chunks.length; i++) {
92 let chunk = chunks[i].replace(/▁/g, ' '); // This is a leading space in sentencepiece. More info: Lower one eighth block (U+2581)92 let chunk = chunks[i].replace(/[▁Ġ]/g, ' '); // This is a leading space in sentencepiece. More info: Lower one eighth block (U+2581)
9393
94 // If <0xHEX>, decode it94 // If <0xHEX>, decode it
95 if (/^<0x[0-9A-F]+>$/i.test(chunk)) {95 if (/^<0x[0-9A-F]+>$/i.test(chunk)) {
public/scripts/logprobs.js+4 -4
@@ -160,7 +160,7 @@ function renderTopLogprobs() {
160 let matched = false;160 let matched = false;
161 for (const [token, probability, log] of candidates) {161 for (const [token, probability, log] of candidates) {
162 const container = $('<button class="flex-container flexFlowColumn logprobs_top_candidate"></button>');162 const container = $('<button class="flex-container flexFlowColumn logprobs_top_candidate"></button>');
163 const tokenNormalized = String(token).replace(/^▁/g, ' ');163 const tokenNormalized = String(token).replace(/^[▁Ġ]/g, ' ');
164164
165 if (token === selectedToken || tokenNormalized === selectedToken) {165 if (token === selectedToken || tokenNormalized === selectedToken) {
166 matched = true;166 matched = true;
@@ -230,7 +230,7 @@ function onAlternativeClicked(tokenLogprobs, alternative) {
230 const replaceIndex = messageLogprobs.findIndex(x => x === tokenLogprobs);230 const replaceIndex = messageLogprobs.findIndex(x => x === tokenLogprobs);
231231
232 const tokens = messageLogprobs.slice(0, replaceIndex + 1).map(({ token }) => token);232 const tokens = messageLogprobs.slice(0, replaceIndex + 1).map(({ token }) => token);
233 tokens[replaceIndex] = String(alternative).replace(/^▁/g, ' ');233 tokens[replaceIndex] = String(alternative).replace(/^[▁Ġ]/g, ' ').replace(/Ċ/g, '\n');
234234
235 const prefix = continueFrom || '';235 const prefix = continueFrom || '';
236 const prompt = prefix + tokens.join('');236 const prompt = prefix + tokens.join('');
@@ -343,7 +343,7 @@ function createSwipe(messageId, prompt) {
343 * @returns {string}343 * @returns {string}
344 */344 */
345function toVisibleWhitespace(input) {345function toVisibleWhitespace(input) {
346 return input.replace(/ /g, '·').replace(/▁/g, '·').replace(/\n/g, '↵');346 return input.replace(/ /g, '·').replace(/[▁Ġ]/g, '·').replace(/[Ċ\n]/g, '↵');
347}347}
348348
349/**349/**
@@ -362,7 +362,7 @@ function withVirtualWhitespace(text, span) {
362 if (text.match(/\s$/)) {362 if (text.match(/\s$/)) {
363 result.push($(document.createTextNode('\u200b')));363 result.push($(document.createTextNode('\u200b')));
364 }364 }
365 if (text.match(/^▁/)) {365 if (text.match(/^[▁Ġ]/)) {
366 result.unshift(document.createTextNode('\u200b'));366 result.unshift(document.createTextNode('\u200b'));
367 }367 }
368 // line breaks are trickier. we don't currently handle consecutive line368 // line breaks are trickier. we don't currently handle consecutive line