correctly interpret some alternate whitespaces in token names

5d5e552cbdaf46bd123a3ce5a90057e93d545e09

50h100a <all_awful@proton.me>

public/scripts/extensions/token-counter/index.js+1 -1 Viewed

     $('#tokenized_chunks_display').empty();
     for (let i = 0; i < chunks.length; i++) {
         let chunk = chunks[i].replace(/[▁Ġ]/g, ' '); // This is a leading space in sentencepiece. More info: Lower one eighth block (U+2581)
         // If <0xHEX>, decode it
         if (/^<0x[0-9A-F]+>$/i.test(chunk)) {

public/scripts/logprobs.js+4 -4 Viewed

     let matched = false;
     for (const [token, probability, log] of candidates) {
         const container = $('<button class="flex-container flexFlowColumn logprobs_top_candidate"></button>');
         const tokenNormalized = String(token).replace(/^[▁Ġ]/g, ' ');
         if (token === selectedToken || tokenNormalized === selectedToken) {
             matched = true;

     const replaceIndex = messageLogprobs.findIndex(x => x === tokenLogprobs);
     const tokens = messageLogprobs.slice(0, replaceIndex + 1).map(({ token }) => token);
     tokens[replaceIndex] = String(alternative).replace(/^[▁Ġ]/g, ' ').replace(/Ċ/g, '\n');
     const prefix = continueFrom || '';
     const prompt = prefix + tokens.join('');

  * @returns {string}
  */
 function toVisibleWhitespace(input) {
     return input.replace(/ /g, '·').replace(/[▁Ġ]/g, '·').replace(/[Ċ\n]/g, '↵');
+}
 /**

@@ -362,7 +362,7 @@ function withVirtualWhitespace(text, span) {
362	362	if (text.match(/\s$/)) {
363	363	result.push($(document.createTextNode('\u200b')));
364	364	}
365	365	if (text.match(/^[▁Ġ]/)) {
366	366	result.unshift(document.createTextNode('\u200b'));
367	367	}
368	368	// line breaks are trickier. we don't currently handle consecutive line