Add encode_special_tokens to tokenizers.js (#5589)

95ca4315bd889bde2cdfadf7e097638e9543611c

Forkoz <59298527+Ph0rk0z@users.noreply.github.com>

Signed

src/endpoints/tokenizers.js+1 -1 Viewed

         switch (request.body.api_type) {
             case TEXTGEN_TYPES.TABBY:
                 url += '/v1/token/encode';
-                args.body = JSON.stringify({ 'text': text, 'add_bos_token': false });
+                args.body = JSON.stringify({ 'text': text, 'add_bos_token': false, 'encode_special_tokens': false });
                 break;
             case TEXTGEN_TYPES.KOBOLDCPP:
                 url += '/api/extra/tokencount';