Add encode_special_tokens to tokenizers.js (#5589)

95ca4315bd889bde2cdfadf7e097638e9543611c

Forkoz <59298527+Ph0rk0z@users.noreply.github.com>

Signed
Parents: 97392a4ca0357f089c9e6e7250d76fc73d5cc3ea
1 files changed, +1 -1Ignore whitespace
src/endpoints/tokenizers.js+1 -1
@@ -1093,7 +1093,7 @@ router.post('/remote/textgenerationwebui/encode', async function (request, respo
10931093 switch (request.body.api_type) {
10941094 case TEXTGEN_TYPES.TABBY:
10951095 url += '/v1/token/encode';
10961096 args.body = JSON.stringify({ 'text': text, 'add_bos_token': false, 'encode_special_tokens': false });
10971097 break;
10981098 case TEXTGEN_TYPES.KOBOLDCPP:
10991099 url += '/api/extra/tokencount';