Merge pull request #2428 from sasha0552/vllm-tc Add a token counting endpoint for vLLM

545b77140f06d79f49800dd25b43265eac536721

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

public/scripts/tokenizers.js+3 -2 Viewed

 import { textgen_types, textgenerationwebui_settings as textgen_settings, getTextGenServer, getTextGenModel } from './textgen-settings.js';
 import { getCurrentDreamGenModelTokenizer, getCurrentOpenRouterModelTokenizer, openRouterModels } from './textgen-models.js';
 const { OOBA, TABBY, KOBOLDCPP, VLLM, APHRODITE, LLAMACPP, OPENROUTER, DREAMGEN } = textgen_types;
 export const CHARACTERS_PER_TOKEN_RATIO = 3.35;
 const TOKENIZER_WARNING_KEY = 'tokenizationWarningShown';

     //tokenizers.NERD2,
 ];
 export const TEXTGEN_TOKENIZERS = [OOBA, TABBY, KOBOLDCPP, LLAMACPP, VLLM, APHRODITE];
 const TOKENIZER_URLS = {
     [tokenizers.GPT2]: {

         api_type: textgen_settings.type,
         url: getTextGenServer(),
         legacy_api: textgen_settings.legacy_api && (textgen_settings.type === OOBA || textgen_settings.type === APHRODITE),
 +        vllm_model: textgen_settings.vllm_model,
     };
+}

src/endpoints/tokenizers.js+5 -2 Viewed

     const text = String(request.body.text) || '';
     const baseUrl = String(request.body.url);
     const legacyApi = Boolean(request.body.legacy_api);
 +    const vllmModel = String(request.body.vllm_model) || '';
     try {
         const args = {

                     args.body = JSON.stringify({ 'content': text });
                     break;
                 case TEXTGEN_TYPES.VLLM:
 -                    return response.send({ error: true });
 +                    url += '/tokenize';
 +                    args.body = JSON.stringify({ 'model': vllmModel, 'prompt': text });
 +                    break;
                 case TEXTGEN_TYPES.APHRODITE:
                     url += '/v1/tokenize';
                     args.body = JSON.stringify({ 'prompt': text });

+        }
         const data = await result.json();
         const count = legacyApi ? data?.results[0]?.tokens : (data?.length ?? data?.count ?? data?.value ?? data?.tokens?.length);
         const ids = legacyApi ? [] : (data?.tokens ?? data?.ids ?? []);
         return response.send({ count, ids });