Enhance text generation API to support model parameter for llama.cpp router mode. Update getStatusTextgen to include model in request body and improve context size handling. (#4914)

29456b9bd22bd39bd72b016712d4d02a331e70f0

my-alt <250851737+my-alt-acct@users.noreply.github.com>

Signed

public/scripts/textgen-settings.js+7 -0 Viewed

         const supportsChatTemplate = [textgen_types.KOBOLDCPP, textgen_types.LLAMACPP].includes(textgenerationwebui_settings.type);
         if (supportsChatTemplate && (wantsInstructDerivation || wantsContextDerivation || wantsContextSize)) {
+            const model = textgenerationwebui_settings.type === textgen_types.LLAMACPP
+                ? textgenerationwebui_settings.llamacpp_model
+                : undefined;
             const response = await fetch('/api/backends/text-completions/props', {
                 method: 'POST',
                 headers: getRequestHeaders(),
                 body: JSON.stringify({
                     api_server: endpoint,
                     api_type: textgenerationwebui_settings.type,
+                    model: model,
                 }),
             });

                     if (wantsContextSize && 'default_generation_settings' in data) {
                         const backend_max_context = data['default_generation_settings']['n_ctx'];
+                        if (backend_max_context && typeof backend_max_context === 'number') {
                             const old_value = max_context;
                             if (max_context !== backend_max_context) {
                                 setGenerationParamsFromPreset({ max_length: backend_max_context });

@@ -756,6 +762,7 @@ async function getStatusTextgen() {
756	toastr.info(`${old_value} ⇒ ${max_context}`, 'Context Size Changed');	762	toastr.info(`${old_value} ⇒ ${max_context}`, 'Context Size Changed');
757	}	763	}
758	}	764	}
		765	}
759	console.log(`We have chat template ${chat_template.split('\n')[0]}...`);	766	console.log(`We have chat template ${chat_template.split('\n')[0]}...`);
760	const savedTemplate = power_user.model_templates_mappings[chat_template_hash];	767	const savedTemplate = power_user.model_templates_mappings[chat_template_hash];
761	const derivedTemplate = await deriveTemplatesFromChatTemplate(chat_template, chat_template_hash);	768	const derivedTemplate = await deriveTemplatesFromChatTemplate(chat_template, chat_template_hash);

src/endpoints/backends/text-completions.js+6 -2 Viewed

         setAdditionalHeaders(request, args, baseUrl);
         const apiType = request.body.api_type;
-        const propsUrl = baseUrl + '/props';
+        let propsUrl = baseUrl + '/props';
+        if (apiType === TEXTGEN_TYPES.LLAMACPP && request.body.model) {
+            propsUrl += `?model=${encodeURIComponent(request.body.model)}`;
+            console.debug(`Querying llama-server props with model parameter: ${request.body.model}`);
+        }
         const propsReply = await fetch(propsUrl, args);
         if (!propsReply.ok) {

         /** @type {any} */
         const props = await propsReply.json();
         // TEMPORARY: llama.cpp's /props endpoint has a bug which replaces the last newline with a \0
-        if (apiType === TEXTGEN_TYPES.LLAMACPP && props['chat_template'].endsWith('\u0000')) {
+        if (apiType === TEXTGEN_TYPES.LLAMACPP && props['chat_template'] && props['chat_template'].endsWith('\u0000')) {
             props['chat_template'] = props['chat_template'].slice(0, -1) + '\n';
+        }
         props['chat_template_hash'] = createHash('sha256').update(props['chat_template']).digest('hex');