template derivation: add support for llama.cpp server backend

feb1b91619d000fb9bb5a25f8a051aab485b5f8d

Karl-Johan Alm <karljohan-alm@garage.co.jp>

Signed

public/scripts/chat-cemplates.js+1 -1 Viewed

@@ -59,6 +59,6 @@ export async function deriveTemplatesFromChatTemplate(chat_template, hash) {
59	59	if (hash in derivations) {
60	60	return derivations[hash];
61	61	}
62	62	console.log(`Unknown chat template hash: ${hash} for [${chat_template}]`);
63	63	return null;
64	64	}

src/endpoints/backends/text-completions.js+15 -1 Viewed

@@ -231,6 +231,9 @@ router.post('/status', jsonParser, async function (request, response) {
231	231	} catch (error) {
232	232	console.error(`Failed to fetch chat template info: ${error}`);
233	233	}
	234	+ } else if (apiType == TEXTGEN_TYPES.LLAMACPP) {
	235	+ // the /props endpoint includes chat template
	236	+ response.setHeader('x-supports-chat-template', 'true');
234	237	}
235	238
236	239	return response.send({ result, data: data.data });

+    }
 });
 +const chat_template_endpoints = {
 +    koboldcpp: '/api/extra/chat_template',
 +    llamacpp: '/props',
 +}
++
 router.post('/chat_template', jsonParser, async function (request, response) {
     if (!request.body.api_server) return response.sendStatus(400);

         setAdditionalHeaders(request, args, baseUrl);
 -        const chatTemplateUrl = baseUrl + '/api/extra/chat_template';
 +        const apiType = request.body.api_type;
 +        const chatTemplateUrl = baseUrl + chat_template_endpoints[apiType];
         const chatTemplateReply = await fetch(chatTemplateUrl, args);
         if (!chatTemplateReply.ok) {

         /** @type {any} */
         const chatTemplate = await chatTemplateReply.json();
 +        // TEMPORARY: llama.cpp's /props endpoint includes a \u0000 at the end of the chat template, resulting in mismatching hashes
 +        if (apiType === TEXTGEN_TYPES.LLAMACPP && chatTemplate['chat_template'].endsWith('\u0000')) {
 +            chatTemplate['chat_template'] = chatTemplate['chat_template'].slice(0, -1);
 +        }
         chatTemplate['chat_template_hash'] = createHash('sha256').update(chatTemplate['chat_template']).digest('hex');
 +        console.log(`We have chat template stuff: ${JSON.stringify(chatTemplate)}`);
         return response.send(chatTemplate);
     } catch (error) {
         console.error(error);