template derivation: add support for llama.cpp server backend

feb1b91619d000fb9bb5a25f8a051aab485b5f8d

Karl-Johan Alm <karljohan-alm@garage.co.jp>

Signed

public/scripts/chat-cemplates.js+1 -1 Viewed

@@ -59,6 +59,6 @@ export async function deriveTemplatesFromChatTemplate(chat_template, hash) {
59	if (hash in derivations) {	59	if (hash in derivations) {
60	return derivations[hash];	60	return derivations[hash];
61	}	61	}
62	console.log(`Unknown chat template hash: ${hash}`);	62	console.log(`Unknown chat template hash: ${hash} for [${chat_template}]`);
63	return null;	63	return null;
64	}	64	}

src/endpoints/backends/text-completions.js+15 -1 Viewed

@@ -231,6 +231,9 @@ router.post('/status', jsonParser, async function (request, response) {
231	} catch (error) {	231	} catch (error) {
232	console.error(`Failed to fetch chat template info: ${error}`);	232	console.error(`Failed to fetch chat template info: ${error}`);
233	}	233	}
		234	} else if (apiType == TEXTGEN_TYPES.LLAMACPP) {
		235	// the /props endpoint includes chat template
		236	response.setHeader('x-supports-chat-template', 'true');
234	}	237	}
235		238
236	return response.send({ result, data: data.data });	239	return response.send({ result, data: data.data });

+    }
 });
+const chat_template_endpoints = {
+    koboldcpp: '/api/extra/chat_template',
+    llamacpp: '/props',
+}
 router.post('/chat_template', jsonParser, async function (request, response) {
     if (!request.body.api_server) return response.sendStatus(400);

         setAdditionalHeaders(request, args, baseUrl);
-        const chatTemplateUrl = baseUrl + '/api/extra/chat_template';
+        const apiType = request.body.api_type;
+        const chatTemplateUrl = baseUrl + chat_template_endpoints[apiType];
         const chatTemplateReply = await fetch(chatTemplateUrl, args);
         if (!chatTemplateReply.ok) {

         /** @type {any} */
         const chatTemplate = await chatTemplateReply.json();
+        // TEMPORARY: llama.cpp's /props endpoint includes a \u0000 at the end of the chat template, resulting in mismatching hashes
+        if (apiType === TEXTGEN_TYPES.LLAMACPP && chatTemplate['chat_template'].endsWith('\u0000')) {
+            chatTemplate['chat_template'] = chatTemplate['chat_template'].slice(0, -1);
+        }
         chatTemplate['chat_template_hash'] = createHash('sha256').update(chatTemplate['chat_template']).digest('hex');
+        console.log(`We have chat template stuff: ${JSON.stringify(chatTemplate)}`);
         return response.send(chatTemplate);
     } catch (error) {
         console.error(error);