template derivation: move hash part to backend

f25ea9f6d6a285a5250ff35286d3a629df381f04

Karl-Johan Alm <karljohan-alm@garage.co.jp>

Signed
Parents: 0e2fdf37a818f287b93bf4e44bed536f5fbe249e
5 files changed, +13 -15Ignore whitespace
package-lock.json+7 -0
@@ -43,6 +43,7 @@
43 "ip-matching": "^2.1.2",43 "ip-matching": "^2.1.2",
44 "ipaddr.js": "^2.0.1",44 "ipaddr.js": "^2.0.1",
45 "jimp": "^0.22.10",45 "jimp": "^0.22.10",
46 "js-sha256": "^0.11.0",
46 "localforage": "^1.10.0",47 "localforage": "^1.10.0",
47 "lodash": "^4.17.21",48 "lodash": "^4.17.21",
48 "mime-types": "^2.1.35",49 "mime-types": "^2.1.35",
@@ -4882,6 +4883,12 @@
4882 "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==",4883 "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==",
4883 "license": "BSD-3-Clause"4884 "license": "BSD-3-Clause"
4884 },4885 },
4886 "node_modules/js-sha256": {
4887 "version": "0.11.0",
4888 "resolved": "https://registry.npmjs.org/js-sha256/-/js-sha256-0.11.0.tgz",
4889 "integrity": "sha512-6xNlKayMZvds9h1Y1VWc0fQHQ82BxTXizWPEtEeGvmOUYpBRy4gbWroHLpzowe6xiQhHpelCQiE7HEdznyBL9Q==",
4890 "license": "MIT"
4891 },
4885 "node_modules/js-yaml": {4892 "node_modules/js-yaml": {
4886 "version": "4.1.0",4893 "version": "4.1.0",
4887 "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",4894 "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
package.json+1 -0
@@ -33,6 +33,7 @@
33 "ip-matching": "^2.1.2",33 "ip-matching": "^2.1.2",
34 "ipaddr.js": "^2.0.1",34 "ipaddr.js": "^2.0.1",
35 "jimp": "^0.22.10",35 "jimp": "^0.22.10",
36 "js-sha256": "^0.11.0",
36 "localforage": "^1.10.0",37 "localforage": "^1.10.0",
37 "lodash": "^4.17.21",38 "lodash": "^4.17.21",
38 "mime-types": "^2.1.35",39 "mime-types": "^2.1.35",
public/script.js+2 -2
@@ -1250,9 +1250,9 @@ async function getStatusTextgen() {
12501250
1251 const data = await response.json();1251 const data = await response.json();
1252 if (data) {1252 if (data) {
1253 const chat_template = data.chat_template;1253 const { chat_template, chat_template_hash } = data;
1254 console.log(`We have chat template ${chat_template.split('\n')[0]}...`);1254 console.log(`We have chat template ${chat_template.split('\n')[0]}...`);
1255 const templates = await deriveTemplatesFromChatTemplate(chat_template);1255 const templates = await deriveTemplatesFromChatTemplate(chat_template, chat_template_hash);
1256 if (templates) {1256 if (templates) {
1257 const { context, instruct } = templates;1257 const { context, instruct } = templates;
1258 selectContextPreset(context, { isAuto: true });1258 selectContextPreset(context, { isAuto: true });
public/scripts/chat-cemplates.js+1 -13
@@ -1,14 +1,3 @@
1// https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest
2async function digestMessage(message) {
3 const msgUint8 = new TextEncoder().encode(message); // encode as (utf-8) Uint8Array
4 const hashBuffer = await window.crypto.subtle.digest('SHA-256', msgUint8); // hash the message
5 const hashArray = Array.from(new Uint8Array(hashBuffer)); // convert buffer to byte array
6 const hashHex = hashArray
7 .map((b) => b.toString(16).padStart(2, '0'))
8 .join(''); // convert bytes to hex string
9 return hashHex;
10}
11
12// the hash can be obtained from command line e.g. via: MODEL=path_to_model; python -c "import json, hashlib, sys; print(hashlib.sha256(json.load(open('"$MODEL"/tokenizer_config.json'))['chat_template'].strip().encode()).hexdigest())"1// the hash can be obtained from command line e.g. via: MODEL=path_to_model; python -c "import json, hashlib, sys; print(hashlib.sha256(json.load(open('"$MODEL"/tokenizer_config.json'))['chat_template'].strip().encode()).hexdigest())"
13// note that chat templates must be trimmed to match the llama.cpp metadata value2// note that chat templates must be trimmed to match the llama.cpp metadata value
14const derivations = {3const derivations = {
@@ -66,8 +55,7 @@ const derivations = {
66 },55 },
67};56};
6857
69export async function deriveTemplatesFromChatTemplate(chat_template) {58export async function deriveTemplatesFromChatTemplate(chat_template, hash) {
70 const hash = await digestMessage(chat_template);
71 if (hash in derivations) {59 if (hash in derivations) {
72 return derivations[hash];60 return derivations[hash];
73 }61 }
src/endpoints/backends/text-completions.js+2 -0
@@ -16,6 +16,7 @@ import {
16} from '../../constants.js';16} from '../../constants.js';
17import { forwardFetchResponse, trimV1, getConfigValue } from '../../util.js';17import { forwardFetchResponse, trimV1, getConfigValue } from '../../util.js';
18import { setAdditionalHeaders } from '../../additional-headers.js';18import { setAdditionalHeaders } from '../../additional-headers.js';
19import { sha256 } from 'js-sha256';
1920
20export const router = express.Router();21export const router = express.Router();
2122
@@ -260,6 +261,7 @@ router.post('/chat_template', jsonParser, async function (request, response) {
260261
261 /** @type {any} */262 /** @type {any} */
262 const chatTemplate = await chatTemplateReply.json();263 const chatTemplate = await chatTemplateReply.json();
264 chatTemplate['chat_template_hash'] = sha256.create().update(chatTemplate['chat_template']).hex();
263 return response.send(chatTemplate);265 return response.send(chatTemplate);
264 } catch (error) {266 } catch (error) {
265 console.error(error);267 console.error(error);