added substr derivations with one example

3789381c6cbd0b495c31a3b6c5236d57d4b7eb79

Karl-Johan Alm <karljohan-alm@garage.co.jp>

Signed
Parents: c30dde847127ebe0fa4ba3601f7a4566937d4979
2 files changed, +18 -8Showing whitespace changes
public/script.js+1 -1
@@ -1252,7 +1252,7 @@ async function getStatusTextgen() {
1252 const data = await response.json();1252 const data = await response.json();
1253 if (data) {1253 if (data) {
1254 const { chat_template, chat_template_hash } = data;1254 const { chat_template, chat_template_hash } = data;
1255 console.log(`${wantsContextDerivation} ${wantsInstructDerivation} We have chat template ${chat_template.split('\n')[0]}...`);1255 console.log(`We have chat template ${chat_template.split('\n')[0]}...`);
1256 const templates = await deriveTemplatesFromChatTemplate(chat_template, chat_template_hash);1256 const templates = await deriveTemplatesFromChatTemplate(chat_template, chat_template_hash);
1257 if (templates) {1257 if (templates) {
1258 const { context, instruct } = templates;1258 const { context, instruct } = templates;
public/scripts/chat-cemplates.js+17 -7
@@ -1,6 +1,6 @@
1// the hash can be obtained from command line e.g. via: MODEL=path_to_model; python -c "import json, hashlib, sys; print(hashlib.sha256(json.load(open('"$MODEL"/tokenizer_config.json'))['chat_template'].encode()).hexdigest())"1// the hash can be obtained from command line e.g. via: MODEL=path_to_model; python -c "import json, hashlib, sys; print(hashlib.sha256(json.load(open('"$MODEL"/tokenizer_config.json'))['chat_template'].encode()).hexdigest())"
2// note that chat templates must be trimmed to match the llama.cpp metadata value2// note that chat templates must be trimmed to match the llama.cpp metadata value
3const derivations = {3const hash_derivations = {
4 // Meta4 // Meta
5 'e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65':5 'e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65':
6 // Meta-Llama-3.1-8B-Instruct6 // Meta-Llama-3.1-8B-Instruct
@@ -51,17 +51,27 @@ const derivations = {
51 ,51 ,
52};52};
5353
54export async function deriveTemplatesFromChatTemplate(chat_template, hash) {54const substr_derivations = {
55 if (hash in derivations) {55 '<|im_start|>': 'ChatML', // qwen2.5, ...
56 const derivation = derivations[hash];56}
57 if (typeof derivation === 'string') {57
58 return {58const parse_derivation = derivation => (typeof derivation === 'string') ? {
59 'context': derivation,59 'context': derivation,
60 'instruct': derivation,60 'instruct': derivation,
61} : derivation;
62
63export async function deriveTemplatesFromChatTemplate(chat_template, hash) {
64 if (hash in hash_derivations) {
65 return parse_derivation(hash_derivations[hash]);
61 }66 }
67
68 // heuristics
69 for (const [substr, derivation] of Object.entries(substr_derivations) ) {
70 if (chat_template.includes(substr)) {
71 return parse_derivation(derivation);
62 }72 }
63 return derivation;
64 }73 }
74
65 console.log(`Unknown chat template hash: ${hash} for [${chat_template}]`);75 console.log(`Unknown chat template hash: ${hash} for [${chat_template}]`);
66 return null;76 return null;
67}77}