feature: derived templates This PR adds a simple hash based method for picking context and instruct templates based on the chat template, when provided by the back end.

0e2fdf37a818f287b93bf4e44bed536f5fbe249e

Karl-Johan Alm <karljohan-alm@garage.co.jp>

Signed

Parents: 67869364a531b206ae48a27c32585fdf8e57b968

public/script.js+26 -0 Viewed

 import { initServerHistory } from './scripts/server-history.js';
 import { initSettingsSearch } from './scripts/setting-search.js';
 import { initBulkEdit } from './scripts/bulk-edit.js';
 +import { deriveTemplatesFromChatTemplate } from './scripts/chat-cemplates.js';
 //exporting functions and vars for mods
 export {

         const supportsTokenization = response.headers.get('x-supports-tokenization') === 'true';
         supportsTokenization ? sessionStorage.setItem(TOKENIZER_SUPPORTED_KEY, 'true') : sessionStorage.removeItem(TOKENIZER_SUPPORTED_KEY);
 +        const supportsChatTemplate = response.headers.get('x-supports-chat-template') === 'true';
++
 +        if (supportsChatTemplate) {
 +            const response = await fetch('/api/backends/text-completions/chat_template', {
 +                method: 'POST',
 +                headers: getRequestHeaders(),
 +                body: JSON.stringify({
 +                    api_server: endpoint,
 +                    api_type: textgen_settings.type,
 +                }),
 +            });
++
 +            const data = await response.json();
 +            if (data) {
 +                const chat_template = data.chat_template;
 +                console.log(`We have chat template ${chat_template.split('\n')[0]}...`);
 +                const templates = await deriveTemplatesFromChatTemplate(chat_template);
 +                if (templates) {
 +                    const { context, instruct } = templates;
 +                    selectContextPreset(context, { isAuto: true });
 +                    selectInstructPreset(instruct, { isAuto: true });
 +                }
 +            }
 +        }
++
         // We didn't get a 200 status code, but the endpoint has an explanation. Which means it DID connect, but I digress.
         if (online_status === 'no_connection' && data.response) {
             toastr.error(data.response, t`API Error`, { timeOut: 5000, preventDuplicates: true });

public/scripts/chat-cemplates.js+76 -0 Viewed

 +// https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest
 +async function digestMessage(message) {
 +    const msgUint8 = new TextEncoder().encode(message); // encode as (utf-8) Uint8Array
 +    const hashBuffer = await window.crypto.subtle.digest('SHA-256', msgUint8); // hash the message
 +    const hashArray = Array.from(new Uint8Array(hashBuffer)); // convert buffer to byte array
 +    const hashHex = hashArray
 +        .map((b) => b.toString(16).padStart(2, '0'))
 +        .join(''); // convert bytes to hex string
 +    return hashHex;
 +}
++
 +// the hash can be obtained from command line e.g. via: MODEL=path_to_model; python -c "import json, hashlib, sys; print(hashlib.sha256(json.load(open('"$MODEL"/tokenizer_config.json'))['chat_template'].strip().encode()).hexdigest())"
 +// note that chat templates must be trimmed to match the llama.cpp metadata value
 +const derivations = {
 +    // Meta
 +    '93c0e9aa3629bbd77e68dbc0f5621f6e6b23aa8d74b932595cdb8d64684526d7': {
 +        // Meta-Llama-3.1-8B-Instruct
 +        // Meta-Llama-3.1-70B-Instruct
 +        context: 'Llama 3 Instruct',
 +        instruct: 'Llama 3 Instruct',
 +    },
 +    'd82792f95932f1c9cef5c4bd992f171225e3bf8c7b609b4557c9e1ec96be819f': {
 +        // Llama-3.2-1B-Instruct
 +        // Llama-3.2-3B-Instruct
 +        context: 'Llama 3 Instruct',
 +        instruct: 'Llama 3 Instruct',
 +    },
++
 +    // Mistral
 +    // Mistral Reference: https://github.com/mistralai/mistral-common
 +    'cafb64e0e9e5fd2503054b3479593fae39cbdfd52338ce8af9bb4664a8eb05bd': {
 +        // Mistral-Small-Instruct-2409
 +        // Mistral-Large-Instruct-2407
 +        context: 'Mistral V2 & V3',
 +        instruct: 'Mistral V2 & V3',
 +    },
 +    '3c4ad5fa60dd8c7ccdf82fa4225864c903e107728fcaf859fa6052cb80c92ee9': {
 +        // Mistral-Large-Instruct-2411
 +        context: 'Mistral V7', // https://huggingface.co/mistralai/Mistral-Large-Instruct-2411
 +        instruct: 'Mistral V7',
 +    },
 +    'e7deee034838db2bfc7487788a3013d8a307ab69f72f3c54a85f06fd76007d4e': {
 +        // Mistral-Nemo-Instruct-2407
 +        context: 'Mistral V3-Tekken',
 +        instruct: 'Mistral V3-Tekken',
 +    },
 +    '26a59556925c987317ce5291811ba3b7f32ec4c647c400c6cc7e3a9993007ba7': {
 +        // Mistral-7B-Instruct-v0.3
 +        context: 'Mistral V2 & V3',
 +        instruct: 'Mistral V2 & V3',
 +    },
++
 +    // Gemma
 +    'ecd6ae513fe103f0eb62e8ab5bfa8d0fe45c1074fa398b089c93a7e70c15cfd6': {
 +        // gemma-2-9b-it
 +        // gemma-2-27b-it
 +        context: 'Gemma 2',
 +        instruct: 'Gemma 2',
 +    },
++
 +    // Cohere
 +    '3b54f5c219ae1caa5c0bb2cdc7c001863ca6807cf888e4240e8739fa7eb9e02e': {
 +        // command-r-08-2024
 +        context: 'Command R',
 +        instruct: 'Command R',
 +    },
 +};
++
 +export async function deriveTemplatesFromChatTemplate(chat_template) {
 +    const hash = await digestMessage(chat_template);
 +    if (hash in derivations) {
 +        return derivations[hash];
 +    }
 +    console.log(`Unknown chat template hash: ${hash}`);
 +    return null;
 +}

src/endpoints/backends/text-completions.js+40 -0 Viewed

             } catch (error) {
                 console.error(`Failed to get TabbyAPI model info: ${error}`);
+            }
 +        } else if (apiType == TEXTGEN_TYPES.KOBOLDCPP) {
 +            try {
 +                const chatTemplateUrl = baseUrl + '/api/extra/chat_template';
 +                const chatTemplateReply = await fetch(chatTemplateUrl);
 +                if (chatTemplateReply.ok) {
 +                    response.setHeader('x-supports-chat-template', 'true');
 +                } else {
 +                    console.log(`ct res = ${JSON.stringify(chatTemplateReply)}`);
 +                }
 +            } catch (error) {
 +                console.error(`Failed to fetch chat template info: ${error}`);
 +            }
+        }
         return response.send({ result, data: data.data });

+    }
 });
 +router.post('/chat_template', jsonParser, async function (request, response) {
 +    if (!request.body.api_server) return response.sendStatus(400);
++
 +    try {
 +        const baseUrl = trimV1(request.body.api_server);
 +        const args = {
 +            headers: { 'Content-Type': 'application/json' },
 +        };
++
 +        setAdditionalHeaders(request, args, baseUrl);
++
 +        const chatTemplateUrl = baseUrl + '/api/extra/chat_template';
 +        const chatTemplateReply = await fetch(chatTemplateUrl, args);
++
 +        if (!chatTemplateReply.ok) {
 +            console.log('Chat template endpoint is offline.');
 +            return response.status(400);
 +        }
++
 +        /** @type {any} */
 +        const chatTemplate = await chatTemplateReply.json();
 +        return response.send(chatTemplate);
 +    } catch (error) {
 +        console.error(error);
 +        return response.status(500);
 +    }
 +});
++
 router.post('/generate', jsonParser, async function (request, response) {
     if (!request.body) return response.sendStatus(400);