Merge pull request #3090 from kallewoof/202411-auto-templates Derived templates

ba91845ced2f72193b88e6f60e133a49f86f52b8

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

Parents: 7cd68d827bed46c7d15ff54825074dd79326395b 803b7fca8941cb0bbce8c821f120f3ea3d080fa0

public/index.html+24 -22 Viewed

                                 <div class="margin0 title_restorable standoutHeader">
                                     <strong>
                                         <span data-i18n="kobldpresets">Kobold Presets</span>
 -                                        <a href="https://docs.sillytavern.app/usage/api-connections/koboldai/" class="notes-link" target="_blank">
 -                                            <span class="fa-solid fa-circle-question note-link-span"></span>
 -                                        </a>
                                     </strong>
                                     <div class="flex-container gap3px">

             </div>
             <div id="AdvancedFormatting" class="drawer-content">
                 <div class="flex-container alignItemsBaseline">
                     <h3 class="margin0 flex1" dataflex-i18n="Advancedcontainer FormattingalignItemsBaseline">
 -                        Advanced Formatting
 +                        <span data-i18n="Advanced Formatting">
 +                            Advanced Formatting
 +                        </span>
++
 +                        <a href="https://docs.sillytavern.app/usage/prompts/" class="notes-link" target="_blank">
 +                            <span class="fa-solid fa-circle-question note-link-span"></span>
 +                        </a>
                     </h3>
                     <div class="flex-container">
                         <input id="af_master_import_file" type="file" hidden accept=".json" class="displayNone">

                             <h4 class="standoutHeader title_restorable">
                                 <div>
                                     <span data-i18n="Context Template">Context Template</span>
 -                                    <a href="https://docs.sillytavern.app/usage/core-concepts/advancedformatting/#context-template" class="notes-link" target="_blank">
 +                                </div>
 -                                        <span class="fa-solid fa-circle-question note-link-span"></span>
 +                                <div class="flex-container">
 -                                    </a>
 +                                    <label for="context_derived" class="checkbox_label flex1" title="Derive from Model Metadata, if possible." data-i18n="[title]context_derived">
 +                                        <input id="context_derived" type="checkbox" style="display:none;" />
 +                                        <small><i class="fa-solid fa-bolt menu_button margin0"></i></small>
 +                                    </label>
                                 </div>
                             </h4>
                             <div class="flex-container" title="Select your current Context Template" data-i18n="[title]Select your current Context Template">

                         <h4 class="standoutHeader title_restorable justifySpaceBetween">
                             <div class="flex-container">
                                 <span data-i18n="Instruct Template">Instruct Template</span>
 -                                <a href="https://docs.sillytavern.app/usage/core-concepts/instructmode/" class="notes-link" target="_blank">
 -                                    <span class="fa-solid fa-circle-question note-link-span"></span>
 -                                </a>
                             </div>
                             <div class="flex-container">
 +                                <label for="instruct_derived" class="checkbox_label flex1" title="Derive from Model Metadata, if possible." data-i18n="[title]instruct_derived">
 +                                    <input id="instruct_derived" type="checkbox" style="display:none;" />
 +                                    <small><i class="fa-solid fa-bolt menu_button margin0"></i></small>
 +                                </label>
                                 <label for="instruct_bind_to_context" class="checkbox_label flex1" title="Bind to Context&#10If enabled, Context templates will be automatically selected based on selected Instruct template name or by preference." data-i18n="[title]instruct_bind_to_context">
                                     <input id="instruct_bind_to_context" type="checkbox" style="display:none;" />
                                     <small><i class="fa-solid fa-link menu_button margin0"></i></small>

                         <h4 class="standoutHeader title_restorable justifySpaceBetween">
                             <div class="flex-container">
                                 <span data-i18n="System Prompt">System Prompt</span>
 -                                <a href="https://docs.sillytavern.app/usage/core-concepts/advancedformatting/#system-prompt" class="notes-link" target="_blank">
 -                                    <span class="fa-solid fa-circle-question note-link-span"></span>
 -                                </a>
                             </div>
                             <div class="flex-container">
                                 <label id="sysprompt_enabled_label" for="sysprompt_enabled" class="checkbox_label flex1" title="Enable System Prompt" data-i18n="[title]sysprompt_enabled">

                         <div name="tokenizerSettingsBlock">
                             <div name="tokenizerSelectorBlock">
                                 <h4 class="standoutHeader"><span data-i18n="Tokenizer">Tokenizer</span>
                                     <a href="https://docs.sillytavern.app/usage/core-concepts/advancedformattingprompts/#tokenizer/" class="notes-link" target="_blank">
                                         <span class="fa-solid fa-circle-question note-link-span"></span>
                                     </a>
                                 </h4>

                             </div>
                             <div class="range-block flex-container flexnowrap" name="tokenPaddingBlock">
                                 <div class="range-block-title justifyLeft">
                                     <small data-i18n="Token Padding" class="flex-container">Token Padding
 -                                        <a href="https://docs.sillytavern.app/usage/core-concepts/advancedformatting/#token-padding" class="notes-link" target="_blank">
 +                                        Token Padding
 -                                            <span class="fa-solid fa-circle-question note-link-span"></span>
 -                                        </a>
                                     </small>
                                 </div>
                                 <input id="token_padding" class="text_pole textarea_compact" type="number" min="-2048" max="2048" />

                     </div>
                     <h3 class="margin0">
                         <span data-i18n="Worlds/Lorebooks">Worlds/Lorebooks</span>
                         <a href="https://docs.sillytavern.app/usage/core-concepts/worldinfo/" class="notes-link" target="_blank">
                             <span class="fa-solid fa-circle-question note-link-span"></span>
                         </a>
                     </h3>

                                 <audio id="audio_message_sound" src="sounds/message.mp3" hidden></audio>
                                 <span>
                                     <small data-i18n="Message Sound">Message Sound</small>
                                     <a href="https://docs.sillytavern.app/usage/core-conceptsuser_settings/uicustomization/#message-sound" class="notes-link" target="_blank">
                                         <span class="fa-solid fa-circle-question note-link-span"></span>
                                     </a>
                                 </span>

                                     <div class="flex-container justifySpaceBetween">
                                         <small for="group">
                                             <span data-i18n="Inclusion Group">Inclusion Group</span>
                                             <a href="https://docs.sillytavern.app/usage/core-concepts/worldinfo/#inclusion-group" class="notes-link" target="_blank" title="Inclusion Groups ensure only one entry from a group is activated at a time, if multiple are triggered.&#13;Supports multiple comma-separated groups.&#13;&#13;Documentation: World Info - Inclusion Group" data-i18n="[title]Inclusion Groups ensure only one entry from a group is activated at a time, if multiple are triggered.Documentation: World Info - Inclusion Group">
                                                 <span class="fa-solid fa-circle-question note-link-span"></span>
                                             </a>
                                         </small>

public/script.js+33 -0 Viewed

 import { initServerHistory } from './scripts/server-history.js';
 import { initSettingsSearch } from './scripts/setting-search.js';
 import { initBulkEdit } from './scripts/bulk-edit.js';
 +import { deriveTemplatesFromChatTemplate } from './scripts/chat-templates.js';
 //exporting functions and vars for mods
 export {

         const supportsTokenization = response.headers.get('x-supports-tokenization') === 'true';
         supportsTokenization ? sessionStorage.setItem(TOKENIZER_SUPPORTED_KEY, 'true') : sessionStorage.removeItem(TOKENIZER_SUPPORTED_KEY);
 +        const wantsInstructDerivation = (power_user.instruct.enabled && power_user.instruct.derived);
 +        const wantsContextDerivation = power_user.context_derived;
 +        const supportsChatTemplate = [textgen_types.KOBOLDCPP, textgen_types.LLAMACPP].includes(textgen_settings.type);
 +        if (supportsChatTemplate && (wantsInstructDerivation || wantsContextDerivation)) {
 +            const response = await fetch('/api/backends/text-completions/props', {
 +                method: 'POST',
 +                headers: getRequestHeaders(),
 +                body: JSON.stringify({
 +                    api_server: endpoint,
 +                    api_type: textgen_settings.type,
 +                }),
 +            });
++
 +            if (response.ok) {
 +                const data = await response.json();
 +                if (data) {
 +                    const { chat_template, chat_template_hash } = data;
 +                    console.log(`We have chat template ${chat_template.split('\n')[0]}...`);
 +                    const templates = await deriveTemplatesFromChatTemplate(chat_template, chat_template_hash);
 +                    if (templates) {
 +                        const { context, instruct } = templates;
 +                        if (wantsContextDerivation) {
 +                            selectContextPreset(context, { isAuto: true });
 +                        }
 +                        if (wantsInstructDerivation) {
 +                            selectInstructPreset(instruct, { isAuto: true });
 +                        }
 +                    }
 +                }
 +            }
 +        }
++
         // We didn't get a 200 status code, but the endpoint has an explanation. Which means it DID connect, but I digress.
         if (online_status === 'no_connection' && data.response) {
             toastr.error(data.response, t`API Error`, { timeOut: 5000, preventDuplicates: true });

public/scripts/chat-templates.js+81 -0 Viewed

 +// the hash can be obtained from command line e.g. via: MODEL=path_to_model; python -c "import json, hashlib, sys; print(hashlib.sha256(json.load(open('"$MODEL"/tokenizer_config.json'))['chat_template'].encode()).hexdigest())"
 +// note that chat templates must be trimmed to match the llama.cpp metadata value
 +const hash_derivations = {
 +    // Meta
 +    'e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65':
 +        // Meta-Llama-3.1-8B-Instruct
 +        // Meta-Llama-3.1-70B-Instruct
 +        'Llama 3 Instruct'
 +    ,
 +    '5816fce10444e03c2e9ee1ef8a4a1ea61ae7e69e438613f3b17b69d0426223a4':
 +        // Llama-3.2-1B-Instruct
 +        // Llama-3.2-3B-Instruct
 +        'Llama 3 Instruct'
 +    ,
 +    '73e87b1667d87ab7d7b579107f01151b29ce7f3ccdd1018fdc397e78be76219d':
 +        // Nemotron 70B
 +        'Llama 3 Instruct'
 +    ,
++
 +    // Mistral
 +    // Mistral Reference: https://github.com/mistralai/mistral-common
 +    'e16746b40344d6c5b5265988e0328a0bf7277be86f1c335156eae07e29c82826':
 +        // Mistral-Small-Instruct-2409
 +        // Mistral-Large-Instruct-2407
 +        'Mistral V2 & V3'
 +    ,
 +    '3c4ad5fa60dd8c7ccdf82fa4225864c903e107728fcaf859fa6052cb80c92ee9':
 +        // Mistral-Large-Instruct-2411
 +        'Mistral V7' // https://huggingface.co/mistralai/Mistral-Large-Instruct-2411
 +    ,
 +    'e4676cb56dffea7782fd3e2b577cfaf1e123537e6ef49b3ec7caa6c095c62272':
 +        // Mistral-Nemo-Instruct-2407
 +        'Mistral V3-Tekken'
 +    ,
 +    '26a59556925c987317ce5291811ba3b7f32ec4c647c400c6cc7e3a9993007ba7':
 +        // Mistral-7B-Instruct-v0.3
 +        'Mistral V2 & V3'
 +    ,
++
 +    // Gemma
 +    'ecd6ae513fe103f0eb62e8ab5bfa8d0fe45c1074fa398b089c93a7e70c15cfd6':
 +        // gemma-2-9b-it
 +        // gemma-2-27b-it
 +        'Gemma 2'
 +    ,
 +    '87fa45af6cdc3d6a9e4dd34a0a6848eceaa73a35dcfe976bd2946a5822a38bf3':
 +        // gemma-2-2b-it
 +        'Gemma 2'
 +    ,
++
 +    // Cohere
 +    '3b54f5c219ae1caa5c0bb2cdc7c001863ca6807cf888e4240e8739fa7eb9e02e':
 +        // command-r-08-2024
 +        'Command R'
 +    ,
 +};
++
 +const substr_derivations = {
 +    '<|im_start|>': 'ChatML', // qwen2.5, ...
 +};
++
 +const parse_derivation = derivation => (typeof derivation === 'string') ? {
 +    'context': derivation,
 +    'instruct': derivation,
 +} : derivation;
++
 +export async function deriveTemplatesFromChatTemplate(chat_template, hash) {
 +    if (hash in hash_derivations) {
 +        return parse_derivation(hash_derivations[hash]);
 +    }
++
 +    // heuristics
 +    for (const [substr, derivation] of Object.entries(substr_derivations) ) {
 +        if (chat_template.includes(substr)) {
 +            return parse_derivation(derivation);
 +        }
 +    }
++
 +    console.log(`Unknown chat template hash: ${hash} for [${chat_template}]`);
 +    return null;
 +}

public/scripts/instruct-mode.js+18 -0 Viewed

     { id: 'instruct_first_input_sequence', property: 'first_input_sequence', isCheckbox: false },
     { id: 'instruct_last_input_sequence', property: 'last_input_sequence', isCheckbox: false },
     { id: 'instruct_activation_regex', property: 'activation_regex', isCheckbox: false },
 +    { id: 'instruct_derived', property: 'derived', isCheckbox: true },
     { id: 'instruct_bind_to_context', property: 'bind_to_context', isCheckbox: true },
     { id: 'instruct_skip_examples', property: 'skip_examples', isCheckbox: true },
     { id: 'instruct_names_behavior', property: 'names_behavior', isCheckbox: false },

     $('#instruct_enabled').parent().find('i').toggleClass('toggleEnabled', !!power_user.instruct.enabled);
     $('#instructSettingsBlock, #InstructSequencesColumn').toggleClass('disabled', !power_user.instruct.enabled);
 +    $('#instruct_derived').parent().find('i').toggleClass('toggleEnabled', !!power_user.instruct.derived);
     $('#instruct_bind_to_context').parent().find('i').toggleClass('toggleEnabled', !!power_user.instruct.bind_to_context);
     controls.forEach(control => {

  * @param {boolean} [options.isAuto=false] Is auto-select.
  */
 export function selectContextPreset(preset, { quiet = false, isAuto = false } = {}) {
 +    const presetExists = context_presets.some(x => x.name === preset);
 +    if (!presetExists) {
 +        console.warn(`Context template "${preset}" not found`);
 +        return;
 +    }
++
     // If context template is not already selected, select it
     if (preset !== power_user.context.preset) {
         $('#context_presets').val(preset).trigger('change');

  * @param {boolean} [options.isAuto=false] Is auto-select.
  */
 export function selectInstructPreset(preset, { quiet = false, isAuto = false } = {}) {
 +    const presetExists = instruct_presets.some(x => x.name === preset);
 +    if (!presetExists) {
 +        console.warn(`Instruct template "${preset}" not found`);
 +        return;
 +    }
++
     // If instruct preset is not already selected, select it
     if (preset !== power_user.instruct.preset) {
         $('#instruct_presets').val(preset).trigger('change');

+        }
     });
 +    $('#instruct_derived').on('change', function () {
 +        $('#instruct_derived').parent().find('i').toggleClass('toggleEnabled', !!power_user.instruct.derived);
 +    });
++
     $('#instruct_bind_to_context').on('change', function () {
         $('#instruct_bind_to_context').parent().find('i').toggleClass('toggleEnabled', !!power_user.instruct.bind_to_context);
     });

public/scripts/power-user.js+14 -0 Viewed

         macro: true,
         names_behavior: names_behavior_types.FORCE,
         activation_regex: '',
 +        derived: false,
         bind_to_context: false,
         user_alignment_message: '',
         system_same_as_user: false,

         names_as_stop_strings: true,
     },
 +    context_derived: false,
++
     sysprompt: {
         enabled: true,
         name: 'Neutral - Chat',

     $('#encode_tags').prop('checked', power_user.encode_tags);
     $('#example_messages_behavior').val(getExampleMessagesBehavior());
     $(`#example_messages_behavior option[value="${getExampleMessagesBehavior()}"]`).prop('selected', true);
 +    $('#context_derived').parent().find('i').toggleClass('toggleEnabled', !!power_user.context_derived);
     $('#console_log_prompts').prop('checked', power_user.console_log_prompts);
     $('#request_token_probabilities').prop('checked', power_user.request_token_probabilities);

         saveSettingsDebounced();
     });
 +    $('#context_derived').on('input', function () {
 +        const value = !!$(this).prop('checked');
 +        power_user.context_derived = value;
 +        saveSettingsDebounced();
 +    });
++
 +    $('#context_derived').on('change', function () {
 +        $('#context_derived').parent().find('i').toggleClass('toggleEnabled', !!power_user.context_derived);
 +    });
++
     $('#always-force-name2-checkbox').change(function () {
         power_user.always_force_name2 = !!$(this).prop('checked');
         saveSettingsDebounced();

public/scripts/preset-manager.js+1 -0 Viewed

             'openrouter_providers',
             'openrouter_allow_fallbacks',
             'tabby_model',
 +            'derived',
         ];
         const settings = Object.assign({}, getSettingsByApiId(this.apiId));

src/endpoints/backends/text-completions.js+35 -0 Viewed

 } from '../../constants.js';
 import { forwardFetchResponse, trimV1, getConfigValue } from '../../util.js';
 import { setAdditionalHeaders } from '../../additional-headers.js';
 +import { createHash } from 'node:crypto';
 export const router = express.Router();

+    }
 });
 +router.post('/props', jsonParser, async function (request, response) {
 +    if (!request.body.api_server) return response.sendStatus(400);
++
 +    try {
 +        const baseUrl = trimV1(request.body.api_server);
 +        const args = {
 +            headers: {},
 +        };
++
 +        setAdditionalHeaders(request, args, baseUrl);
++
 +        const apiType = request.body.api_type;
 +        const propsUrl = baseUrl + '/props';
 +        const propsReply = await fetch(propsUrl, args);
++
 +        if (!propsReply.ok) {
 +            return response.status(400);
 +        }
++
 +        /** @type {any} */
 +        const props = await propsReply.json();
 +        // TEMPORARY: llama.cpp's /props endpoint has a bug which replaces the last newline with a \0
 +        if (apiType === TEXTGEN_TYPES.LLAMACPP && props['chat_template'].endsWith('\u0000')) {
 +            props['chat_template'] = props['chat_template'].slice(0, -1) + '\n';
 +        }
 +        props['chat_template_hash'] = createHash('sha256').update(props['chat_template']).digest('hex');
 +        console.log(`Model properties: ${JSON.stringify(props)}`);
 +        return response.send(props);
 +    } catch (error) {
 +        console.error(error);
 +        return response.status(500);
 +    }
 +});
++
 router.post('/generate', jsonParser, async function (request, response) {
     if (!request.body) return response.sendStatus(400);