Merge pull request #3090 from kallewoof/202411-auto-templates Derived templates

ba91845ced2f72193b88e6f60e133a49f86f52b8

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: 7cd68d827bed46c7d15ff54825074dd79326395b 803b7fca8941cb0bbce8c821f120f3ea3d080fa0
7 files changed, +205 -21Showing whitespace changes
public/index.html+23 -21
@@ -91,9 +91,6 @@
9191 <div class="margin0 title_restorable standoutHeader">
9292 <strong>
9393 <span data-i18n="kobldpresets">Kobold Presets</span>
94- <a href="https://docs.sillytavern.app/usage/api-connections/koboldai/" class="notes-link" target="_blank">
95- <span class="fa-solid fa-circle-question note-link-span"></span>
96- </a>
9794 </strong>
9895
9996 <div class="flex-container gap3px">
@@ -3274,8 +3271,14 @@
32743271 </div>
32753272 <div id="AdvancedFormatting" class="drawer-content">
32763273 <div class="flex-container alignItemsBaseline">
32773274 <h3 class="margin0 flex1" dataflex-i18n="Advancedcontainer FormattingalignItemsBaseline">
3275+ <span data-i18n="Advanced Formatting">
32783276 Advanced Formatting
3277+ </span>
3278+
3279+ <a href="https://docs.sillytavern.app/usage/prompts/" class="notes-link" target="_blank">
3280+ <span class="fa-solid fa-circle-question note-link-span"></span>
3281+ </a>
32793282 </h3>
32803283 <div class="flex-container">
32813284 <input id="af_master_import_file" type="file" hidden accept=".json" class="displayNone">
@@ -3295,9 +3298,12 @@
32953298 <h4 class="standoutHeader title_restorable">
32963299 <div>
32973300 <span data-i18n="Context Template">Context Template</span>
3298- <a href="https://docs.sillytavern.app/usage/core-concepts/advancedformatting/#context-template" class="notes-link" target="_blank">
3301+ </div>
3299- <span class="fa-solid fa-circle-question note-link-span"></span>
3302+ <div class="flex-container">
3300- </a>
3303+ <label for="context_derived" class="checkbox_label flex1" title="Derive from Model Metadata, if possible." data-i18n="[title]context_derived">
3304+ <input id="context_derived" type="checkbox" style="display:none;" />
3305+ <small><i class="fa-solid fa-bolt menu_button margin0"></i></small>
3306+ </label>
33013307 </div>
33023308 </h4>
33033309 <div class="flex-container" title="Select your current Context Template" data-i18n="[title]Select your current Context Template">
@@ -3394,11 +3400,12 @@
33943400 <h4 class="standoutHeader title_restorable justifySpaceBetween">
33953401 <div class="flex-container">
33963402 <span data-i18n="Instruct Template">Instruct Template</span>
3397- <a href="https://docs.sillytavern.app/usage/core-concepts/instructmode/" class="notes-link" target="_blank">
3398- <span class="fa-solid fa-circle-question note-link-span"></span>
3399- </a>
34003403 </div>
34013404 <div class="flex-container">
3405+ <label for="instruct_derived" class="checkbox_label flex1" title="Derive from Model Metadata, if possible." data-i18n="[title]instruct_derived">
3406+ <input id="instruct_derived" type="checkbox" style="display:none;" />
3407+ <small><i class="fa-solid fa-bolt menu_button margin0"></i></small>
3408+ </label>
34023409 <label for="instruct_bind_to_context" class="checkbox_label flex1" title="Bind to Context&#10If enabled, Context templates will be automatically selected based on selected Instruct template name or by preference." data-i18n="[title]instruct_bind_to_context">
34033410 <input id="instruct_bind_to_context" type="checkbox" style="display:none;" />
34043411 <small><i class="fa-solid fa-link menu_button margin0"></i></small>
@@ -3579,9 +3586,6 @@
35793586 <h4 class="standoutHeader title_restorable justifySpaceBetween">
35803587 <div class="flex-container">
35813588 <span data-i18n="System Prompt">System Prompt</span>
3582- <a href="https://docs.sillytavern.app/usage/core-concepts/advancedformatting/#system-prompt" class="notes-link" target="_blank">
3583- <span class="fa-solid fa-circle-question note-link-span"></span>
3584- </a>
35853589 </div>
35863590 <div class="flex-container">
35873591 <label id="sysprompt_enabled_label" for="sysprompt_enabled" class="checkbox_label flex1" title="Enable System Prompt" data-i18n="[title]sysprompt_enabled">
@@ -3649,7 +3653,7 @@
36493653 <div name="tokenizerSettingsBlock">
36503654 <div name="tokenizerSelectorBlock">
36513655 <h4 class="standoutHeader"><span data-i18n="Tokenizer">Tokenizer</span>
36523656 <a href="https://docs.sillytavern.app/usage/core-concepts/advancedformattingprompts/#tokenizer/" class="notes-link" target="_blank">
36533657 <span class="fa-solid fa-circle-question note-link-span"></span>
36543658 </a>
36553659 </h4>
@@ -3675,10 +3679,8 @@
36753679 </div>
36763680 <div class="range-block flex-container flexnowrap" name="tokenPaddingBlock">
36773681 <div class="range-block-title justifyLeft">
36783682 <small data-i18n="Token Padding" class="flex-container">Token Padding
3679- <a href="https://docs.sillytavern.app/usage/core-concepts/advancedformatting/#token-padding" class="notes-link" target="_blank">
3683+ Token Padding
3680- <span class="fa-solid fa-circle-question note-link-span"></span>
3681- </a>
36823684 </small>
36833685 </div>
36843686 <input id="token_padding" class="text_pole textarea_compact" type="number" min="-2048" max="2048" />
@@ -3736,7 +3738,7 @@
37363738 </div>
37373739 <h3 class="margin0">
37383740 <span data-i18n="Worlds/Lorebooks">Worlds/Lorebooks</span>
37393741 <a href="https://docs.sillytavern.app/usage/core-concepts/worldinfo/" class="notes-link" target="_blank">
37403742 <span class="fa-solid fa-circle-question note-link-span"></span>
37413743 </a>
37423744 </h3>
@@ -4263,7 +4265,7 @@
42634265 <audio id="audio_message_sound" src="sounds/message.mp3" hidden></audio>
42644266 <span>
42654267 <small data-i18n="Message Sound">Message Sound</small>
42664268 <a href="https://docs.sillytavern.app/usage/core-conceptsuser_settings/uicustomization/#message-sound" class="notes-link" target="_blank">
42674269 <span class="fa-solid fa-circle-question note-link-span"></span>
42684270 </a>
42694271 </span>
@@ -5795,7 +5797,7 @@
57955797 <div class="flex-container justifySpaceBetween">
57965798 <small for="group">
57975799 <span data-i18n="Inclusion Group">Inclusion Group</span>
57985800 <a href="https://docs.sillytavern.app/usage/core-concepts/worldinfo/#inclusion-group" class="notes-link" target="_blank" title="Inclusion Groups ensure only one entry from a group is activated at a time, if multiple are triggered.&#13;Supports multiple comma-separated groups.&#13;&#13;Documentation: World Info - Inclusion Group" data-i18n="[title]Inclusion Groups ensure only one entry from a group is activated at a time, if multiple are triggered.Documentation: World Info - Inclusion Group">
57995801 <span class="fa-solid fa-circle-question note-link-span"></span>
58005802 </a>
58015803 </small>
public/script.js+33 -0
@@ -267,6 +267,7 @@ import { applyBrowserFixes } from './scripts/browser-fixes.js';
267267import { initServerHistory } from './scripts/server-history.js';
268268import { initSettingsSearch } from './scripts/setting-search.js';
269269import { initBulkEdit } from './scripts/bulk-edit.js';
270+import { deriveTemplatesFromChatTemplate } from './scripts/chat-templates.js';
270271
271272//exporting functions and vars for mods
272273export {
@@ -1235,6 +1236,38 @@ async function getStatusTextgen() {
12351236 const supportsTokenization = response.headers.get('x-supports-tokenization') === 'true';
12361237 supportsTokenization ? sessionStorage.setItem(TOKENIZER_SUPPORTED_KEY, 'true') : sessionStorage.removeItem(TOKENIZER_SUPPORTED_KEY);
12371238
1239+ const wantsInstructDerivation = (power_user.instruct.enabled && power_user.instruct.derived);
1240+ const wantsContextDerivation = power_user.context_derived;
1241+ const supportsChatTemplate = [textgen_types.KOBOLDCPP, textgen_types.LLAMACPP].includes(textgen_settings.type);
1242+ if (supportsChatTemplate && (wantsInstructDerivation || wantsContextDerivation)) {
1243+ const response = await fetch('/api/backends/text-completions/props', {
1244+ method: 'POST',
1245+ headers: getRequestHeaders(),
1246+ body: JSON.stringify({
1247+ api_server: endpoint,
1248+ api_type: textgen_settings.type,
1249+ }),
1250+ });
1251+
1252+ if (response.ok) {
1253+ const data = await response.json();
1254+ if (data) {
1255+ const { chat_template, chat_template_hash } = data;
1256+ console.log(`We have chat template ${chat_template.split('\n')[0]}...`);
1257+ const templates = await deriveTemplatesFromChatTemplate(chat_template, chat_template_hash);
1258+ if (templates) {
1259+ const { context, instruct } = templates;
1260+ if (wantsContextDerivation) {
1261+ selectContextPreset(context, { isAuto: true });
1262+ }
1263+ if (wantsInstructDerivation) {
1264+ selectInstructPreset(instruct, { isAuto: true });
1265+ }
1266+ }
1267+ }
1268+ }
1269+ }
1270+
12381271 // We didn't get a 200 status code, but the endpoint has an explanation. Which means it DID connect, but I digress.
12391272 if (online_status === 'no_connection' && data.response) {
12401273 toastr.error(data.response, t`API Error`, { timeOut: 5000, preventDuplicates: true });
public/scripts/chat-templates.js+81 -0
@@ -0,0 +1,81 @@
1+// the hash can be obtained from command line e.g. via: MODEL=path_to_model; python -c "import json, hashlib, sys; print(hashlib.sha256(json.load(open('"$MODEL"/tokenizer_config.json'))['chat_template'].encode()).hexdigest())"
2+// note that chat templates must be trimmed to match the llama.cpp metadata value
3+const hash_derivations = {
4+ // Meta
5+ 'e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65':
6+ // Meta-Llama-3.1-8B-Instruct
7+ // Meta-Llama-3.1-70B-Instruct
8+ 'Llama 3 Instruct'
9+ ,
10+ '5816fce10444e03c2e9ee1ef8a4a1ea61ae7e69e438613f3b17b69d0426223a4':
11+ // Llama-3.2-1B-Instruct
12+ // Llama-3.2-3B-Instruct
13+ 'Llama 3 Instruct'
14+ ,
15+ '73e87b1667d87ab7d7b579107f01151b29ce7f3ccdd1018fdc397e78be76219d':
16+ // Nemotron 70B
17+ 'Llama 3 Instruct'
18+ ,
19+
20+ // Mistral
21+ // Mistral Reference: https://github.com/mistralai/mistral-common
22+ 'e16746b40344d6c5b5265988e0328a0bf7277be86f1c335156eae07e29c82826':
23+ // Mistral-Small-Instruct-2409
24+ // Mistral-Large-Instruct-2407
25+ 'Mistral V2 & V3'
26+ ,
27+ '3c4ad5fa60dd8c7ccdf82fa4225864c903e107728fcaf859fa6052cb80c92ee9':
28+ // Mistral-Large-Instruct-2411
29+ 'Mistral V7' // https://huggingface.co/mistralai/Mistral-Large-Instruct-2411
30+ ,
31+ 'e4676cb56dffea7782fd3e2b577cfaf1e123537e6ef49b3ec7caa6c095c62272':
32+ // Mistral-Nemo-Instruct-2407
33+ 'Mistral V3-Tekken'
34+ ,
35+ '26a59556925c987317ce5291811ba3b7f32ec4c647c400c6cc7e3a9993007ba7':
36+ // Mistral-7B-Instruct-v0.3
37+ 'Mistral V2 & V3'
38+ ,
39+
40+ // Gemma
41+ 'ecd6ae513fe103f0eb62e8ab5bfa8d0fe45c1074fa398b089c93a7e70c15cfd6':
42+ // gemma-2-9b-it
43+ // gemma-2-27b-it
44+ 'Gemma 2'
45+ ,
46+ '87fa45af6cdc3d6a9e4dd34a0a6848eceaa73a35dcfe976bd2946a5822a38bf3':
47+ // gemma-2-2b-it
48+ 'Gemma 2'
49+ ,
50+
51+ // Cohere
52+ '3b54f5c219ae1caa5c0bb2cdc7c001863ca6807cf888e4240e8739fa7eb9e02e':
53+ // command-r-08-2024
54+ 'Command R'
55+ ,
56+};
57+
58+const substr_derivations = {
59+ '<|im_start|>': 'ChatML', // qwen2.5, ...
60+};
61+
62+const parse_derivation = derivation => (typeof derivation === 'string') ? {
63+ 'context': derivation,
64+ 'instruct': derivation,
65+} : derivation;
66+
67+export async function deriveTemplatesFromChatTemplate(chat_template, hash) {
68+ if (hash in hash_derivations) {
69+ return parse_derivation(hash_derivations[hash]);
70+ }
71+
72+ // heuristics
73+ for (const [substr, derivation] of Object.entries(substr_derivations) ) {
74+ if (chat_template.includes(substr)) {
75+ return parse_derivation(derivation);
76+ }
77+ }
78+
79+ console.log(`Unknown chat template hash: ${hash} for [${chat_template}]`);
80+ return null;
81+}
public/scripts/instruct-mode.js+18 -0
@@ -39,6 +39,7 @@ const controls = [
3939 { id: 'instruct_first_input_sequence', property: 'first_input_sequence', isCheckbox: false },
4040 { id: 'instruct_last_input_sequence', property: 'last_input_sequence', isCheckbox: false },
4141 { id: 'instruct_activation_regex', property: 'activation_regex', isCheckbox: false },
42+ { id: 'instruct_derived', property: 'derived', isCheckbox: true },
4243 { id: 'instruct_bind_to_context', property: 'bind_to_context', isCheckbox: true },
4344 { id: 'instruct_skip_examples', property: 'skip_examples', isCheckbox: true },
4445 { id: 'instruct_names_behavior', property: 'names_behavior', isCheckbox: false },
@@ -100,6 +101,7 @@ export async function loadInstructMode(data) {
100101
101102 $('#instruct_enabled').parent().find('i').toggleClass('toggleEnabled', !!power_user.instruct.enabled);
102103 $('#instructSettingsBlock, #InstructSequencesColumn').toggleClass('disabled', !power_user.instruct.enabled);
104+ $('#instruct_derived').parent().find('i').toggleClass('toggleEnabled', !!power_user.instruct.derived);
103105 $('#instruct_bind_to_context').parent().find('i').toggleClass('toggleEnabled', !!power_user.instruct.bind_to_context);
104106
105107 controls.forEach(control => {
@@ -146,6 +148,12 @@ export async function loadInstructMode(data) {
146148 * @param {boolean} [options.isAuto=false] Is auto-select.
147149 */
148150export function selectContextPreset(preset, { quiet = false, isAuto = false } = {}) {
151+ const presetExists = context_presets.some(x => x.name === preset);
152+ if (!presetExists) {
153+ console.warn(`Context template "${preset}" not found`);
154+ return;
155+ }
156+
149157 // If context template is not already selected, select it
150158 if (preset !== power_user.context.preset) {
151159 $('#context_presets').val(preset).trigger('change');
@@ -163,6 +171,12 @@ export function selectContextPreset(preset, { quiet = false, isAuto = false } =
163171 * @param {boolean} [options.isAuto=false] Is auto-select.
164172 */
165173export function selectInstructPreset(preset, { quiet = false, isAuto = false } = {}) {
174+ const presetExists = instruct_presets.some(x => x.name === preset);
175+ if (!presetExists) {
176+ console.warn(`Instruct template "${preset}" not found`);
177+ return;
178+ }
179+
166180 // If instruct preset is not already selected, select it
167181 if (preset !== power_user.instruct.preset) {
168182 $('#instruct_presets').val(preset).trigger('change');
@@ -715,6 +729,10 @@ jQuery(() => {
715729 }
716730 });
717731
732+ $('#instruct_derived').on('change', function () {
733+ $('#instruct_derived').parent().find('i').toggleClass('toggleEnabled', !!power_user.instruct.derived);
734+ });
735+
718736 $('#instruct_bind_to_context').on('change', function () {
719737 $('#instruct_bind_to_context').parent().find('i').toggleClass('toggleEnabled', !!power_user.instruct.bind_to_context);
720738 });
public/scripts/power-user.js+14 -0
@@ -226,6 +226,7 @@ let power_user = {
226226 macro: true,
227227 names_behavior: names_behavior_types.FORCE,
228228 activation_regex: '',
229+ derived: false,
229230 bind_to_context: false,
230231 user_alignment_message: '',
231232 system_same_as_user: false,
@@ -243,6 +244,8 @@ let power_user = {
243244 names_as_stop_strings: true,
244245 },
245246
247+ context_derived: false,
248+
246249 sysprompt: {
247250 enabled: true,
248251 name: 'Neutral - Chat',
@@ -1472,6 +1475,7 @@ async function loadPowerUserSettings(settings, data) {
14721475 $('#encode_tags').prop('checked', power_user.encode_tags);
14731476 $('#example_messages_behavior').val(getExampleMessagesBehavior());
14741477 $(`#example_messages_behavior option[value="${getExampleMessagesBehavior()}"]`).prop('selected', true);
1478+ $('#context_derived').parent().find('i').toggleClass('toggleEnabled', !!power_user.context_derived);
14751479
14761480 $('#console_log_prompts').prop('checked', power_user.console_log_prompts);
14771481 $('#request_token_probabilities').prop('checked', power_user.request_token_probabilities);
@@ -3057,6 +3061,16 @@ $(document).ready(() => {
30573061 saveSettingsDebounced();
30583062 });
30593063
3064+ $('#context_derived').on('input', function () {
3065+ const value = !!$(this).prop('checked');
3066+ power_user.context_derived = value;
3067+ saveSettingsDebounced();
3068+ });
3069+
3070+ $('#context_derived').on('change', function () {
3071+ $('#context_derived').parent().find('i').toggleClass('toggleEnabled', !!power_user.context_derived);
3072+ });
3073+
30603074 $('#always-force-name2-checkbox').change(function () {
30613075 power_user.always_force_name2 = !!$(this).prop('checked');
30623076 saveSettingsDebounced();
public/scripts/preset-manager.js+1 -0
@@ -584,6 +584,7 @@ class PresetManager {
584584 'openrouter_providers',
585585 'openrouter_allow_fallbacks',
586586 'tabby_model',
587+ 'derived',
587588 ];
588589 const settings = Object.assign({}, getSettingsByApiId(this.apiId));
589590
src/endpoints/backends/text-completions.js+35 -0
@@ -16,6 +16,7 @@ import {
1616} from '../../constants.js';
1717import { forwardFetchResponse, trimV1, getConfigValue } from '../../util.js';
1818import { setAdditionalHeaders } from '../../additional-headers.js';
19+import { createHash } from 'node:crypto';
1920
2021export const router = express.Router();
2122
@@ -227,6 +228,40 @@ router.post('/status', jsonParser, async function (request, response) {
227228 }
228229});
229230
231+router.post('/props', jsonParser, async function (request, response) {
232+ if (!request.body.api_server) return response.sendStatus(400);
233+
234+ try {
235+ const baseUrl = trimV1(request.body.api_server);
236+ const args = {
237+ headers: {},
238+ };
239+
240+ setAdditionalHeaders(request, args, baseUrl);
241+
242+ const apiType = request.body.api_type;
243+ const propsUrl = baseUrl + '/props';
244+ const propsReply = await fetch(propsUrl, args);
245+
246+ if (!propsReply.ok) {
247+ return response.status(400);
248+ }
249+
250+ /** @type {any} */
251+ const props = await propsReply.json();
252+ // TEMPORARY: llama.cpp's /props endpoint has a bug which replaces the last newline with a \0
253+ if (apiType === TEXTGEN_TYPES.LLAMACPP && props['chat_template'].endsWith('\u0000')) {
254+ props['chat_template'] = props['chat_template'].slice(0, -1) + '\n';
255+ }
256+ props['chat_template_hash'] = createHash('sha256').update(props['chat_template']).digest('hex');
257+ console.log(`Model properties: ${JSON.stringify(props)}`);
258+ return response.send(props);
259+ } catch (error) {
260+ console.error(error);
261+ return response.status(500);
262+ }
263+});
264+
230265router.post('/generate', jsonParser, async function (request, response) {
231266 if (!request.body) return response.sendStatus(400);
232267