Merge pull request #2932 from SillyTavern/tool-calling Tool calling

4cc8b8d0d97652f09eb406330787a1fe52857c6c

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: 912525fdd9c546b57a1b818fa56744d999b2fa3e 4f46a8ff10ae68a65742415304036267ea129e15
26 files changed, +1501 -495Showing whitespace changes
.eslintrc.js+11 -8
@@ -59,15 +59,18 @@ module.exports = {
5959 },
6060 },
6161 ],
62- // There are various vendored libraries that shouldn't be linted
6362 ignorePatterns: [
6463 'public/lib/**/node_modules/**',
6564 '*.min.js*/dist/**',
6665 'src/ai_horde/**/.git/**',
6766 'pluginspublic/**lib/**',
6867 'databackups/**/*',
6968 'backupsdata/**/*',
7069 'node_modulescache/**/*',
70+ 'src/tokenizers/**',
71+ 'docker/**',
72+ 'plugins/**',
73+ '**/*.min.js',
7174 ],
7275 rules: {
7376 'no-unused-vars': ['error', { args: 'none' }],
default/config.yaml+2 -0
@@ -121,6 +121,8 @@ extras:
121121 speechToTextModel: Xenova/whisper-small
122122 textToSpeechModel: Xenova/speecht5_tts
123123# -- OPENAI CONFIGURATION --
124+# A placeholder message to use in strict prompt post-processing mode when the prompt doesn't start with a user message
125+promptPlaceholder: "[Start a new chat]"
124126openai:
125127 # Will send a random user ID to OpenAI completion API
126128 randomizeUserId: false
jsconfig.json+9 -10
@@ -11,15 +11,14 @@
1111 "resolveJsonModule": true
1212 },
1313 "exclude": [
1414 "**/node_modules/**",
1515 "**/node_modulesdist/**",
1616 "public**/lib.git/**",
1717 "backupspublic/lib/**",
1818 "databackups/**",
1919 "**/distdata/**",
2020 "distcache/**",
2121 "cachesrc/tokenizers/**",
2222 "src/tokenizersdocker/**",
23- "docker/*",
2423 ]
2524}
package-lock.json+34 -12
@@ -59,9 +59,10 @@
5959 "sillytavern": "server.js"
6060 },
6161 "devDependencies": {
62+ "@types/dompurify": "^3.0.5",
6263 "@types/jquery": "^3.5.29",
6364 "eslint@types/toastr": "^82.571.043",
6465 "jqueryeslint": "^38.657.40"
6566 },
6667 "engines": {
6768 "node": ">= 18"
@@ -960,6 +961,16 @@
960961 "@types/responselike": "^1.0.0"
961962 }
962963 },
964+ "node_modules/@types/dompurify": {
965+ "version": "3.0.5",
966+ "resolved": "https://registry.npmjs.org/@types/dompurify/-/dompurify-3.0.5.tgz",
967+ "integrity": "sha512-1Wg0g3BtQF7sSb27fJQAKck1HECM6zV1EB66j8JH9i3LCjYabJa0FSdiSgsD5K/RbrsR0SiraKacLB+T8ZVYAg==",
968+ "dev": true,
969+ "license": "MIT",
970+ "dependencies": {
971+ "@types/trusted-types": "*"
972+ }
973+ },
963974 "node_modules/@types/http-cache-semantics": {
964975 "version": "4.0.2",
965976 "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.2.tgz",
@@ -967,10 +978,11 @@
967978 "license": "MIT"
968979 },
969980 "node_modules/@types/jquery": {
970981 "version": "3.5.2931",
971982 "resolved": "https://registry.npmjs.org/@types/jquery/-/jquery-3.5.2931.tgz",
972983 "integrity": "sha512-oXQQC9X9MOPRrMhPHHOsXqeQDnWeCDT3PelUIgrf/Oy8FAbzSZtFHRjc7IpbfFVmpLtJiB+UOoywpRsuO5JxjybyegcPJ/YZfMwr+FVuQbm7IaWC4y3FVYfVDxRGqmUCFjjPII0HWaP0vTPJGp6m4o13AXySCcMbWfrWtBFAKw==",
973984 "dev": true,
985+ "license": "MIT",
974986 "dependencies": {
975987 "@types/sizzle": "*"
976988 }
@@ -1021,6 +1033,23 @@
10211033 "integrity": "sha512-0vWLNK2D5MT9dg0iOo8GlKguPAU02QjmZitPEsXRuJXU/OGIOt9vT9Fc26wtYuavLxtO45v9PGleoL9Z0k1LHg==",
10221034 "dev": true
10231035 },
1036+ "node_modules/@types/toastr": {
1037+ "version": "2.1.43",
1038+ "resolved": "https://registry.npmjs.org/@types/toastr/-/toastr-2.1.43.tgz",
1039+ "integrity": "sha512-sLC2fr2OXeE1iyhUixpQ64wQ2tA26awmLidn4tXTLBz4yP/VhtYUKHpmiIyDtztKkHjucdiTLH8F5uRRyhNi2Q==",
1040+ "dev": true,
1041+ "license": "MIT",
1042+ "dependencies": {
1043+ "@types/jquery": "*"
1044+ }
1045+ },
1046+ "node_modules/@types/trusted-types": {
1047+ "version": "2.0.7",
1048+ "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz",
1049+ "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==",
1050+ "dev": true,
1051+ "license": "MIT"
1052+ },
10241053 "node_modules/@ungap/structured-clone": {
10251054 "version": "1.2.0",
10261055 "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz",
@@ -3881,13 +3910,6 @@
38813910 "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==",
38823911 "license": "BSD-3-Clause"
38833912 },
3884- "node_modules/jquery": {
3885- "version": "3.7.0",
3886- "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.7.0.tgz",
3887- "integrity": "sha512-umpJ0/k8X0MvD1ds0P9SfowREz2LenHsQaxSohMZ5OMNEU2r0tf8pdeEFTHMFxWVxKNyU9rTtK3CWzUCTKJUeQ==",
3888- "dev": true,
3889- "license": "MIT"
3890- },
38913913 "node_modules/js-yaml": {
38923914 "version": "4.1.0",
38933915 "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
package.json+3 -2
@@ -85,8 +85,9 @@
8585 },
8686 "main": "server.js",
8787 "devDependencies": {
88+ "@types/dompurify": "^3.0.5",
8889 "@types/jquery": "^3.5.29",
8990 "eslint@types/toastr": "^82.571.043",
9091 "jqueryeslint": "^38.657.40"
9192 }
9293}
public/css/st-tailwind.css+4 -0
@@ -612,3 +612,7 @@ ul.li-padding-bot5 li {
612612ul.li-padding-bot10 li {
613613 padding-bottom: 10px;
614614}
615+
616+.wordBreakAll {
617+ word-break: break-all;
618+}
public/css/toggle-dependent.css+11 -0
@@ -463,3 +463,14 @@ body.expandMessageActions .mes .mes_buttons .extraMesButtonsHint {
463463label[for="trim_spaces"]:has(input:checked) i.warning {
464464 display: none;
465465}
466+
467+#claude_function_prefill_warning {
468+ display: none;
469+}
470+
471+#openai_settings:has(#openai_function_calling:checked):has(#claude_assistant_prefill:not(:placeholder-shown), #claude_assistant_impersonation:not(:placeholder-shown)) #claude_function_prefill_warning {
472+ display: flex;
473+ align-items: center;
474+ gap: 5px;
475+ margin: 10px 0;
476+}
public/global.d.ts+0 -42
@@ -1,5 +1,4 @@
11// Global namespace modules
2-declare var DOMPurify;
32declare var droll;
43declare var Handlebars;
54declare var hljs;
@@ -1365,44 +1364,3 @@ declare namespace moment {
13651364declare global {
13661365 const moment: typeof moment;
13671366}
1368-
1369-/**
1370- * Callback data for the `LLM_FUNCTION_TOOL_REGISTER` event type that is triggered when a function tool can be registered.
1371- */
1372-interface FunctionToolRegister {
1373- /**
1374- * The type of generation that is being used
1375- */
1376- type?: string;
1377- /**
1378- * Generation data, including messages and sampling parameters
1379- */
1380- data: Record<string, object>;
1381- /**
1382- * Callback to register an LLM function tool.
1383- */
1384- registerFunctionTool: typeof registerFunctionTool;
1385-}
1386-
1387-/**
1388- * Callback data for the `LLM_FUNCTION_TOOL_REGISTER` event type that is triggered when a function tool is registered.
1389- * @param name Name of the function tool to register
1390- * @param description Description of the function tool
1391- * @param params JSON schema for the parameters of the function tool
1392- * @param required Whether the function tool should be forced to be used
1393- */
1394-declare function registerFunctionTool(name: string, description: string, params: object, required: boolean): Promise<void>;
1395-
1396-/**
1397- * Callback data for the `LLM_FUNCTION_TOOL_CALL` event type that is triggered when a function tool is called.
1398- */
1399-interface FunctionToolCall {
1400- /**
1401- * Name of the function tool to call
1402- */
1403- name: string;
1404- /**
1405- * JSON object with the parameters to pass to the function tool
1406- */
1407- arguments: string;
1408-}
public/index.html+6 -1
@@ -1873,6 +1873,10 @@
18731873 </div>
18741874 <textarea id="claude_assistant_impersonation" class="text_pole textarea_compact autoSetHeight" name="assistant_impersonation" rows="2" data-i18n="[placeholder]Start Claude's answer with..." placeholder="Start Claude's answer with..."></textarea>
18751875 </div>
1876+ <div id="claude_function_prefill_warning">
1877+ <i class="fa-solid fa-circle-info"></i>
1878+ <span>Prefills won't work when function calling is enabled and any tools are registered.</span>
1879+ </div>
18761880 <label for="claude_use_sysprompt" class="checkbox_label widthFreeExpand">
18771881 <input id="claude_use_sysprompt" type="checkbox" />
18781882 <span data-i18n="Use system prompt (Claude 2.1+ only)">
@@ -3098,7 +3102,8 @@
30983102 <h4 data-i18n="Prompt Post-Processing">Prompt Post-Processing</h4>
30993103 <select id="custom_prompt_post_processing" class="text_pole" title="Applies additional processing to the prompt before sending it to the API." data-i18n="[title]Applies additional processing to the prompt before sending it to the API.">
31003104 <option data-i18n="prompt_post_processing_none" value="">None</option>
31013105 <option value="claudemerge">ClaudeMerge consecutive roles</option>
3106+ <option value="strict">Strict (user first, alternating roles)</option>
31023107 </select>
31033108 </form>
31043109 <div id="01ai_form" data-source="01ai">
public/jsconfig.json+5 -5
@@ -8,16 +8,16 @@
88 "allowSyntheticDefaultImports": true
99 },
1010 "exclude": [
1111 "**/node_modules/**",
12+ "**/dist/**",
13+ "**/.git/**",
14+ "lib/**",
15+ "**/*.min.js"
1216 ],
1317 "typeAcquisition": {
1418 "include": [
15- "jquery",
1619 "@popperjs/core",
17- "toastr",
1820 "showdown",
19- "dompurify",
20- "moment",
2121 "seedrandom",
2222 "showdown-katex",
2323 "droll",
public/script.js+70 -7
@@ -246,6 +246,7 @@ import { initInputMarkdown } from './scripts/input-md-formatting.js';
246246import { AbortReason } from './scripts/util/AbortReason.js';
247247import { initSystemPrompts } from './scripts/sysprompt.js';
248248import { registerExtensionSlashCommands as initExtensionSlashCommands } from './scripts/extensions-slashcommands.js';
249+import { ToolManager } from './scripts/tool-calling.js';
249250
250251//exporting functions and vars for mods
251252export {
@@ -470,11 +471,11 @@ export const event_types = {
470471 FILE_ATTACHMENT_DELETED: 'file_attachment_deleted',
471472 WORLDINFO_FORCE_ACTIVATE: 'worldinfo_force_activate',
472473 OPEN_CHARACTER_LIBRARY: 'open_character_library',
473- LLM_FUNCTION_TOOL_REGISTER: 'llm_function_tool_register',
474- LLM_FUNCTION_TOOL_CALL: 'llm_function_tool_call',
475474 ONLINE_STATUS_CHANGED: 'online_status_changed',
476475 IMAGE_SWIPED: 'image_swiped',
477476 CONNECTION_PROFILE_LOADED: 'connection_profile_loaded',
477+ TOOL_CALLS_PERFORMED: 'tool_calls_performed',
478+ TOOL_CALLS_RENDERED: 'tool_calls_rendered',
478479};
479480
480481export const eventSource = new EventEmitter();
@@ -947,6 +948,7 @@ async function firstLoadInit() {
947948 initSystemPrompts();
948949 initExtensions();
949950 initExtensionSlashCommands();
951+ ToolManager.initToolSlashCommands();
950952 await initPresetManager();
951953 await getSystemMessages();
952954 sendSystemMessage(system_message_types.WELCOME);
@@ -2027,7 +2029,7 @@ export function messageFormatting(mes, ch_name, isSystem, isUser, messageId, san
20272029 // Return the original match if no quotes are found
20282030 return match;
20292031 }
20302032 },
20312033 );
20322034
20332035 // Restore double quotes in tags
@@ -2372,6 +2374,10 @@ export function addOneMessage(mes, { type = 'normal', insertAfter = null, scroll
23722374 newMessage.addClass('smallSysMes');
23732375 }
23742376
2377+ if (Array.isArray(mes?.extra?.tool_invocations)) {
2378+ newMessage.addClass('toolCall');
2379+ }
2380+
23752381 //shows or hides the Prompt display button
23762382 let mesIdToFind = type === 'swipe' ? params.mesId - 1 : params.mesId; //Number(newMessage.attr('mesId'));
23772383
@@ -2965,6 +2971,7 @@ class StreamingProcessor {
29652971 this.swipes = [];
29662972 /** @type {import('./scripts/logprobs.js').TokenLogprobs[]} */
29672973 this.messageLogprobs = [];
2974+ this.toolCalls = [];
29682975 }
29692976
29702977 #checkDomElements(messageId) {
@@ -2976,6 +2983,13 @@ class StreamingProcessor {
29762983 }
29772984 }
29782985
2986+ #updateMessageBlockVisibility() {
2987+ if (this.messageDom instanceof HTMLElement && Array.isArray(this.toolCalls) && this.toolCalls.length > 0) {
2988+ const shouldHide = ['', '...'].includes(this.result);
2989+ this.messageDom.classList.toggle('displayNone', shouldHide);
2990+ }
2991+ }
2992+
29792993 showMessageButtons(messageId) {
29802994 if (messageId == -1) {
29812995 return;
@@ -3041,6 +3055,7 @@ class StreamingProcessor {
30413055 }
30423056 else {
30433057 this.#checkDomElements(messageId);
3058+ this.#updateMessageBlockVisibility();
30443059 const currentTime = new Date();
30453060 // Don't waste time calculating token count for streaming
30463061 const currentTokenCount = isFinal && power_user.message_token_count_enabled ? getTokenCount(processedText, 0) : 0;
@@ -3183,7 +3198,7 @@ class StreamingProcessor {
31833198 }
31843199
31853200 /**
31863201 * @returns {Generator<{ text: string, swipes: string[], logprobs: import('./scripts/logprobs.js').TokenLogprobs, toolCalls: any[] }, void, void>}
31873202 */
31883203 *nullStreamingGeneration() {
31893204 throw new Error('Generation function for streaming is not hooked up');
@@ -3205,12 +3220,13 @@ class StreamingProcessor {
32053220 try {
32063221 const sw = new Stopwatch(1000 / power_user.streaming_fps);
32073222 const timestamps = [];
32083223 for await (const { text, swipes, logprobs, toolCalls } of this.generator()) {
32093224 timestamps.push(Date.now());
32103225 if (this.isStopped) {
32113226 return;
32123227 }
32133228
3229+ this.toolCalls = toolCalls;
32143230 this.result = text;
32153231 this.swipes = Array.from(swipes ?? []);
32163232 if (logprobs) {
@@ -3614,7 +3630,9 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
36143630 }
36153631
36163632 // Collect messages with usable content
36173633 letconst coreChatcanUseTools = chatToolManager.filterisToolCallingSupported(x => !x.is_system);
3634+ const canPerformToolCalls = !dryRun && ToolManager.canPerformToolCalls(type);
3635+ let coreChat = chat.filter(x => !x.is_system || (canUseTools && Array.isArray(x.extra?.tool_invocations)));
36183636 if (type === 'swipe') {
36193637 coreChat.pop();
36203638 }
@@ -4449,7 +4467,30 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
44494467 getMessage = continue_mag + getMessage;
44504468 }
44514469
44524470 ifconst (isStreamFinished = streamingProcessor && !streamingProcessor.isStopped && streamingProcessor.isFinished) {;
4471+ const isStreamWithToolCalls = streamingProcessor && Array.isArray(streamingProcessor.toolCalls) && streamingProcessor.toolCalls.length;
4472+ if (canPerformToolCalls && isStreamFinished && isStreamWithToolCalls) {
4473+ const lastMessage = chat[chat.length - 1];
4474+ const hasToolCalls = ToolManager.hasToolCalls(streamingProcessor.toolCalls);
4475+ const shouldDeleteMessage = ['', '...'].includes(lastMessage?.mes) && ['', '...'].includes(streamingProcessor?.result);
4476+ hasToolCalls && shouldDeleteMessage && await deleteLastMessage();
4477+ const invocationResult = await ToolManager.invokeFunctionTools(streamingProcessor.toolCalls);
4478+ if (hasToolCalls) {
4479+ if (!invocationResult.invocations.length && shouldDeleteMessage) {
4480+ ToolManager.showToolCallError(invocationResult.errors);
4481+ unblockGeneration(type);
4482+ generatedPromptCache = '';
4483+ streamingProcessor = null;
4484+ return;
4485+ }
4486+
4487+ streamingProcessor = null;
4488+ await ToolManager.saveFunctionToolInvocations(invocationResult.invocations);
4489+ return Generate('normal', { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName }, dryRun);
4490+ }
4491+ }
4492+
4493+ if (isStreamFinished) {
44534494 await streamingProcessor.onFinishStreaming(streamingProcessor.messageId, getMessage);
44544495 streamingProcessor = null;
44554496 triggerAutoContinue(messageChunk, isImpersonate);
@@ -4523,6 +4564,24 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
45234564 parseAndSaveLogprobs(data, continue_mag);
45244565 }
45254566
4567+ if (canPerformToolCalls) {
4568+ const hasToolCalls = ToolManager.hasToolCalls(data);
4569+ const shouldDeleteMessage = ['', '...'].includes(getMessage);
4570+ hasToolCalls && shouldDeleteMessage && await deleteLastMessage();
4571+ const invocationResult = await ToolManager.invokeFunctionTools(data);
4572+ if (hasToolCalls) {
4573+ if (!invocationResult.invocations.length && shouldDeleteMessage) {
4574+ ToolManager.showToolCallError(invocationResult.errors);
4575+ unblockGeneration(type);
4576+ generatedPromptCache = '';
4577+ return;
4578+ }
4579+
4580+ await ToolManager.saveFunctionToolInvocations(invocationResult.invocations);
4581+ return Generate('normal', { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName }, dryRun);
4582+ }
4583+ }
4584+
45264585 if (type !== 'quiet') {
45274586 playMessageSound();
45284587 }
@@ -8174,6 +8233,10 @@ window['SillyTavern'].getContext = function () {
81748233 registerHelper: () => { },
81758234 registerMacro: MacrosParser.registerMacro.bind(MacrosParser),
81768235 unregisterMacro: MacrosParser.unregisterMacro.bind(MacrosParser),
8236+ registerFunctionTool: ToolManager.registerFunctionTool.bind(ToolManager),
8237+ unregisterFunctionTool: ToolManager.unregisterFunctionTool.bind(ToolManager),
8238+ isToolCallingSupported: ToolManager.isToolCallingSupported.bind(ToolManager),
8239+ canPerformToolCalls: ToolManager.canPerformToolCalls.bind(ToolManager),
81778240 registerDebugFunction: registerDebugFunction,
81788241 /** @deprecated Use renderExtensionTemplateAsync instead. */
81798242 renderExtensionTemplate: renderExtensionTemplate,
public/scripts/extensions/expressions/index.js+1 -51
@@ -9,7 +9,6 @@ import { debounce_timeout } from '../../constants.js';
99import { SlashCommandParser } from '../../slash-commands/SlashCommandParser.js';
1010import { SlashCommand } from '../../slash-commands/SlashCommand.js';
1111import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from '../../slash-commands/SlashCommandArgument.js';
12-import { isFunctionCallingSupported } from '../../openai.js';
1312import { SlashCommandEnumValue, enumTypes } from '../../slash-commands/SlashCommandEnumValue.js';
1413import { commonEnumProviders } from '../../slash-commands/SlashCommandCommonEnumsProvider.js';
1514import { slashCommandReturnHelper } from '../../slash-commands/SlashCommandReturnHelper.js';
@@ -21,7 +20,6 @@ const UPDATE_INTERVAL = 2000;
2120const STREAMING_UPDATE_INTERVAL = 10000;
2221const TALKINGCHECK_UPDATE_INTERVAL = 500;
2322const DEFAULT_FALLBACK_EXPRESSION = 'joy';
24-const FUNCTION_NAME = 'set_emotion';
2523const DEFAULT_LLM_PROMPT = 'Ignore previous instructions. Classify the emotion of the last message. Output just one word, e.g. "joy" or "anger". Choose only one of the following labels: {{labels}}';
2624const DEFAULT_EXPRESSIONS = [
2725 'talkinghead',
@@ -1017,10 +1015,6 @@ async function getLlmPrompt(labels) {
10171015 return '';
10181016 }
10191017
1020- if (isFunctionCallingSupported()) {
1021- return '';
1022- }
1023-
10241018 const labelsString = labels.map(x => `"${x}"`).join(', ');
10251019 const prompt = substituteParamsExtended(String(extension_settings.expressions.llmPrompt), { labels: labelsString });
10261020 return prompt;
@@ -1056,41 +1050,6 @@ function parseLlmResponse(emotionResponse, labels) {
10561050 throw new Error('Could not parse emotion response ' + emotionResponse);
10571051}
10581052
1059-/**
1060- * Registers the function tool for the LLM API.
1061- * @param {FunctionToolRegister} args Function tool register arguments.
1062- */
1063-function onFunctionToolRegister(args) {
1064- if (inApiCall && extension_settings.expressions.api === EXPRESSION_API.llm && isFunctionCallingSupported()) {
1065- // Only trigger on quiet mode
1066- if (args.type !== 'quiet') {
1067- return;
1068- }
1069-
1070- const emotions = DEFAULT_EXPRESSIONS.filter((e) => e != 'talkinghead');
1071- const jsonSchema = {
1072- $schema: 'http://json-schema.org/draft-04/schema#',
1073- type: 'object',
1074- properties: {
1075- emotion: {
1076- type: 'string',
1077- enum: emotions,
1078- description: `One of the following: ${JSON.stringify(emotions)}`,
1079- },
1080- },
1081- required: [
1082- 'emotion',
1083- ],
1084- };
1085- args.registerFunctionTool(
1086- FUNCTION_NAME,
1087- substituteParams('Sets the label that best describes the current emotional state of {{char}}. Only select one of the enumerated values.'),
1088- jsonSchema,
1089- true,
1090- );
1091- }
1092-}
1093-
10941053function onTextGenSettingsReady(args) {
10951054 // Only call if inside an API call
10961055 if (inApiCall && extension_settings.expressions.api === EXPRESSION_API.llm && isJsonSchemaSupported()) {
@@ -1164,18 +1123,9 @@ export async function getExpressionLabel(text, expressionsApi = extension_settin
11641123
11651124 const expressionsList = await getExpressionsList();
11661125 const prompt = substituteParamsExtended(customPrompt, { labels: expressionsList }) || await getLlmPrompt(expressionsList);
1167- let functionResult = null;
11681126 eventSource.once(event_types.TEXT_COMPLETION_SETTINGS_READY, onTextGenSettingsReady);
1169- eventSource.once(event_types.LLM_FUNCTION_TOOL_REGISTER, onFunctionToolRegister);
1170- eventSource.once(event_types.LLM_FUNCTION_TOOL_CALL, (/** @type {FunctionToolCall} */ args) => {
1171- if (args.name !== FUNCTION_NAME) {
1172- return;
1173- }
1174-
1175- functionResult = args?.arguments;
1176- });
11771127 const emotionResponse = await generateRaw(text, main_api, false, false, prompt);
11781128 return parseLlmResponse(functionResult || emotionResponse, expressionsList);
11791129 }
11801130 // Extras
11811131 default: {
public/scripts/extensions/stable-diffusion/index.js+67 -4
@@ -32,6 +32,7 @@ import { debounce_timeout } from '../../constants.js';
3232import { SlashCommandEnumValue } from '../../slash-commands/SlashCommandEnumValue.js';
3333import { POPUP_RESULT, POPUP_TYPE, Popup, callGenericPopup } from '../../popup.js';
3434import { commonEnumProviders } from '../../slash-commands/SlashCommandCommonEnumsProvider.js';
35+import { ToolManager } from '../../tool-calling.js';
3536export { MODULE_NAME };
3637
3738const MODULE_NAME = 'sd';
@@ -62,9 +63,11 @@ const initiators = {
6263 interactive: 'interactive',
6364 wand: 'wand',
6465 swipe: 'swipe',
66+ tool: 'tool',
6567};
6668
6769const generationMode = {
70+ TOOL: -2,
6871 MESSAGE: -1,
6972 CHARACTER: 0,
7073 USER: 1,
@@ -87,6 +90,7 @@ const multimodalMap = {
8790};
8891
8992const modeLabels = {
93+ [generationMode.TOOL]: 'Function Tool Prompt Description',
9094 [generationMode.MESSAGE]: 'Chat Message Template',
9195 [generationMode.CHARACTER]: 'Character ("Yourself")',
9296 [generationMode.FACE]: 'Portrait ("Your Face")',
@@ -124,8 +128,12 @@ const messageTrigger = {
124128};
125129
126130const promptTemplates = {
127131 // Not really a prompt template, rather an outcome message template and function tool prompt
128132 [generationMode.MESSAGE]: '[{{char}} sends a picture that contains: {{prompt}}].',
133+ [generationMode.TOOL]: [
134+ 'The text prompt used to generate the image.',
135+ 'Must represent an exhaustive description of the desired image that will allow an artist or a photographer to perfectly recreate it.',
136+ ].join(' '),
129137 [generationMode.CHARACTER]: 'In the next response I want you to provide only a detailed comma-delimited list of keywords and phrases which describe {{char}}. The list must include all of the following items in this order: name, species and race, gender, age, clothing, occupation, physical features and appearances. Do not include descriptions of non-visual qualities such as personality, movements, scents, mental traits, or anything which could not be seen in a still photograph. Do not write in full sentences. Prefix your description with the phrase \'full body portrait,\'',
130138 //face-specific prompt
131139 [generationMode.FACE]: 'In the next response I want you to provide only a detailed comma-delimited list of keywords and phrases which describe {{char}}. The list must include all of the following items in this order: name, species and race, gender, age, facial features and expressions, occupation, hair and hair accessories (if any), what they are wearing on their upper body (if anything). Do not describe anything below their neck. Do not include descriptions of non-visual qualities such as personality, movements, scents, mental traits, or anything which could not be seen in a still photograph. Do not write in full sentences. Prefix your description with the phrase \'close up facial portrait,\'',
@@ -226,6 +234,7 @@ const defaultSettings = {
226234 multimodal_captioning: false,
227235 snap: false,
228236 free_extend: false,
237+ function_tool: false,
229238
230239 prompts: promptTemplates,
231240
@@ -291,6 +300,10 @@ const defaultSettings = {
291300const writePromptFieldsDebounced = debounce(writePromptFields, debounce_timeout.relaxed);
292301
293302function processTriggers(chat, _, abort) {
303+ if (extension_settings.sd.function_tool && ToolManager.isToolCallingSupported()) {
304+ return;
305+ }
306+
294307 if (!extension_settings.sd.interactive_mode) {
295308 return;
296309 }
@@ -447,6 +460,7 @@ async function loadSettings() {
447460 $('#sd_interactive_visible').prop('checked', extension_settings.sd.interactive_visible);
448461 $('#sd_stability_style_preset').val(extension_settings.sd.stability_style_preset);
449462 $('#sd_huggingface_model_id').val(extension_settings.sd.huggingface_model_id);
463+ $('#sd_function_tool').prop('checked', extension_settings.sd.function_tool);
450464
451465 for (const style of extension_settings.sd.styles) {
452466 const option = document.createElement('option');
@@ -461,6 +475,7 @@ async function loadSettings() {
461475
462476 toggleSourceControls();
463477 addPromptTemplates();
478+ registerFunctionTool();
464479
465480 await loadSettingOptions();
466481}
@@ -524,6 +539,9 @@ function addPromptTemplates() {
524539 .on('click', () => {
525540 textarea.val(promptTemplates[name]);
526541 extension_settings.sd.prompts[name] = promptTemplates[name];
542+ if (String(name) === String(generationMode.TOOL)) {
543+ registerFunctionTool();
544+ }
527545 saveSettingsDebounced();
528546 });
529547 const container = $('<div></div>')
@@ -910,6 +928,12 @@ async function onSourceChange() {
910928 await loadSettingOptions();
911929}
912930
931+function onFunctionToolInput() {
932+ extension_settings.sd.function_tool = !!$(this).prop('checked');
933+ saveSettingsDebounced();
934+ registerFunctionTool();
935+}
936+
913937async function onOpenAiStyleSelect() {
914938 extension_settings.sd.openai_style = String($('#sd_openai_style').find(':selected').val());
915939 saveSettingsDebounced();
@@ -2290,9 +2314,9 @@ async function generatePicture(initiator, args, trigger, message, callback) {
22902314 eventSource.emit(event_types.FORCE_SET_BACKGROUND, { url: imgUrl, path: imagePath });
22912315
22922316 if (typeof callbackOriginal === 'function') {
22932317 await callbackOriginal(prompt, imagePath, generationType, negativePromptPrefix, initiator);
22942318 } else {
22952319 await sendMessage(prompt, imagePath, generationType, negativePromptPrefix, initiator);
22962320 }
22972321 };
22982322 }
@@ -2621,7 +2645,9 @@ async function sendGenerationRequest(generationType, prompt, additionalNegativeP
26212645
26222646 const filename = `${characterName}_${humanizedDateTime()}`;
26232647 const base64Image = await saveBase64AsFile(result.data, characterName, filename, result.format);
2624- callback ? callback(prompt, base64Image, generationType, additionalNegativePrefix, initiator) : sendMessage(prompt, base64Image, generationType, additionalNegativePrefix, initiator);
2648+ callback
2649+ ? await callback(prompt, base64Image, generationType, additionalNegativePrefix, initiator)
2650+ : await sendMessage(prompt, base64Image, generationType, additionalNegativePrefix, initiator);
26252651 return base64Image;
26262652}
26272653
@@ -3822,6 +3848,42 @@ function applyCommandArguments(args) {
38223848 return currentSettings;
38233849}
38243850
3851+function registerFunctionTool() {
3852+ if (!extension_settings.sd.function_tool) {
3853+ return ToolManager.unregisterFunctionTool('GenerateImage');
3854+ }
3855+
3856+ ToolManager.registerFunctionTool({
3857+ name: 'GenerateImage',
3858+ displayName: 'Generate Image',
3859+ description: [
3860+ 'Generate an image from a given text prompt.',
3861+ 'Use when a user asks for an image, a selfie, to picture a scene, etc.',
3862+ ].join(' '),
3863+ parameters: Object.freeze({
3864+ $schema: 'http://json-schema.org/draft-04/schema#',
3865+ type: 'object',
3866+ properties: {
3867+ prompt: {
3868+ type: 'string',
3869+ description: extension_settings.sd.prompts[generationMode.TOOL] || promptTemplates[generationMode.TOOL],
3870+ },
3871+ },
3872+ required: [
3873+ 'prompt',
3874+ ],
3875+ }),
3876+ action: async (args) => {
3877+ if (!isValidState()) throw new Error('Image generation is not configured.');
3878+ if (!args) throw new Error('Missing arguments');
3879+ if (!args.prompt) throw new Error('Missing prompt');
3880+ const url = await generatePicture(initiators.tool, {}, args.prompt);
3881+ return encodeURI(url);
3882+ },
3883+ formatMessage: () => 'Generating an image...',
3884+ });
3885+}
3886+
38253887jQuery(async () => {
38263888 await addSDGenButtons();
38273889
@@ -4175,6 +4237,7 @@ jQuery(async () => {
41754237 $('#sd_stability_key').on('click', onStabilityKeyClick);
41764238 $('#sd_stability_style_preset').on('change', onStabilityStylePresetChange);
41774239 $('#sd_huggingface_model_id').on('input', onHFModelInput);
4240+ $('#sd_function_tool').on('input', onFunctionToolInput);
41784241
41794242 if (!CSS.supports('field-sizing', 'content')) {
41804243 $('.sd_settings .inline-drawer-toggle').on('click', function () {
public/scripts/extensions/stable-diffusion/settings.html+4 -0
@@ -18,6 +18,10 @@
1818 <input id="sd_interactive_mode" type="checkbox" />
1919 <span data-i18n="sd_interactive_mode_txt">Interactive mode</span>
2020 </label>
21+ <label for="sd_function_tool" class="checkbox_label" data-i18n="[title]sd_function_tool" title="Use the function tool to automatically detect intents to generate images.">
22+ <input id="sd_function_tool" type="checkbox" />
23+ <span data-i18n="sd_function_tool_txt">Enable function tool</span>
24+ </label>
2125 <label for="sd_multimodal_captioning" class="checkbox_label" data-i18n="[title]sd_multimodal_captioning" title="Use multimodal captioning to generate prompts for user and character portraits based on their avatars.">
2226 <input id="sd_multimodal_captioning" type="checkbox" />
2327 <span data-i18n="sd_multimodal_captioning_txt">Use multimodal captioning for portraits</span>
public/scripts/instruct-mode.js+0 -2
@@ -554,11 +554,9 @@ export function formatInstructModePrompt(name, isImpersonate, promptBias, name1,
554554 * @param {string} name Preset name.
555555 */
556556function selectMatchingContextTemplate(name) {
557- let foundMatch = false;
558557 for (const context_preset of context_presets) {
559558 // If context template matches the instruct preset
560559 if (context_preset.name === name) {
561- foundMatch = true;
562560 selectContextPreset(context_preset.name, { isAuto: true });
563561 break;
564562 }
public/scripts/kai-settings.js+1 -1
@@ -188,7 +188,7 @@ export async function generateKoboldWithStreaming(generate_data, signal) {
188188 if (data?.token) {
189189 text += data.token;
190190 }
191191 yield { text, swipes: [], toolCalls: [] };
192192 }
193193 };
194194}
public/scripts/nai-settings.js+1 -1
@@ -746,7 +746,7 @@ export async function generateNovelWithStreaming(generate_data, signal) {
746746 text += data.token;
747747 }
748748
749749 yield { text, swipes: [], logprobs: parseNovelAILogprobs(data.logprobs), toolCalls: [] };
750750 }
751751 };
752752}
public/scripts/openai.js+77 -149
@@ -70,6 +70,7 @@ import { renderTemplateAsync } from './templates.js';
7070import { SlashCommandEnumValue } from './slash-commands/SlashCommandEnumValue.js';
7171import { Popup, POPUP_RESULT } from './popup.js';
7272import { t } from './i18n.js';
73+import { ToolManager } from './tool-calling.js';
7374
7475export {
7576 openai_messages_count,
@@ -198,7 +199,10 @@ const continue_postfix_types = {
198199
199200const custom_prompt_post_processing_types = {
200201 NONE: '',
202+ /** @deprecated Use MERGE instead. */
201203 CLAUDE: 'claude',
204+ MERGE: 'merge',
205+ STRICT: 'strict',
202206};
203207
204208const sensitiveFields = [
@@ -453,7 +457,8 @@ function setOpenAIMessages(chat) {
453457 if (role == 'user' && oai_settings.wrap_in_quotes) content = `"${content}"`;
454458 const name = chat[j]['name'];
455459 const image = chat[j]?.extra?.image;
456- messages[i] = { 'role': role, 'content': content, name: name, 'image': image };
460+ const invocations = chat[j]?.extra?.tool_invocations;
461+ messages[i] = { 'role': role, 'content': content, name: name, 'image': image, 'invocations': invocations };
457462 j++;
458463 }
459464
@@ -701,6 +706,7 @@ async function populateChatHistory(messages, prompts, chatCompletion, type = nul
701706 }
702707
703708 const imageInlining = isImageInliningSupported();
709+ const canUseTools = ToolManager.isToolCallingSupported();
704710
705711 // Insert chat messages as long as there is budget available
706712 const chatPool = [...messages].reverse();
@@ -722,6 +728,28 @@ async function populateChatHistory(messages, prompts, chatCompletion, type = nul
722728 await chatMessage.addImage(chatPrompt.image);
723729 }
724730
731+ if (canUseTools && Array.isArray(chatPrompt.invocations)) {
732+ /** @type {import('./tool-calling.js').ToolInvocation[]} */
733+ const invocations = chatPrompt.invocations;
734+ const toolCallMessage = new Message(chatMessage.role, undefined, 'toolCall-' + chatMessage.identifier);
735+ toolCallMessage.setToolCalls(invocations);
736+ if (chatCompletion.canAfford(toolCallMessage)) {
737+ chatCompletion.reserveBudget(toolCallMessage);
738+ for (const invocation of invocations.slice().reverse()) {
739+ const toolResultMessage = new Message('tool', invocation.result || '[No content]', invocation.id);
740+ const canAfford = chatCompletion.canAfford(toolResultMessage);
741+ if (!canAfford) {
742+ break;
743+ }
744+ chatCompletion.insertAtStart(toolResultMessage, 'chatHistory');
745+ }
746+ chatCompletion.freeBudget(toolCallMessage);
747+ chatCompletion.insertAtStart(toolCallMessage, 'chatHistory');
748+ }
749+
750+ continue;
751+ }
752+
725753 if (chatCompletion.canAfford(chatMessage)) {
726754 if (type === 'continue' && oai_settings.continue_prefill && chatPrompt === firstNonInjected) {
727755 // in case we are using continue_prefill and the latest message is an assistant message, we want to prepend the users assistant prefill on the message
@@ -1262,7 +1290,7 @@ export async function prepareOpenAIMessages({
12621290 const eventData = { chat, dryRun };
12631291 await eventSource.emit(event_types.CHAT_COMPLETION_PROMPT_READY, eventData);
12641292
12651293 openai_messages_count = chat.filter(x => !x?.tool_calls && (x?.role === 'user' || x?.role === 'assistant'))?.length || 0;
12661294
12671295 return [chat, promptManager.tokenHandler.counts];
12681296}
@@ -1687,7 +1715,6 @@ async function sendOpenAIRequest(type, messages, signal) {
16871715 messages = messages.filter(msg => msg && typeof msg === 'object');
16881716
16891717 let logit_bias = {};
1690- const messageId = getNextMessageId(type);
16911718 const isClaude = oai_settings.chat_completion_source == chat_completion_sources.CLAUDE;
16921719 const isOpenRouter = oai_settings.chat_completion_source == chat_completion_sources.OPENROUTER;
16931720 const isScale = oai_settings.chat_completion_source == chat_completion_sources.SCALE;
@@ -1860,8 +1887,8 @@ async function sendOpenAIRequest(type, messages, signal) {
18601887 generate_data['seed'] = oai_settings.seed;
18611888 }
18621889
18631890 if (isFunctionCallingSupported()!canMultiSwipe && !streamToolManager.canPerformToolCalls(type)) {
18641891 await registerFunctionToolsToolManager.registerFunctionToolsOpenAI(type, generate_data);
18651892 }
18661893
18671894 if (isOAI && oai_settings.openai_model.startsWith('o1-')) {
@@ -1908,6 +1935,7 @@ async function sendOpenAIRequest(type, messages, signal) {
19081935 return async function* streamData() {
19091936 let text = '';
19101937 const swipes = [];
1938+ const toolCalls = [];
19111939 while (true) {
19121940 const { done, value } = await reader.read();
19131941 if (done) return;
@@ -1923,7 +1951,9 @@ async function sendOpenAIRequest(type, messages, signal) {
19231951 text += getStreamingReply(parsed);
19241952 }
19251953
1926- yield { text, swipes: swipes, logprobs: parseChatCompletionLogprobs(parsed) };
1954+ ToolManager.parseToolCalls(toolCalls, parsed);
1955+
1956+ yield { text, swipes: swipes, logprobs: parseChatCompletionLogprobs(parsed), toolCalls: toolCalls };
19271957 }
19281958 };
19291959 }
@@ -1945,147 +1975,10 @@ async function sendOpenAIRequest(type, messages, signal) {
19451975 delay(1).then(() => saveLogprobsForActiveMessage(logprobs, null));
19461976 }
19471977
1948- if (isFunctionCallingSupported()) {
1949- await checkFunctionToolCalls(data);
1950- }
1951-
19521978 return data;
19531979 }
19541980}
19551981
1956-/**
1957- * Register function tools for the next chat completion request.
1958- * @param {string} type Generation type
1959- * @param {object} data Generation data
1960- */
1961-async function registerFunctionTools(type, data) {
1962- let toolChoice = 'auto';
1963- const tools = [];
1964-
1965- /**
1966- * @type {registerFunctionTool}
1967- */
1968- const registerFunctionTool = (name, description, parameters, required) => {
1969- tools.push({
1970- type: 'function',
1971- function: {
1972- name,
1973- description,
1974- parameters,
1975- },
1976- });
1977-
1978- if (required) {
1979- toolChoice = 'required';
1980- }
1981- };
1982-
1983- /**
1984- * @type {FunctionToolRegister}
1985- */
1986- const args = {
1987- type,
1988- data,
1989- registerFunctionTool,
1990- };
1991-
1992- await eventSource.emit(event_types.LLM_FUNCTION_TOOL_REGISTER, args);
1993-
1994- if (tools.length) {
1995- console.log('Registered function tools:', tools);
1996-
1997- data['tools'] = tools;
1998- data['tool_choice'] = toolChoice;
1999- }
2000-}
2001-
2002-async function checkFunctionToolCalls(data) {
2003- const oaiCompat = [
2004- chat_completion_sources.OPENAI,
2005- chat_completion_sources.CUSTOM,
2006- chat_completion_sources.MISTRALAI,
2007- chat_completion_sources.OPENROUTER,
2008- chat_completion_sources.GROQ,
2009- ];
2010- if (oaiCompat.includes(oai_settings.chat_completion_source)) {
2011- if (!Array.isArray(data?.choices)) {
2012- return;
2013- }
2014-
2015- // Find a choice with 0-index
2016- const choice = data.choices.find(choice => choice.index === 0);
2017-
2018- if (!choice) {
2019- return;
2020- }
2021-
2022- const toolCalls = choice.message.tool_calls;
2023-
2024- if (!Array.isArray(toolCalls)) {
2025- return;
2026- }
2027-
2028- for (const toolCall of toolCalls) {
2029- if (typeof toolCall.function !== 'object') {
2030- continue;
2031- }
2032-
2033- /** @type {FunctionToolCall} */
2034- const args = toolCall.function;
2035- console.log('Function tool call:', toolCall);
2036- await eventSource.emit(event_types.LLM_FUNCTION_TOOL_CALL, args);
2037- }
2038- }
2039-
2040- if ([chat_completion_sources.CLAUDE].includes(oai_settings.chat_completion_source)) {
2041- if (!Array.isArray(data?.content)) {
2042- return;
2043- }
2044-
2045- for (const content of data.content) {
2046- if (content.type === 'tool_use') {
2047- /** @type {FunctionToolCall} */
2048- const args = { name: content.name, arguments: JSON.stringify(content.input) };
2049- await eventSource.emit(event_types.LLM_FUNCTION_TOOL_CALL, args);
2050- }
2051- }
2052- }
2053-
2054- if ([chat_completion_sources.COHERE].includes(oai_settings.chat_completion_source)) {
2055- if (!Array.isArray(data?.tool_calls)) {
2056- return;
2057- }
2058-
2059- for (const toolCall of data.tool_calls) {
2060- /** @type {FunctionToolCall} */
2061- const args = { name: toolCall.name, arguments: JSON.stringify(toolCall.parameters) };
2062- console.log('Function tool call:', toolCall);
2063- await eventSource.emit(event_types.LLM_FUNCTION_TOOL_CALL, args);
2064- }
2065- }
2066-}
2067-
2068-export function isFunctionCallingSupported() {
2069- if (main_api !== 'openai') {
2070- return false;
2071- }
2072-
2073- if (!oai_settings.function_calling) {
2074- return false;
2075- }
2076-
2077- const supportedSources = [
2078- chat_completion_sources.OPENAI,
2079- chat_completion_sources.COHERE,
2080- chat_completion_sources.CUSTOM,
2081- chat_completion_sources.MISTRALAI,
2082- chat_completion_sources.CLAUDE,
2083- chat_completion_sources.OPENROUTER,
2084- chat_completion_sources.GROQ,
2085- ];
2086- return supportedSources.includes(oai_settings.chat_completion_source);
2087-}
2088-
20891982function getStreamingReply(data) {
20901983 if (oai_settings.chat_completion_source == chat_completion_sources.CLAUDE) {
20911984 return data?.delta?.text || '';
@@ -2323,6 +2216,8 @@ class Message {
23232216 content;
23242217 /** @type {string} */
23252218 name;
2219+ /** @type {object} */
2220+ tool_call = null;
23262221
23272222 /**
23282223 * @constructor
@@ -2347,6 +2242,22 @@ class Message {
23472242 }
23482243 }
23492244
2245+ /**
2246+ * Reconstruct the message from a tool invocation.
2247+ * @param {import('./tool-calling.js').ToolInvocation[]} invocations
2248+ */
2249+ setToolCalls(invocations) {
2250+ this.tool_calls = invocations.map(i => ({
2251+ id: i.id,
2252+ type: 'function',
2253+ function: {
2254+ arguments: i.parameters,
2255+ name: i.name,
2256+ },
2257+ }));
2258+ this.tokens = tokenHandler.count({ role: this.role, tool_calls: JSON.stringify(this.tool_calls) });
2259+ }
2260+
23502261 setName(name) {
23512262 this.name = name;
23522263 this.tokens = tokenHandler.count({ role: this.role, content: this.content, name: this.name });
@@ -2483,13 +2394,20 @@ class MessageCollection {
24832394 }
24842395
24852396 /**
24862397 * Get chat in the format of {role, name, content, tool_calls}.
24872398 * @returns {Array} Array of objects with role, name, and content properties.
24882399 */
24892400 getChat() {
24902401 return this.collection.reduce((acc, message) => {
2491- const name = message.name;
2402+ if (message.content || message.tool_calls) {
2492- if (message.content) acc.push({ role: message.role, ...(name && { name }), content: message.content });
2403+ acc.push({
2404+ role: message.role,
2405+ content: message.content,
2406+ ...(message.name && { name: message.name }),
2407+ ...(message.tool_calls && { tool_calls: message.tool_calls }),
2408+ ...(message.role === 'tool' && { tool_call_id: message.identifier }),
2409+ });
2410+ }
24932411 return acc;
24942412 }, []);
24952413 }
@@ -2694,7 +2612,7 @@ export class ChatCompletion {
26942612 this.checkTokenBudget(message, message.identifier);
26952613
26962614 const index = this.findMessageIndex(identifier);
26972615 if (message.content || message.tool_calls) {
26982616 if ('start' === position) this.messages.collection[index].collection.unshift(message);
26992617 else if ('end' === position) this.messages.collection[index].collection.push(message);
27002618 else if (typeof position === 'number') this.messages.collection[index].collection.splice(position, 0, message);
@@ -2763,8 +2681,14 @@ export class ChatCompletion {
27632681 for (let item of this.messages.collection) {
27642682 if (item instanceof MessageCollection) {
27652683 chat.push(...item.getChat());
27662684 } else if (item instanceof Message && (item.content || item.tool_calls)) {
2767- const message = { role: item.role, content: item.content, ...(item.name ? { name: item.name } : {}) };
2685+ const message = {
2686+ role: item.role,
2687+ content: item.content,
2688+ ...(item.name ? { name: item.name } : {}),
2689+ ...(item.tool_calls ? { tool_calls: item.tool_calls } : {}),
2690+ ...(item.role === 'tool' ? { tool_call_id: item.identifier } : {}),
2691+ };
27682692 chat.push(message);
27692693 } else {
27702694 this.log(`Skipping invalid or empty message in collection: ${JSON.stringify(item)}`);
@@ -3118,6 +3042,10 @@ function loadOpenAISettings(data, settings) {
31183042 setNamesBehaviorControls();
31193043 setContinuePostfixControls();
31203044
3045+ if (oai_settings.custom_prompt_post_processing === custom_prompt_post_processing_types.CLAUDE) {
3046+ oai_settings.custom_prompt_post_processing = custom_prompt_post_processing_types.MERGE;
3047+ }
3048+
31213049 $('#chat_completion_source').val(oai_settings.chat_completion_source).trigger('change');
31223050 $('#oai_max_context_unlocked').prop('checked', oai_settings.max_context_unlocked);
31233051 $('#custom_prompt_post_processing').val(oai_settings.custom_prompt_post_processing);
public/scripts/slash-commands/SlashCommandReturnHelper.js+1 -1
@@ -58,7 +58,7 @@ export const slashCommandReturnHelper = {
5858 case 'toast-html': {
5959 const htmlOrNotHtml = shouldHtml ? DOMPurify.sanitize((new showdown.Converter()).makeHtml(stringValue)) : escapeHtml(stringValue);
6060
6161 if (type.startsWith('popup')) await callGenericPopup(htmlOrNotHtml, POPUP_TYPE.TEXT, '', { allowVerticalScrolling: true, wide: true });
6262 if (type.startsWith('chat')) sendSystemMessage(system_message_types.GENERIC, htmlOrNotHtml);
6363 if (type.startsWith('toast')) toastr.info(htmlOrNotHtml, null, { escapeHtml: !shouldHtml });
6464
public/scripts/textgen-settings.js+2 -1
@@ -916,6 +916,7 @@ async function generateTextGenWithStreaming(generate_data, signal) {
916916 /** @type {import('./logprobs.js').TokenLogprobs | null} */
917917 let logprobs = null;
918918 const swipes = [];
919+ const toolCalls = [];
919920 while (true) {
920921 const { done, value } = await reader.read();
921922 if (done) return;
@@ -934,7 +935,7 @@ async function generateTextGenWithStreaming(generate_data, signal) {
934935 logprobs = parseTextgenLogprobs(newText, data.choices?.[0]?.logprobs || data?.completion_probabilities);
935936 }
936937
937938 yield { text, swipes, logprobs, toolCalls };
938939 }
939940 };
940941}
public/scripts/tool-calling.js+876 -0
@@ -0,0 +1,876 @@
1+import { addOneMessage, chat, event_types, eventSource, main_api, saveChatConditional, system_avatar, systemUserName } from '../script.js';
2+import { chat_completion_sources, oai_settings } from './openai.js';
3+import { Popup } from './popup.js';
4+import { SlashCommand } from './slash-commands/SlashCommand.js';
5+import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from './slash-commands/SlashCommandArgument.js';
6+import { SlashCommandClosure } from './slash-commands/SlashCommandClosure.js';
7+import { enumIcons } from './slash-commands/SlashCommandCommonEnumsProvider.js';
8+import { enumTypes, SlashCommandEnumValue } from './slash-commands/SlashCommandEnumValue.js';
9+import { SlashCommandParser } from './slash-commands/SlashCommandParser.js';
10+import { slashCommandReturnHelper } from './slash-commands/SlashCommandReturnHelper.js';
11+
12+/**
13+ * @typedef {object} ToolInvocation
14+ * @property {string} id - A unique identifier for the tool invocation.
15+ * @property {string} displayName - The display name of the tool.
16+ * @property {string} name - The name of the tool.
17+ * @property {string} parameters - The parameters for the tool invocation.
18+ * @property {string} result - The result of the tool invocation.
19+ */
20+
21+/**
22+ * @typedef {object} ToolInvocationResult
23+ * @property {ToolInvocation[]} invocations Successful tool invocations
24+ * @property {Error[]} errors Errors that occurred during tool invocation
25+ */
26+
27+/**
28+ * @typedef {object} ToolRegistration
29+ * @property {string} name - The name of the tool.
30+ * @property {string} displayName - The display name of the tool.
31+ * @property {string} description - A description of the tool.
32+ * @property {object} parameters - The parameters for the tool.
33+ * @property {function} action - The action to perform when the tool is invoked.
34+ * @property {function} formatMessage - A function to format the tool call message.
35+ */
36+
37+/**
38+ * @typedef {object} ToolDefinitionOpenAI
39+ * @property {string} type - The type of the tool.
40+ * @property {object} function - The function definition.
41+ * @property {string} function.name - The name of the function.
42+ * @property {string} function.description - The description of the function.
43+ * @property {object} function.parameters - The parameters of the function.
44+ * @property {function} toString - A function to convert the tool to a string.
45+ */
46+
47+/**
48+ * Assigns nested variables to a scope.
49+ * @param {import('./slash-commands/SlashCommandScope.js').SlashCommandScope} scope The scope to assign variables to.
50+ * @param {object} arg Object to assign variables from.
51+ * @param {string} prefix Prefix for the variable names.
52+ */
53+function assignNestedVariables(scope, arg, prefix) {
54+ Object.entries(arg).forEach(([key, value]) => {
55+ const newPrefix = `${prefix}.${key}`;
56+ if (typeof value === 'object' && value !== null) {
57+ assignNestedVariables(scope, value, newPrefix);
58+ } else {
59+ scope.letVariable(newPrefix, value);
60+ }
61+ });
62+}
63+
64+/**
65+ * Checks if a string is a valid JSON string.
66+ * @param {string} str The string to check
67+ * @returns {boolean} If the string is a valid JSON string
68+ */
69+function isJson(str) {
70+ try {
71+ JSON.parse(str);
72+ return true;
73+ } catch {
74+ return false;
75+ }
76+}
77+
78+/**
79+ * Tries to parse a string as JSON, returning the original string if parsing fails.
80+ * @param {string} str The string to try to parse
81+ * @returns {object|string} Parsed JSON or the original string
82+ */
83+function tryParse(str) {
84+ try {
85+ return JSON.parse(str);
86+ } catch {
87+ return str;
88+ }
89+}
90+
91+/**
92+ * A class that represents a tool definition.
93+ */
94+class ToolDefinition {
95+ /**
96+ * A unique name for the tool.
97+ * @type {string}
98+ */
99+ #name;
100+
101+ /**
102+ * A user-friendly display name for the tool.
103+ * @type {string}
104+ */
105+ #displayName;
106+
107+ /**
108+ * A description of what the tool does.
109+ * @type {string}
110+ */
111+ #description;
112+
113+ /**
114+ * A JSON schema for the parameters that the tool accepts.
115+ * @type {object}
116+ */
117+ #parameters;
118+
119+ /**
120+ * A function that will be called when the tool is executed.
121+ * @type {function}
122+ */
123+ #action;
124+
125+ /**
126+ * A function that will be called to format the tool call toast.
127+ * @type {function}
128+ */
129+ #formatMessage;
130+
131+ /**
132+ * Creates a new ToolDefinition.
133+ * @param {string} name A unique name for the tool.
134+ * @param {string} displayName A user-friendly display name for the tool.
135+ * @param {string} description A description of what the tool does.
136+ * @param {object} parameters A JSON schema for the parameters that the tool accepts.
137+ * @param {function} action A function that will be called when the tool is executed.
138+ * @param {function} formatMessage A function that will be called to format the tool call toast.
139+ */
140+ constructor(name, displayName, description, parameters, action, formatMessage) {
141+ this.#name = name;
142+ this.#displayName = displayName;
143+ this.#description = description;
144+ this.#parameters = parameters;
145+ this.#action = action;
146+ this.#formatMessage = formatMessage;
147+ }
148+
149+ /**
150+ * Converts the ToolDefinition to an OpenAI API representation
151+ * @returns {ToolDefinitionOpenAI} OpenAI API representation of the tool.
152+ */
153+ toFunctionOpenAI() {
154+ return {
155+ type: 'function',
156+ function: {
157+ name: this.#name,
158+ description: this.#description,
159+ parameters: this.#parameters,
160+ },
161+ toString: function () {
162+ return `<div><b>${this.function.name}</b></div><div><small>${this.function.description}</small></div><pre class="justifyLeft wordBreakAll"><code class="flex padding5">${JSON.stringify(this.function.parameters, null, 2)}</code></pre><hr>`;
163+ },
164+ };
165+ }
166+
167+ /**
168+ * Invokes the tool with the given parameters.
169+ * @param {object} parameters The parameters to pass to the tool.
170+ * @returns {Promise<any>} The result of the tool's action function.
171+ */
172+ async invoke(parameters) {
173+ return await this.#action(parameters);
174+ }
175+
176+ /**
177+ * Formats a message with the tool invocation.
178+ * @param {object} parameters The parameters to pass to the tool.
179+ * @returns {string} The formatted message.
180+ */
181+ formatMessage(parameters) {
182+ return typeof this.#formatMessage === 'function'
183+ ? this.#formatMessage(parameters)
184+ : `Invoking tool: ${this.#displayName || this.#name}`;
185+ }
186+
187+ get displayName() {
188+ return this.#displayName;
189+ }
190+}
191+
192+/**
193+ * A class that manages the registration and invocation of tools.
194+ */
195+export class ToolManager {
196+ /**
197+ * A map of tool names to tool definitions.
198+ * @type {Map<string, ToolDefinition>}
199+ */
200+ static #tools = new Map();
201+
202+ static #INPUT_DELTA_KEY = '__input_json_delta';
203+
204+ /**
205+ * Returns an Array of all tools that have been registered.
206+ * @type {ToolDefinition[]}
207+ */
208+ static get tools() {
209+ return Array.from(this.#tools.values());
210+ }
211+
212+ /**
213+ * Registers a new tool with the tool registry.
214+ * @param {ToolRegistration} tool The tool to register.
215+ */
216+ static registerFunctionTool({ name, displayName, description, parameters, action, formatMessage }) {
217+ // Convert WIP arguments
218+ if (typeof arguments[0] !== 'object') {
219+ [name, description, parameters, action] = arguments;
220+ }
221+
222+ if (this.#tools.has(name)) {
223+ console.warn(`A tool with the name "${name}" has already been registered. The definition will be overwritten.`);
224+ }
225+
226+ const definition = new ToolDefinition(name, displayName, description, parameters, action, formatMessage);
227+ this.#tools.set(name, definition);
228+ console.log('[ToolManager] Registered function tool:', definition);
229+ }
230+
231+ /**
232+ * Removes a tool from the tool registry.
233+ * @param {string} name The name of the tool to unregister.
234+ */
235+ static unregisterFunctionTool(name) {
236+ if (!this.#tools.has(name)) {
237+ return;
238+ }
239+
240+ this.#tools.delete(name);
241+ console.log(`[ToolManager] Unregistered function tool: ${name}`);
242+ }
243+
244+ /**
245+ * Invokes a tool by name. Returns the result of the tool's action function.
246+ * @param {string} name The name of the tool to invoke.
247+ * @param {object} parameters Function parameters. For example, if the tool requires a "name" parameter, you would pass {name: "value"}.
248+ * @returns {Promise<string|Error>} The result of the tool's action function. If an error occurs, null is returned. Non-string results are JSON-stringified.
249+ */
250+ static async invokeFunctionTool(name, parameters) {
251+ try {
252+ if (!this.#tools.has(name)) {
253+ throw new Error(`No tool with the name "${name}" has been registered.`);
254+ }
255+
256+ const invokeParameters = typeof parameters === 'string' ? JSON.parse(parameters) : parameters;
257+ const tool = this.#tools.get(name);
258+ const result = await tool.invoke(invokeParameters);
259+ return typeof result === 'string' ? result : JSON.stringify(result);
260+ } catch (error) {
261+ console.error(`An error occurred while invoking the tool "${name}":`, error);
262+
263+ if (error instanceof Error) {
264+ error.cause = name;
265+ return error;
266+ }
267+
268+ return new Error('Unknown error occurred while invoking the tool.', { cause: name });
269+ }
270+ }
271+
272+ /**
273+ * Formats a message for a tool call by name.
274+ * @param {string} name The name of the tool to format the message for.
275+ * @param {object} parameters Function tool call parameters.
276+ * @returns {string} The formatted message for the tool call.
277+ */
278+ static formatToolCallMessage(name, parameters) {
279+ if (!this.#tools.has(name)) {
280+ return `Invoked unknown tool: ${name}`;
281+ }
282+
283+ try {
284+ const tool = this.#tools.get(name);
285+ const formatParameters = typeof parameters === 'string' ? JSON.parse(parameters) : parameters;
286+ return tool.formatMessage(formatParameters);
287+ } catch (error) {
288+ console.error(`An error occurred while formatting the tool call message for "${name}":`, error);
289+ return `Invoking tool: ${name}`;
290+ }
291+ }
292+
293+ /**
294+ * Gets the display name of a tool by name.
295+ * @param {string} name
296+ * @returns {string} The display name of the tool.
297+ */
298+ static getDisplayName(name) {
299+ if (!this.#tools.has(name)) {
300+ return name;
301+ }
302+
303+ const tool = this.#tools.get(name);
304+ return tool.displayName || name;
305+ }
306+
307+ /**
308+ * Register function tools for the next chat completion request.
309+ * @param {object} data Generation data
310+ */
311+ static async registerFunctionToolsOpenAI(data) {
312+ const tools = [];
313+
314+ for (const tool of ToolManager.tools) {
315+ tools.push(tool.toFunctionOpenAI());
316+ }
317+
318+ if (tools.length) {
319+ console.log('Registered function tools:', tools);
320+
321+ data['tools'] = tools;
322+ data['tool_choice'] = 'auto';
323+ }
324+ }
325+
326+ /**
327+ * Utility function to parse tool calls from a parsed response.
328+ * @param {any[]} toolCalls The tool calls to update.
329+ * @param {any} parsed The parsed response from the OpenAI API.
330+ * @returns {void}
331+ */
332+ static parseToolCalls(toolCalls, parsed) {
333+ if (Array.isArray(parsed?.choices)) {
334+ for (const choice of parsed.choices) {
335+ const choiceIndex = (typeof choice.index === 'number') ? choice.index : null;
336+ const choiceDelta = choice.delta;
337+
338+ if (choiceIndex === null || !choiceDelta) {
339+ continue;
340+ }
341+
342+ const toolCallDeltas = choiceDelta?.tool_calls;
343+
344+ if (!Array.isArray(toolCallDeltas)) {
345+ continue;
346+ }
347+
348+ if (!Array.isArray(toolCalls[choiceIndex])) {
349+ toolCalls[choiceIndex] = [];
350+ }
351+
352+ for (const toolCallDelta of toolCallDeltas) {
353+ const toolCallIndex = (typeof toolCallDelta?.index === 'number') ? toolCallDelta.index : toolCallDeltas.indexOf(toolCallDelta);
354+
355+ if (isNaN(toolCallIndex) || toolCallIndex < 0) {
356+ continue;
357+ }
358+
359+ if (toolCalls[choiceIndex][toolCallIndex] === undefined) {
360+ toolCalls[choiceIndex][toolCallIndex] = {};
361+ }
362+
363+ const targetToolCall = toolCalls[choiceIndex][toolCallIndex];
364+
365+ ToolManager.#applyToolCallDelta(targetToolCall, toolCallDelta);
366+ }
367+ }
368+ }
369+ if (typeof parsed?.content_block === 'object') {
370+ const choiceIndex = 0;
371+ const toolCallIndex = parsed?.index ?? 0;
372+
373+ if (parsed?.content_block?.type === 'tool_use') {
374+ if (!Array.isArray(toolCalls[choiceIndex])) {
375+ toolCalls[choiceIndex] = [];
376+ }
377+ if (toolCalls[choiceIndex][toolCallIndex] === undefined) {
378+ toolCalls[choiceIndex][toolCallIndex] = {};
379+ }
380+ const targetToolCall = toolCalls[choiceIndex][toolCallIndex];
381+ ToolManager.#applyToolCallDelta(targetToolCall, parsed.content_block);
382+ }
383+ }
384+ if (typeof parsed?.delta === 'object') {
385+ const choiceIndex = 0;
386+ const toolCallIndex = parsed?.index ?? 0;
387+ const targetToolCall = toolCalls[choiceIndex]?.[toolCallIndex];
388+ if (targetToolCall) {
389+ if (parsed?.delta?.type === 'input_json_delta') {
390+ const jsonDelta = parsed?.delta?.partial_json;
391+ if (!targetToolCall[this.#INPUT_DELTA_KEY]) {
392+ targetToolCall[this.#INPUT_DELTA_KEY] = '';
393+ }
394+ targetToolCall[this.#INPUT_DELTA_KEY] += jsonDelta;
395+ }
396+ }
397+ }
398+ if (parsed?.type === 'content_block_stop') {
399+ const choiceIndex = 0;
400+ const toolCallIndex = parsed?.index ?? 0;
401+ const targetToolCall = toolCalls[choiceIndex]?.[toolCallIndex];
402+ if (targetToolCall) {
403+ const jsonDeltaString = targetToolCall[this.#INPUT_DELTA_KEY];
404+ if (jsonDeltaString) {
405+ try {
406+ const jsonDelta = { input: JSON.parse(jsonDeltaString) };
407+ delete targetToolCall[this.#INPUT_DELTA_KEY];
408+ ToolManager.#applyToolCallDelta(targetToolCall, jsonDelta);
409+ } catch (error) {
410+ console.warn('Failed to apply input JSON delta:', error);
411+ }
412+ }
413+ }
414+ }
415+ }
416+
417+ /**
418+ * Apply a tool call delta to a target object.
419+ * @param {object} target The target object to apply the delta to
420+ * @param {object} delta The delta object to apply
421+ */
422+ static #applyToolCallDelta(target, delta) {
423+ for (const key in delta) {
424+ if (!Object.prototype.hasOwnProperty.call(delta, key)) continue;
425+ if (key === '__proto__' || key === 'constructor') continue;
426+
427+ const deltaValue = delta[key];
428+ const targetValue = target[key];
429+
430+ if (deltaValue === null || deltaValue === undefined) {
431+ target[key] = deltaValue;
432+ continue;
433+ }
434+
435+ if (typeof deltaValue === 'string') {
436+ if (typeof targetValue === 'string') {
437+ // Concatenate strings
438+ target[key] = targetValue + deltaValue;
439+ } else {
440+ target[key] = deltaValue;
441+ }
442+ } else if (typeof deltaValue === 'object' && !Array.isArray(deltaValue)) {
443+ if (typeof targetValue !== 'object' || targetValue === null || Array.isArray(targetValue)) {
444+ target[key] = {};
445+ }
446+ // Recursively apply deltas to nested objects
447+ ToolManager.#applyToolCallDelta(target[key], deltaValue);
448+ } else {
449+ // Assign other types directly
450+ target[key] = deltaValue;
451+ }
452+ }
453+ }
454+
455+ /**
456+ * Checks if tool calling is supported for the current settings and generation type.
457+ * @returns {boolean} Whether tool calling is supported for the given type
458+ */
459+ static isToolCallingSupported() {
460+ if (main_api !== 'openai' || !oai_settings.function_calling) {
461+ return false;
462+ }
463+
464+ const supportedSources = [
465+ chat_completion_sources.OPENAI,
466+ chat_completion_sources.CUSTOM,
467+ chat_completion_sources.MISTRALAI,
468+ chat_completion_sources.CLAUDE,
469+ chat_completion_sources.OPENROUTER,
470+ chat_completion_sources.GROQ,
471+ ];
472+ return supportedSources.includes(oai_settings.chat_completion_source);
473+ }
474+
475+ /**
476+ * Checks if tool calls can be performed for the current settings and generation type.
477+ * @param {string} type Generation type
478+ * @returns {boolean} Whether tool calls can be performed for the given type
479+ */
480+ static canPerformToolCalls(type) {
481+ const noToolCallTypes = ['swipe', 'impersonate', 'quiet', 'continue'];
482+ const isSupported = ToolManager.isToolCallingSupported();
483+ return isSupported && !noToolCallTypes.includes(type);
484+ }
485+
486+ /**
487+ * Utility function to get tool calls from the response data.
488+ * @param {any} data Response data
489+ * @returns {any[]} Tool calls from the response data
490+ */
491+ static #getToolCallsFromData(data) {
492+ const isClaudeToolCall = c => Array.isArray(c) ? c.filter(x => x).every(isClaudeToolCall) : c?.input && c?.name && c?.id;
493+ const convertClaudeToolCall = c => ({ id: c.id, function: { name: c.name, arguments: c.input } });
494+
495+ // Parsed tool calls from streaming data
496+ if (Array.isArray(data) && data.length > 0 && Array.isArray(data[0])) {
497+ return isClaudeToolCall(data[0]) ? data[0].filter(x => x).map(convertClaudeToolCall) : data[0];
498+ }
499+
500+ // Parsed tool calls from non-streaming data
501+ if (Array.isArray(data?.choices)) {
502+ // Find a choice with 0-index
503+ const choice = data.choices.find(choice => choice.index === 0);
504+
505+ if (choice) {
506+ return choice.message.tool_calls;
507+ }
508+ }
509+
510+ // Claude tool calls to OpenAI tool calls
511+ if (Array.isArray(data?.content)) {
512+ const content = data.content.filter(c => c.type === 'tool_use').map(convertClaudeToolCall);
513+
514+ if (content) {
515+ return content;
516+ }
517+ }
518+ }
519+
520+ /**
521+ * Checks if the response data contains tool calls.
522+ * @param {object} data Response data
523+ * @returns {boolean} Whether the response data contains tool calls
524+ */
525+ static hasToolCalls(data) {
526+ const toolCalls = ToolManager.#getToolCallsFromData(data);
527+ return Array.isArray(toolCalls) && toolCalls.length > 0;
528+ }
529+
530+ /**
531+ * Check for function tool calls in the response data and invoke them.
532+ * @param {any} data Reply data
533+ * @returns {Promise<ToolInvocationResult>} Successful tool invocations
534+ */
535+ static async invokeFunctionTools(data) {
536+ /** @type {ToolInvocationResult} */
537+ const result = {
538+ invocations: [],
539+ errors: [],
540+ };
541+ const toolCalls = ToolManager.#getToolCallsFromData(data);
542+
543+ if (!Array.isArray(toolCalls)) {
544+ return result;
545+ }
546+
547+ for (const toolCall of toolCalls) {
548+ if (typeof toolCall.function !== 'object') {
549+ continue;
550+ }
551+
552+ console.log('Function tool call:', toolCall);
553+ const id = toolCall.id;
554+ const parameters = toolCall.function.arguments;
555+ const name = toolCall.function.name;
556+ const displayName = ToolManager.getDisplayName(name);
557+
558+ const message = ToolManager.formatToolCallMessage(name, parameters);
559+ const toast = message && toastr.info(message, 'Tool Calling', { timeOut: 0 });
560+ const toolResult = await ToolManager.invokeFunctionTool(name, parameters);
561+ toastr.clear(toast);
562+ console.log('Function tool result:', result);
563+
564+ // Save a successful invocation
565+ if (toolResult instanceof Error) {
566+ result.errors.push(toolResult);
567+ continue;
568+ }
569+
570+ const invocation = {
571+ id,
572+ displayName,
573+ name,
574+ parameters,
575+ result: toolResult,
576+ };
577+ result.invocations.push(invocation);
578+ }
579+
580+ return result;
581+ }
582+
583+ /**
584+ * Groups tool names by count.
585+ * @param {string[]} toolNames Tool names
586+ * @returns {string} Grouped tool names
587+ */
588+ static #groupToolNames(toolNames) {
589+ const toolCounts = toolNames.reduce((acc, name) => {
590+ acc[name] = (acc[name] || 0) + 1;
591+ return acc;
592+ }, {});
593+ return Object.entries(toolCounts).map(([name, count]) => count > 1 ? `${name} (${count})` : name).join(', ');
594+ }
595+
596+ /**
597+ * Formats a message with tool invocations.
598+ * @param {ToolInvocation[]} invocations Tool invocations.
599+ * @returns {string} Formatted message with tool invocations.
600+ */
601+ static #formatToolInvocationMessage(invocations) {
602+ const data = structuredClone(invocations);
603+ const detailsElement = document.createElement('details');
604+ const summaryElement = document.createElement('summary');
605+ const preElement = document.createElement('pre');
606+ const codeElement = document.createElement('code');
607+ codeElement.classList.add('language-json');
608+ data.forEach(i => {
609+ i.parameters = tryParse(i.parameters);
610+ i.result = tryParse(i.result);
611+ });
612+ codeElement.textContent = JSON.stringify(data, null, 2);
613+ const toolNames = data.map(i => i.displayName || i.name);
614+ summaryElement.textContent = `Tool calls: ${this.#groupToolNames(toolNames)}`;
615+ preElement.append(codeElement);
616+ detailsElement.append(summaryElement, preElement);
617+ return detailsElement.outerHTML;
618+ }
619+
620+ /**
621+ * Saves function tool invocations to the last user chat message extra metadata.
622+ * @param {ToolInvocation[]} invocations Successful tool invocations
623+ */
624+ static async saveFunctionToolInvocations(invocations) {
625+ if (!Array.isArray(invocations) || invocations.length === 0) {
626+ return;
627+ }
628+ const message = {
629+ name: systemUserName,
630+ force_avatar: system_avatar,
631+ is_system: true,
632+ is_user: false,
633+ mes: ToolManager.#formatToolInvocationMessage(invocations),
634+ extra: {
635+ isSmallSys: true,
636+ tool_invocations: invocations,
637+ },
638+ };
639+ chat.push(message);
640+ await eventSource.emit(event_types.TOOL_CALLS_PERFORMED, invocations);
641+ addOneMessage(message);
642+ await eventSource.emit(event_types.TOOL_CALLS_RENDERED, invocations);
643+ await saveChatConditional();
644+ }
645+
646+ /**
647+ * Shows an error message for tool calls.
648+ * @param {Error[]} errors Errors that occurred during tool invocation
649+ * @returns {void}
650+ */
651+ static showToolCallError(errors) {
652+ toastr.error('An error occurred while invoking function tools. Click here for more details.', 'Tool Calling', {
653+ onclick: () => Popup.show.text('Tool Calling Errors', DOMPurify.sanitize(errors.map(e => `${e.cause}: ${e.message}`).join('<br>'))),
654+ timeOut: 5000,
655+ });
656+ }
657+
658+ static initToolSlashCommands() {
659+ const toolsEnumProvider = () => ToolManager.tools.map(tool => {
660+ const toolOpenAI = tool.toFunctionOpenAI();
661+ return new SlashCommandEnumValue(toolOpenAI.function.name, toolOpenAI.function.description, enumTypes.enum, enumIcons.closure);
662+ });
663+
664+ SlashCommandParser.addCommandObject(SlashCommand.fromProps({
665+ name: 'tools-list',
666+ aliases: ['tool-list'],
667+ helpString: 'Gets a list of all registered tools in the OpenAI function JSON format. Use the <code>return</code> argument to specify the return value type.',
668+ returns: 'A list of all registered tools.',
669+ namedArgumentList: [
670+ SlashCommandNamedArgument.fromProps({
671+ name: 'return',
672+ description: 'The way how you want the return value to be provided',
673+ typeList: [ARGUMENT_TYPE.STRING],
674+ defaultValue: 'none',
675+ enumList: slashCommandReturnHelper.enumList({ allowObject: true }),
676+ forceEnum: true,
677+ }),
678+ ],
679+ callback: async (args) => {
680+ /** @type {any} */
681+ const returnType = String(args?.return ?? 'popup-html').trim().toLowerCase();
682+ const objectToStringFunc = (tools) => Array.isArray(tools) ? tools.map(x => x.toString()).join('\n\n') : tools.toString();
683+ const tools = ToolManager.tools.map(tool => tool.toFunctionOpenAI());
684+ return await slashCommandReturnHelper.doReturn(returnType ?? 'popup-html', tools ?? [], { objectToStringFunc });
685+ },
686+ }));
687+
688+ SlashCommandParser.addCommandObject(SlashCommand.fromProps({
689+ name: 'tools-invoke',
690+ aliases: ['tool-invoke'],
691+ helpString: 'Invokes a registered tool by name. The <code>parameters</code> argument MUST be a JSON-serialized object.',
692+ namedArgumentList: [
693+ SlashCommandNamedArgument.fromProps({
694+ name: 'parameters',
695+ description: 'The parameters to pass to the tool.',
696+ typeList: [ARGUMENT_TYPE.DICTIONARY],
697+ isRequired: true,
698+ acceptsMultiple: false,
699+ }),
700+ ],
701+ unnamedArgumentList: [
702+ SlashCommandArgument.fromProps({
703+ description: 'The name of the tool to invoke.',
704+ typeList: [ARGUMENT_TYPE.STRING],
705+ isRequired: true,
706+ acceptsMultiple: false,
707+ forceEnum: true,
708+ enumProvider: toolsEnumProvider,
709+ }),
710+ ],
711+ callback: async (args, name) => {
712+ const { parameters } = args;
713+
714+ const result = await ToolManager.invokeFunctionTool(String(name), parameters);
715+ if (result instanceof Error) {
716+ throw result;
717+ }
718+
719+ return result;
720+ },
721+ }));
722+
723+ SlashCommandParser.addCommandObject(SlashCommand.fromProps({
724+ name: 'tools-register',
725+ aliases: ['tool-register'],
726+ helpString: `<div>Registers a new tool with the tool registry.</div>
727+ <ul>
728+ <li>The <code>parameters</code> argument MUST be a JSON-serialized object with a valid JSON schema.</li>
729+ <li>The unnamed argument MUST be a closure that accepts the function parameters as local script variables.</li>
730+ </ul>
731+ <div>See <a target="_blank" href="https://json-schema.org/learn/">json-schema.org</a> and <a target="_blank" href="https://platform.openai.com/docs/guides/function-calling">OpenAI Function Calling</a> for more information.</div>
732+ <div>Example:</div>
733+ <pre><code>/let key=echoSchema
734+{
735+ "$schema": "http://json-schema.org/draft-04/schema#",
736+ "type": "object",
737+ "properties": {
738+ "message": {
739+ "type": "string",
740+ "description": "The message to echo."
741+ }
742+ },
743+ "required": [
744+ "message"
745+ ]
746+}
747+||
748+/tools-register name=Echo description="Echoes a message. Call when the user is asking to repeat something" parameters={{var::echoSchema}} {: /echo {{var::arg.message}} :}</code></pre>`,
749+ namedArgumentList: [
750+ SlashCommandNamedArgument.fromProps({
751+ name: 'name',
752+ description: 'The name of the tool.',
753+ typeList: [ARGUMENT_TYPE.STRING],
754+ isRequired: true,
755+ acceptsMultiple: false,
756+ }),
757+ SlashCommandNamedArgument.fromProps({
758+ name: 'description',
759+ description: 'A description of what the tool does.',
760+ typeList: [ARGUMENT_TYPE.STRING],
761+ isRequired: true,
762+ acceptsMultiple: false,
763+ }),
764+ SlashCommandNamedArgument.fromProps({
765+ name: 'parameters',
766+ description: 'The parameters for the tool.',
767+ typeList: [ARGUMENT_TYPE.DICTIONARY],
768+ isRequired: true,
769+ acceptsMultiple: false,
770+ }),
771+ SlashCommandNamedArgument.fromProps({
772+ name: 'displayName',
773+ description: 'The display name of the tool.',
774+ typeList: [ARGUMENT_TYPE.STRING],
775+ isRequired: false,
776+ acceptsMultiple: false,
777+ }),
778+ SlashCommandNamedArgument.fromProps({
779+ name: 'formatMessage',
780+ description: 'The closure to be executed to format the tool call message. Must return a string.',
781+ typeList: [ARGUMENT_TYPE.CLOSURE],
782+ isRequired: true,
783+ acceptsMultiple: false,
784+ }),
785+ ],
786+ unnamedArgumentList: [
787+ SlashCommandArgument.fromProps({
788+ description: 'The closure to be executed when the tool is invoked.',
789+ typeList: [ARGUMENT_TYPE.CLOSURE],
790+ isRequired: true,
791+ acceptsMultiple: false,
792+ }),
793+ ],
794+ callback: async (args, action) => {
795+ /**
796+ * Converts a slash command closure to a function.
797+ * @param {SlashCommandClosure} action Closure to convert to a function
798+ * @returns {function} Function that executes the closure
799+ */
800+ function closureToFunction(action) {
801+ return async (args) => {
802+ const localClosure = action.getCopy();
803+ localClosure.onProgress = () => { };
804+ const scope = localClosure.scope;
805+ if (typeof args === 'object' && args !== null) {
806+ assignNestedVariables(scope, args, 'arg');
807+ } else if (typeof args !== 'undefined') {
808+ scope.letVariable('arg', args);
809+ }
810+ const result = await localClosure.execute();
811+ return result.pipe;
812+ };
813+ }
814+
815+ const { name, displayName, description, parameters, formatMessage } = args;
816+
817+ if (!(action instanceof SlashCommandClosure)) {
818+ throw new Error('The unnamed argument must be a closure.');
819+ }
820+ if (typeof name !== 'string' || !name) {
821+ throw new Error('The "name" argument must be a non-empty string.');
822+ }
823+ if (typeof description !== 'string' || !description) {
824+ throw new Error('The "description" argument must be a non-empty string.');
825+ }
826+ if (typeof parameters !== 'string' || !isJson(parameters)) {
827+ throw new Error('The "parameters" argument must be a JSON-serialized object.');
828+ }
829+ if (displayName && typeof displayName !== 'string') {
830+ throw new Error('The "displayName" argument must be a string.');
831+ }
832+ if (formatMessage && !(formatMessage instanceof SlashCommandClosure)) {
833+ throw new Error('The "formatMessage" argument must be a closure.');
834+ }
835+
836+ const actionFunc = closureToFunction(action);
837+ const formatMessageFunc = formatMessage instanceof SlashCommandClosure ? closureToFunction(formatMessage) : null;
838+
839+ ToolManager.registerFunctionTool({
840+ name: String(name ?? ''),
841+ displayName: String(displayName ?? ''),
842+ description: String(description ?? ''),
843+ parameters: JSON.parse(parameters ?? '{}'),
844+ action: actionFunc,
845+ formatMessage: formatMessageFunc,
846+ });
847+
848+ return '';
849+ },
850+ }));
851+
852+ SlashCommandParser.addCommandObject(SlashCommand.fromProps({
853+ name: 'tools-unregister',
854+ aliases: ['tool-unregister'],
855+ helpString: 'Unregisters a tool from the tool registry.',
856+ unnamedArgumentList: [
857+ SlashCommandArgument.fromProps({
858+ description: 'The name of the tool to unregister.',
859+ typeList: [ARGUMENT_TYPE.STRING],
860+ isRequired: true,
861+ acceptsMultiple: false,
862+ forceEnum: true,
863+ enumProvider: toolsEnumProvider,
864+ }),
865+ ],
866+ callback: async (name) => {
867+ if (typeof name !== 'string' || !name) {
868+ throw new Error('The unnamed argument must be a non-empty string.');
869+ }
870+
871+ ToolManager.unregisterFunctionTool(name);
872+ return '';
873+ },
874+ }));
875+ }
876+}
public/style.css+10 -0
@@ -424,6 +424,16 @@ small {
424424 text-align: center;
425425}
426426
427+.mes.smallSysMes pre {
428+ text-align: initial;
429+ word-break: break-all;
430+ margin-top: 5px;
431+}
432+
433+.mes.smallSysMes summary {
434+ cursor: pointer;
435+}
436+
427437.mes.smallSysMes .mes_text p:last-child {
428438 margin: 0;
429439}
src/endpoints/backends/chat-completions.js+20 -29
@@ -4,7 +4,7 @@ const fetch = require('node-fetch').default;
44const { jsonParser } = require('../../express-common');
55const { CHAT_COMPLETION_SOURCES, GEMINI_SAFETY, BISON_SAFETY, OPENROUTER_HEADERS } = require('../../constants');
66const { forwardFetchResponse, getConfigValue, tryParse, uuidv4, mergeObjectWithYaml, excludeKeysByYaml, color } = require('../../util');
77const { convertClaudeMessages, convertGooglePrompt, convertTextCompletionPrompt, convertCohereMessages, convertMistralMessages, convertCohereToolsconvertAI21Messages, convertAI21MessagesmergeMessages } = require('../../prompt-converters');
88const CohereStream = require('../../cohere-stream');
99
1010const { readSecret, SECRET_KEYS } = require('../secrets');
@@ -31,8 +31,11 @@ const API_AI21 = 'https://api.ai21.com/studio/v1';
3131 */
3232function postProcessPrompt(messages, type, charName, userName) {
3333 switch (type) {
34+ case 'merge':
3435 case 'claude':
3536 return convertClaudeMessagesmergeMessages(messages, '', false, '', charName, userName, false).messages;
37+ case 'strict':
38+ return mergeMessages(messages, charName, userName, true);
3639 default:
3740 return messages;
3841 }
@@ -84,7 +87,7 @@ async function sendClaudeRequest(request, response) {
8487 const apiUrl = new URL(request.body.reverse_proxy || API_CLAUDE).toString();
8588 const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.CLAUDE);
8689 const divider = '-'.repeat(process.stdout.columns);
8790 const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false) && request.body.model.startsWith('claude-3');
8891
8992 if (!apiKey) {
9093 console.log(color.red(`Claude API key is missing.\n${divider}`));
@@ -107,7 +110,7 @@ async function sendClaudeRequest(request, response) {
107110 }
108111
109112 const requestBody = {
110113 /** @type {any} */ system: ''[],
111114 messages: convertedPrompt.messages,
112115 model: request.body.model,
113116 max_tokens: request.body.max_tokens,
@@ -118,9 +121,11 @@ async function sendClaudeRequest(request, response) {
118121 stream: request.body.stream,
119122 };
120123 if (useSystemPrompt) {
121- requestBody.system = enableSystemPromptCache
124+ if (enableSystemPromptCache && Array.isArray(convertedPrompt.systemPrompt) && convertedPrompt.systemPrompt.length) {
122125 ? convertedPrompt.systemPrompt[{ type: 'text', text: convertedPrompt.systemPrompt,.length - 1]['cache_control:'] = { type: 'ephemeral' } }];
123- : convertedPrompt.systemPrompt;
126+ }
127+
128+ requestBody.system = convertedPrompt.systemPrompt;
124129 } else {
125130 delete requestBody.system;
126131 }
@@ -130,11 +135,15 @@ async function sendClaudeRequest(request, response) {
130135 convertedPrompt.messages.push({ role: 'user', content: '.' });
131136 }
132137 additionalHeaders['anthropic-beta'] = 'tools-2024-05-16';
133138 requestBody.tool_choice = { type: request.body.tool_choice === 'required' ? 'any' : 'auto' };
134139 requestBody.tools = request.body.tools
135140 .filter(tool => tool.type === 'function')
136141 .map(tool => tool.function)
137142 .map(fn => ({ name: fn.name, description: fn.description, input_schema: fn.parameters }));
143+
144+ if (enableSystemPromptCache && requestBody.tools.length) {
145+ requestBody.tools[requestBody.tools.length - 1]['cache_control'] = { type: 'ephemeral' };
146+ }
138147 }
139148 if (enableSystemPromptCache) {
140149 additionalHeaders['anthropic-beta'] = 'prompt-caching-2024-07-31';
@@ -483,7 +492,7 @@ async function sendMistralAIRequest(request, response) {
483492
484493 if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
485494 requestBody['tools'] = request.body.tools;
486495 requestBody['tool_choice'] = request.body.tool_choice === 'required' ? 'any' : 'auto';
487496 }
488497
489498 const config = {
@@ -553,12 +562,6 @@ async function sendCohereRequest(request, response) {
553562 });
554563 }
555564
556- if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
557- tools.push(...convertCohereTools(request.body.tools));
558- // Can't have both connectors and tools in the same request
559- connectors.splice(0, connectors.length);
560- }
561-
562565 // https://docs.cohere.com/reference/chat
563566 const requestBody = {
564567 stream: Boolean(request.body.stream),
@@ -908,24 +911,12 @@ router.post('/generate', jsonParser, function (request, response) {
908911 apiKey = readSecret(request.user.directories, SECRET_KEYS.PERPLEXITY);
909912 headers = {};
910913 bodyParams = {};
911914 request.body.messages = postProcessPrompt(request.body.messages, 'claudestrict', request.body.char_name, request.body.user_name);
912915 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.GROQ) {
913916 apiUrl = API_GROQ;
914917 apiKey = readSecret(request.user.directories, SECRET_KEYS.GROQ);
915918 headers = {};
916919 bodyParams = {};
917-
918- // 'required' tool choice is not supported by Groq
919- if (request.body.tool_choice === 'required') {
920- if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
921- request.body.tool_choice = request.body.tools.length > 1
922- ? 'auto' :
923- { type: 'function', function: { name: request.body.tools[0]?.function?.name } };
924-
925- } else {
926- request.body.tool_choice = 'none';
927- }
928- }
929920 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.ZEROONEAI) {
930921 apiUrl = API_01AI;
931922 apiKey = readSecret(request.user.directories, SECRET_KEYS.ZEROONEAI);
@@ -962,7 +953,7 @@ router.post('/generate', jsonParser, function (request, response) {
962953 controller.abort();
963954 });
964955
965- if (!isTextCompletion) {
956+ if (!isTextCompletion && Array.isArray(request.body.tools) && request.body.tools.length > 0) {
966957 bodyParams['tools'] = request.body.tools;
967958 bodyParams['tool_choice'] = request.body.tool_choice;
968959 }
src/endpoints/novelai.js+1 -1
@@ -51,7 +51,7 @@ const eratoRepPenWhitelist = [
5151 6, 1, 11, 13, 25, 198, 12, 9, 8, 279, 264, 459, 323, 477, 539, 912, 374, 574, 1051, 1550, 1587, 4536, 5828, 15058,
5252 3287, 3250, 1461, 1077, 813, 11074, 872, 1202, 1436, 7846, 1288, 13434, 1053, 8434, 617, 9167, 1047, 19117, 706,
5353 12775, 649, 4250, 527, 7784, 690, 2834, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 1210, 1359, 608, 220, 596, 956,
5454 3077, 44886, 4265, 3358, 2351, 2846, 311, 389, 315, 304, 520, 505, 430,
5555];
5656
5757// Ban the dinkus and asterism
src/endpoints/search.js+70 -49
@@ -22,58 +22,15 @@ const visitHeaders = {
2222 'Sec-Fetch-User': '?1',
2323};
2424
25-router.post('/serpapi', jsonParser, async (request, response) => {
26- try {
27- const key = readSecret(request.user.directories, SECRET_KEYS.SERPAPI);
28-
29- if (!key) {
30- console.log('No SerpApi key found');
31- return response.sendStatus(400);
32- }
33-
34- const { query } = request.body;
35- const result = await fetch(`https://serpapi.com/search.json?q=${encodeURIComponent(query)}&api_key=${key}`);
36-
37- console.log('SerpApi query', query);
38-
39- if (!result.ok) {
40- const text = await result.text();
41- console.log('SerpApi request failed', result.statusText, text);
42- return response.status(500).send(text);
43- }
44-
45- const data = await result.json();
46- return response.json(data);
47- } catch (error) {
48- console.log(error);
49- return response.sendStatus(500);
50- }
51-});
52-
5325/**
5426 * GetExtract the transcript of a YouTube video
55- * @copyright https://github.com/Kakulukian/youtube-transcript (MIT License)
27+ * @param {string} videoPageBody HTML of the video page
28+ * @param {string} lang Language code
29+ * @returns {Promise<string>} Transcript text
5630 */
57-router.post('/transcript', jsonParser, async (request, response) => {
31+async function extractTranscript(videoPageBody, lang) {
58- try {
5932 const he = require('he');
6033 const RE_XML_TRANSCRIPT = /<text start="([^"]*)" dur="([^"]*)">([^<]*)<\/text>/g;
61- const id = request.body.id;
62- const lang = request.body.lang;
63-
64- if (!id) {
65- console.log('Id is required for /transcript');
66- return response.sendStatus(400);
67- }
68-
69- const videoPageResponse = await fetch(`https://www.youtube.com/watch?v=${id}`, {
70- headers: {
71- ...(lang && { 'Accept-Language': lang }),
72- 'User-Agent': visitHeaders['User-Agent'],
73- },
74- });
75-
76- const videoPageBody = await videoPageResponse.text();
7734 const splittedHTML = videoPageBody.split('"captions":');
7835
7936 if (splittedHTML.length <= 1) {
@@ -128,8 +85,72 @@ router.post('/transcript', jsonParser, async (request, response) => {
12885 }));
12986 // The text is double-encoded
13087 const transcriptText = transcript.map((line) => he.decode(he.decode(line.text))).join(' ');
88+ return transcriptText;
89+}
90+
91+router.post('/serpapi', jsonParser, async (request, response) => {
92+ try {
93+ const key = readSecret(request.user.directories, SECRET_KEYS.SERPAPI);
94+
95+ if (!key) {
96+ console.log('No SerpApi key found');
97+ return response.sendStatus(400);
98+ }
99+
100+ const { query } = request.body;
101+ const result = await fetch(`https://serpapi.com/search.json?q=${encodeURIComponent(query)}&api_key=${key}`);
102+
103+ console.log('SerpApi query', query);
104+
105+ if (!result.ok) {
106+ const text = await result.text();
107+ console.log('SerpApi request failed', result.statusText, text);
108+ return response.status(500).send(text);
109+ }
110+
111+ const data = await result.json();
112+ return response.json(data);
113+ } catch (error) {
114+ console.log(error);
115+ return response.sendStatus(500);
116+ }
117+});
118+
119+/**
120+ * Get the transcript of a YouTube video
121+ * @copyright https://github.com/Kakulukian/youtube-transcript (MIT License)
122+ */
123+router.post('/transcript', jsonParser, async (request, response) => {
124+ try {
125+ const id = request.body.id;
126+ const lang = request.body.lang;
127+ const json = request.body.json;
128+
129+ if (!id) {
130+ console.log('Id is required for /transcript');
131+ return response.sendStatus(400);
132+ }
131133
132- return response.send(transcriptText);
134+ const videoPageResponse = await fetch(`https://www.youtube.com/watch?v=${id}`, {
135+ headers: {
136+ ...(lang && { 'Accept-Language': lang }),
137+ 'User-Agent': visitHeaders['User-Agent'],
138+ },
139+ });
140+
141+ const videoPageBody = await videoPageResponse.text();
142+
143+ try {
144+ const transcriptText = await extractTranscript(videoPageBody, lang);
145+ return json
146+ ? response.json({ transcript: transcriptText, html: videoPageBody })
147+ : response.send(transcriptText);
148+ } catch (error) {
149+ if (json) {
150+ return response.json({ html: videoPageBody, transcript: '' });
151+ }
152+ throw error;
153+ }
133154 } catch (error) {
134155 console.log(error);
135156 return response.sendStatus(500);
src/prompt-converters.js+215 -119
@@ -1,5 +1,8 @@
11require('./polyfill.js');
22const { getConfigValue } = require('./util.js');
3+const crypto = require('crypto');
4+
5+const PROMPT_PLACEHOLDER = getConfigValue('promptPlaceholder', 'Let\'s get started.');
36
47/**
58 * Convert a prompt from the ChatML objects to the format used by Claude.
@@ -19,6 +22,14 @@ function convertClaudePrompt(messages, addAssistantPostfix, addAssistantPrefill,
1922 //Prepare messages for claude.
2023 //When 'Exclude Human/Assistant prefixes' checked, setting messages role to the 'system'(last message is exception).
2124 if (messages.length > 0) {
25+ messages.forEach((m) => {
26+ if (!m.content) {
27+ m.content = '';
28+ }
29+ if (m.tool_calls) {
30+ m.content += JSON.stringify(m.tool_calls);
31+ }
32+ });
2233 if (excludePrefixes) {
2334 messages.slice(0, -1).forEach(message => message.role = 'system');
2435 } else {
@@ -85,7 +96,7 @@ function convertClaudePrompt(messages, addAssistantPostfix, addAssistantPrefill,
8596 * @param {string} userName User name
8697 */
8798function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFix, charName = '', userName = '') {
8899 let systemPrompt = ''[];
89100 if (useSysPrompt) {
90101 // Collect all the system messages up until the first instance of a non-system message, and then remove them from the messages array.
91102 let i;
@@ -104,7 +115,7 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFi
104115 messages[i].content = `${charName}: ${messages[i].content}`;
105116 }
106117 }
107- systemPrompt += `${messages[i].content}\n\n`;
118+ systemPrompt.push({ type: 'text', text: messages[i].content });
108119 }
109120
110121 messages.splice(0, i);
@@ -114,12 +125,31 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFi
114125 if (messages.length === 0 || (messages.length > 0 && messages[0].role !== 'user')) {
115126 messages.unshift({
116127 role: 'user',
117128 content: humanMsgFix || '[Start a new chat]'PROMPT_PLACEHOLDER,
118129 });
119130 }
120131 }
132+
121133 // Now replace all further messages that have the role 'system' with the role 'user'. (or all if we're not using one)
122134 messages.forEach((message) => {
135+ if (message.role === 'assistant' && message.tool_calls) {
136+ message.content = message.tool_calls.map((tc) => ({
137+ type: 'tool_use',
138+ id: tc.id,
139+ name: tc.function.name,
140+ input: tc.function.arguments,
141+ }));
142+ }
143+
144+ if (message.role === 'tool') {
145+ message.role = 'user';
146+ message.content = [{
147+ type: 'tool_result',
148+ tool_use_id: message.tool_call_id,
149+ content: message.content,
150+ }];
151+ }
152+
123153 if (message.role === 'system') {
124154 if (userName && message.name === 'example_user') {
125155 message.content = `${userName}: ${message.content}`;
@@ -128,68 +158,96 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFi
128158 message.content = `${charName}: ${message.content}`;
129159 }
130160 message.role = 'user';
161+
162+ // Delete name here so it doesn't get added later
163+ delete message.name;
131164 }
132- });
133165
134- // Shouldn't be conditional anymore, messages api expects the last role to be user unless we're explicitly prefilling
166+ // Convert everything to an array of it would be easier to work with
135- if (prefillString) {
167+ if (typeof message.content === 'string') {
136- messages.push({
168+ // Take care of name properties since claude messages don't support them
137- role: 'assistant',
169+ if (message.name) {
138- content: prefillString.trimEnd(),
170+ message.content = `${message.name}: ${message.content}`;
139- });
140171 }
141172
142- // Since the messaging endpoint only supports user assistant roles in turns, we have to merge messages with the same role if they follow eachother
173+ message.content = [{ type: 'text', text: message.content }];
143- // Also handle multi-modality, holy slop.
174+ } else if (Array.isArray(message.content)) {
144- let mergedMessages = [];
175+ message.content = message.content.map((content) => {
145- messages.forEach((message) => {
176+ if (content.type === 'image_url') {
146177 const imageEntry = message.content?.[1]?.image_url;
147178 const imageData = imageEntry?.url;
148179 const mimeType = imageData?.split(';')?.[0].split(':')?.[1];
149180 const base64Data = imageData?.split(',')?.[1];
150181
151- // Take care of name properties since claude messages don't support them
182+ return {
183+ type: 'image',
184+ source: {
185+ type: 'base64',
186+ media_type: mimeType,
187+ data: base64Data,
188+ },
189+ };
190+ }
191+
192+ if (content.type === 'text') {
152193 if (message.name) {
153- if (Array.isArray(message.content)) {
194+ content.text = `${message.name}: ${content.text}`;
154- message.content[0].text = `${message.name}: ${message.content[0].text}`;
195+ }
155- } else {
196+
156- message.content = `${message.name}: ${message.content}`;
197+ return content;
198+ }
199+
200+ return content;
201+ });
157202 }
203+
204+ // Remove offending properties
158205 delete message.name;
206+ delete message.tool_calls;
207+ delete message.tool_call_id;
208+ });
209+
210+ // Images in assistant messages should be moved to the next user message
211+ for (let i = 0; i < messages.length; i++) {
212+ if (messages[i].role === 'assistant' && messages[i].content.some(c => c.type === 'image')) {
213+ // Find the next user message
214+ let j = i + 1;
215+ while (j < messages.length && messages[j].role !== 'user') {
216+ j++;
159217 }
160218
161- if (mergedMessages.length > 0 && mergedMessages[mergedMessages.length - 1].role === message.role) {
219+ // Move the images
162220 if (Array.isArray(messagej >= messages.content)length) {
163- if (Array.isArray(mergedMessages[mergedMessages.length - 1].content)) {
221+ // If there is no user message after the assistant message, add a new one
164- mergedMessages[mergedMessages.length - 1].content[0].text += '\n\n' + message.content[0].text;
222+ messages.splice(i + 1, 0, { role: 'user', content: [] });
165- } else {
223+ }
166- mergedMessages[mergedMessages.length - 1].content += '\n\n' + message.content[0].text;
224+
225+ messages[j].content.push(...messages[i].content.filter(c => c.type === 'image'));
226+ messages[i].content = messages[i].content.filter(c => c.type !== 'image');
167227 }
168- } else {
169- if (Array.isArray(mergedMessages[mergedMessages.length - 1].content)) {
170- mergedMessages[mergedMessages.length - 1].content[0].text += '\n\n' + message.content;
171- } else {
172- mergedMessages[mergedMessages.length - 1].content += '\n\n' + message.content;
173228 }
229+
230+ // Shouldn't be conditional anymore, messages api expects the last role to be user unless we're explicitly prefilling
231+ if (prefillString) {
232+ messages.push({
233+ role: 'assistant',
234+ // Dangling whitespace are not allowed for prefilling
235+ content: [{ type: 'text', text: prefillString.trimEnd() }],
236+ });
174237 }
238+
239+ // Since the messaging endpoint only supports user assistant roles in turns, we have to merge messages with the same role if they follow eachother
240+ // Also handle multi-modality, holy slop.
241+ let mergedMessages = [];
242+ messages.forEach((message) => {
243+ if (mergedMessages.length > 0 && mergedMessages[mergedMessages.length - 1].role === message.role) {
244+ mergedMessages[mergedMessages.length - 1].content.push(...message.content);
175245 } else {
176246 mergedMessages.push(message);
177247 }
178- if (imageData) {
179- mergedMessages[mergedMessages.length - 1].content = [
180- { type: 'text', text: mergedMessages[mergedMessages.length - 1].content[0]?.text || mergedMessages[mergedMessages.length - 1].content },
181- {
182- type: 'image', source: {
183- type: 'base64',
184- media_type: mimeType,
185- data: base64Data,
186- },
187- },
188- ];
189- }
190248 });
191249
192250 return { messages: mergedMessages, systemPrompt: systemPrompt.trim() };
193251}
194252
195253/**
@@ -205,7 +263,6 @@ function convertCohereMessages(messages, charName = '', userName = '') {
205263 'user': 'USER',
206264 'assistant': 'CHATBOT',
207265 };
208- const placeholder = '[Start a new chat]';
209266 let systemPrompt = '';
210267
211268 // Collect all the system messages up until the first instance of a non-system message, and then remove them from the messages array.
@@ -233,12 +290,12 @@ function convertCohereMessages(messages, charName = '', userName = '') {
233290 if (messages.length === 0) {
234291 messages.unshift({
235292 role: 'user',
236293 content: placeholderPROMPT_PLACEHOLDER,
237294 });
238295 }
239296
240297 const lastNonSystemMessageIndex = messages.findLastIndex(msg => msg.role === 'user' || msg.role === 'assistant');
241298 const userPrompt = messages.slice(lastNonSystemMessageIndex).map(msg => msg.content).join('\n\n') || placeholderPROMPT_PLACEHOLDER;
242299
243300 const chatHistory = messages.slice(0, lastNonSystemMessageIndex).map(msg => {
244301 return {
@@ -414,7 +471,7 @@ function convertAI21Messages(messages, charName = '', userName = '') {
414471 if (messages.length === 0) {
415472 messages.unshift({
416473 role: 'user',
417- content: '[Start a new chat]',
474+ content: PROMPT_PLACEHOLDER,
418475 });
419476 }
420477
@@ -466,8 +523,18 @@ function convertMistralMessages(messages, charName = '', userName = '') {
466523 lastMsg.prefix = true;
467524 }
468525
526+ const sanitizeToolId = (id) => crypto.hash('sha512', id, 'base64').slice(0, 9);
527+
469528 // Doesn't support completion names, so prepend if not already done by the frontend (e.g. for group chats).
470529 messages.forEach(msg => {
530+ if ('tool_calls' in msg && Array.isArray(msg.tool_calls)) {
531+ msg.tool_calls.forEach(tool => {
532+ tool.id = sanitizeToolId(tool.id);
533+ });
534+ }
535+ if ('tool_call_id' in msg && msg.role === 'tool') {
536+ msg.tool_call_id = sanitizeToolId(msg.tool_call_id);
537+ }
471538 if (msg.role === 'system' && msg.name === 'example_assistant') {
472539 if (charName && !msg.content.startsWith(`${charName}: `)) {
473540 msg.content = `${charName}: ${msg.content}`;
@@ -488,6 +555,28 @@ function convertMistralMessages(messages, charName = '', userName = '') {
488555 }
489556 });
490557
558+ // If user role message immediately follows a tool message, append it to the last user message
559+ const fixToolMessages = () => {
560+ let rerun = true;
561+ while (rerun) {
562+ rerun = false;
563+ messages.forEach((message, i) => {
564+ if (i === messages.length - 1) {
565+ return;
566+ }
567+ if (message.role === 'tool' && messages[i + 1].role === 'user') {
568+ const lastUserMessage = messages.slice(0, i).findLastIndex(m => m.role === 'user' && m.content);
569+ if (lastUserMessage !== -1) {
570+ messages[lastUserMessage].content += '\n\n' + messages[i + 1].content;
571+ messages.splice(i + 1, 1);
572+ rerun = true;
573+ }
574+ }
575+ });
576+ }
577+ };
578+ fixToolMessages();
579+
491580 // If system role message immediately follows an assistant message, change its role to user
492581 for (let i = 0; i < messages.length - 1; i++) {
493582 if (messages[i].role === 'assistant' && messages[i + 1].role === 'system') {
@@ -499,6 +588,83 @@ function convertMistralMessages(messages, charName = '', userName = '') {
499588}
500589
501590/**
591+ * Merge messages with the same consecutive role, removing names if they exist.
592+ * @param {any[]} messages Messages to merge
593+ * @param {string} charName Character name
594+ * @param {string} userName User name
595+ * @param {boolean} strict Enable strict mode: only allow one system message at the start, force user first message
596+ * @returns {any[]} Merged messages
597+ */
598+function mergeMessages(messages, charName, userName, strict) {
599+ let mergedMessages = [];
600+
601+ // Remove names from the messages
602+ messages.forEach((message) => {
603+ if (!message.content) {
604+ message.content = '';
605+ }
606+ if (message.role === 'system' && message.name === 'example_assistant') {
607+ if (charName && !message.content.startsWith(`${charName}: `)) {
608+ message.content = `${charName}: ${message.content}`;
609+ }
610+ }
611+ if (message.role === 'system' && message.name === 'example_user') {
612+ if (userName && !message.content.startsWith(`${userName}: `)) {
613+ message.content = `${userName}: ${message.content}`;
614+ }
615+ }
616+ if (message.name && message.role !== 'system') {
617+ if (!message.content.startsWith(`${message.name}: `)) {
618+ message.content = `${message.name}: ${message.content}`;
619+ }
620+ }
621+ if (message.role === 'tool') {
622+ message.role = 'user';
623+ }
624+ delete message.name;
625+ delete message.tool_calls;
626+ delete message.tool_call_id;
627+ });
628+
629+ // Squash consecutive messages with the same role
630+ messages.forEach((message) => {
631+ if (mergedMessages.length > 0 && mergedMessages[mergedMessages.length - 1].role === message.role && message.content) {
632+ mergedMessages[mergedMessages.length - 1].content += '\n\n' + message.content;
633+ } else {
634+ mergedMessages.push(message);
635+ }
636+ });
637+
638+ // Prevent erroring out if the messages array is empty.
639+ if (messages.length === 0) {
640+ messages.unshift({
641+ role: 'user',
642+ content: PROMPT_PLACEHOLDER,
643+ });
644+ }
645+
646+ if (strict) {
647+ for (let i = 0; i < mergedMessages.length; i++) {
648+ // Force mid-prompt system messages to be user messages
649+ if (i > 0 && mergedMessages[i].role === 'system') {
650+ mergedMessages[i].role = 'user';
651+ }
652+ }
653+ if (mergedMessages.length) {
654+ if (mergedMessages[0].role === 'system' && (mergedMessages.length === 1 || mergedMessages[1].role !== 'user')) {
655+ mergedMessages.splice(1, 0, { role: 'user', content: PROMPT_PLACEHOLDER });
656+ }
657+ else if (mergedMessages[0].role !== 'system' && mergedMessages[0].role !== 'user') {
658+ mergedMessages.unshift({ role: 'user', content: PROMPT_PLACEHOLDER });
659+ }
660+ }
661+ return mergeMessages(mergedMessages, charName, userName, false);
662+ }
663+
664+ return mergedMessages;
665+}
666+
667+/**
502668 * Convert a prompt from the ChatML objects to the format used by Text Completion API.
503669 * @param {object[]} messages Array of messages
504670 * @returns {string} Prompt for Text Completion API
@@ -523,76 +689,6 @@ function convertTextCompletionPrompt(messages) {
523689 return messageStrings.join('\n') + '\nassistant:';
524690}
525691
526-/**
527- * Convert OpenAI Chat Completion tools to the format used by Cohere.
528- * @param {object[]} tools OpenAI Chat Completion tool definitions
529- */
530-function convertCohereTools(tools) {
531- if (!Array.isArray(tools) || tools.length === 0) {
532- return [];
533- }
534-
535- const jsonSchemaToPythonTypes = {
536- 'string': 'str',
537- 'number': 'float',
538- 'integer': 'int',
539- 'boolean': 'bool',
540- 'array': 'list',
541- 'object': 'dict',
542- };
543-
544- const cohereTools = [];
545-
546- for (const tool of tools) {
547- if (tool?.type !== 'function') {
548- console.log(`Unsupported tool type: ${tool.type}`);
549- continue;
550- }
551-
552- const name = tool?.function?.name;
553- const description = tool?.function?.description;
554- const properties = tool?.function?.parameters?.properties;
555- const required = tool?.function?.parameters?.required;
556- const parameters = {};
557-
558- if (!name) {
559- console.log('Tool name is missing');
560- continue;
561- }
562-
563- if (!description) {
564- console.log('Tool description is missing');
565- }
566-
567- if (!properties || typeof properties !== 'object') {
568- console.log(`No properties found for tool: ${tool?.function?.name}`);
569- continue;
570- }
571-
572- for (const property in properties) {
573- const parameterDefinition = properties[property];
574- const description = parameterDefinition.description || (parameterDefinition.enum ? JSON.stringify(parameterDefinition.enum) : '');
575- const type = jsonSchemaToPythonTypes[parameterDefinition.type] || 'str';
576- const isRequired = Array.isArray(required) && required.includes(property);
577- parameters[property] = {
578- description: description,
579- type: type,
580- required: isRequired,
581- };
582- }
583-
584- const cohereTool = {
585- name: tool.function.name,
586- description: tool.function.description,
587- parameter_definitions: parameters,
588- };
589-
590- cohereTools.push(cohereTool);
591- }
592-
593- return cohereTools;
594-}
595-
596692module.exports = {
597693 convertClaudePrompt,
598694 convertClaudeMessages,
@@ -600,6 +696,6 @@ module.exports = {
600696 convertTextCompletionPrompt,
601697 convertCohereMessages,
602698 convertMistralMessages,
603- convertCohereTools,
604699 convertAI21Messages,
700+ mergeMessages,
605701};