Merge branch 'staging' into feat/AdditionalLogins

5765ea8500f7ac99e8b03dba538468f0652a5ae4

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: 1cda7003d1441769239ea06042ccf945aec23286 9e50ff34c99e3d405897e2df0d13fb19a01047dd

.eslintrc.js+11 -8 Viewed

             },
         },
     ],
-    // There are various vendored libraries that shouldn't be linted
     ignorePatterns: [
-        'public/lib/**/*',
+        '**/node_modules/**',
-        '*.min.js',
+        '**/dist/**',
-        'src/ai_horde/**/*',
+        '**/.git/**',
-        'plugins/**/*',
+        'public/lib/**',
-        'data/**/*',
+        'backups/**',
-        'backups/**/*',
+        'data/**',
-        'node_modules/**/*',
+        'cache/**',
+        'src/tokenizers/**',
+        'docker/**',
+        'plugins/**',
+        '**/*.min.js',
     ],
     rules: {
         'no-unused-vars': ['error', { args: 'none' }],

default/config.yaml+2 -0 Viewed

   speechToTextModel: Xenova/whisper-small
   textToSpeechModel: Xenova/speecht5_tts
 # -- OPENAI CONFIGURATION --
+# A placeholder message to use in strict prompt post-processing mode when the prompt doesn't start with a user message
+promptPlaceholder: "[Start a new chat]"
 openai:
   # Will send a random user ID to OpenAI completion API
   randomizeUserId: false

jsconfig.json+9 -10 Viewed

         "resolveJsonModule": true
     },
     "exclude": [
-        "node_modules",
+        "**/node_modules/**",
-        "**/node_modules/*",
+        "**/dist/**",
-        "public/lib",
+        "**/.git/**",
-        "backups/*",
+        "public/lib/**",
-        "data/*",
+        "backups/**",
-        "**/dist/*",
+        "data/**",
-        "dist/*",
+        "cache/**",
-        "cache/*",
+        "src/tokenizers/**",
-        "src/tokenizers/*",
+        "docker/**"
-        "docker/*",
+    ]
+}

package-lock.json+34 -12 Viewed

                 "sillytavern": "server.js"
             },
             "devDependencies": {
+                "@types/dompurify": "^3.0.5",
                 "@types/jquery": "^3.5.29",
-                "eslint": "^8.57.0",
+                "@types/toastr": "^2.1.43",
-                "jquery": "^3.6.4"
+                "eslint": "^8.57.0"
             },
             "engines": {
                 "node": ">= 18"

                 "@types/responselike": "^1.0.0"
+            }
         },
+        "node_modules/@types/dompurify": {
+            "version": "3.0.5",
+            "resolved": "https://registry.npmjs.org/@types/dompurify/-/dompurify-3.0.5.tgz",
+            "integrity": "sha512-1Wg0g3BtQF7sSb27fJQAKck1HECM6zV1EB66j8JH9i3LCjYabJa0FSdiSgsD5K/RbrsR0SiraKacLB+T8ZVYAg==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@types/trusted-types": "*"
+            }
+        },
         "node_modules/@types/http-cache-semantics": {
             "version": "4.0.2",
             "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.2.tgz",

             "license": "MIT"
         },
         "node_modules/@types/jquery": {
-            "version": "3.5.29",
+            "version": "3.5.31",
-            "resolved": "https://registry.npmjs.org/@types/jquery/-/jquery-3.5.29.tgz",
+            "resolved": "https://registry.npmjs.org/@types/jquery/-/jquery-3.5.31.tgz",
-            "integrity": "sha512-oXQQC9X9MOPRrMhPHHOsXqeQDnWeCDT3PelUIg/Oy8FAbzSZtFHRjc7IpbfFVmpLtJ+UOoywpRsuO5Jxjybyeg==",
+            "integrity": "sha512-rf/iB+cPJ/YZfMwr+FVuQbm7IaWC4y3FVYfVDxRGqmUCFjjPII0HWaP0vTPJGp6m4o13AXySCcMbWfrWtBFAKw==",
             "dev": true,
+            "license": "MIT",
             "dependencies": {
                 "@types/sizzle": "*"
+            }

             "integrity": "sha512-0vWLNK2D5MT9dg0iOo8GlKguPAU02QjmZitPEsXRuJXU/OGIOt9vT9Fc26wtYuavLxtO45v9PGleoL9Z0k1LHg==",
             "dev": true
         },
+        "node_modules/@types/toastr": {
+            "version": "2.1.43",
+            "resolved": "https://registry.npmjs.org/@types/toastr/-/toastr-2.1.43.tgz",
+            "integrity": "sha512-sLC2fr2OXeE1iyhUixpQ64wQ2tA26awmLidn4tXTLBz4yP/VhtYUKHpmiIyDtztKkHjucdiTLH8F5uRRyhNi2Q==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@types/jquery": "*"
+            }
+        },
+        "node_modules/@types/trusted-types": {
+            "version": "2.0.7",
+            "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz",
+            "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==",
+            "dev": true,
+            "license": "MIT"
+        },
         "node_modules/@ungap/structured-clone": {
             "version": "1.2.0",
             "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz",

             "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==",
             "license": "BSD-3-Clause"
         },
-        "node_modules/jquery": {
-            "version": "3.7.0",
-            "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.7.0.tgz",
-            "integrity": "sha512-umpJ0/k8X0MvD1ds0P9SfowREz2LenHsQaxSohMZ5OMNEU2r0tf8pdeEFTHMFxWVxKNyU9rTtK3CWzUCTKJUeQ==",
-            "dev": true,
-            "license": "MIT"
-        },
         "node_modules/js-yaml": {
             "version": "4.1.0",
             "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",

package.json+3 -2 Viewed

@@ -85,8 +85,9 @@
85	},	85	},
86	"main": "server.js",	86	"main": "server.js",
87	"devDependencies": {	87	"devDependencies": {
		88	"@types/dompurify": "^3.0.5",
88	"@types/jquery": "^3.5.29",	89	"@types/jquery": "^3.5.29",
89	"eslint": "^8.57.0",	90	"@types/toastr": "^2.1.43",
90	"jquery": "^3.6.4"	91	"eslint": "^8.57.0"
91	}	92	}
92	}	93	}

public/css/select2-overrides.css+2 -1 Viewed

 .select2-container .select2-selection .select2-selection__clear {
     color: var(--SmartThemeBodyColor);
-    font-size: 24px;
+    font-size: 20px;
     padding: 0;
     position: absolute;
     right: 5px;

     padding: revert;
     border-right: 1px solid var(--SmartThemeBorderColor);
     font-size: 1.1em;
+    line-height: 1;
+}
 .select2-container .select2-selection--multiple .select2-selection__choice__display {

public/css/st-tailwind.css+4 -0 Viewed

 ul.li-padding-bot10 li {
     padding-bottom: 10px;
+}
+.wordBreakAll {
+    word-break: break-all;
+}

public/css/toggle-dependent.css+11 -0 Viewed

 label[for="trim_spaces"]:has(input:checked) i.warning {
     display: none;
+}
+#claude_function_prefill_warning {
+    display: none;
+}
+#openai_settings:has(#openai_function_calling:checked):has(#claude_assistant_prefill:not(:placeholder-shown), #claude_assistant_impersonation:not(:placeholder-shown)) #claude_function_prefill_warning {
+    display: flex;
+    align-items: center;
+    gap: 5px;
+    margin: 10px 0;
+}

public/global.d.ts+0 -42 Viewed

 // Global namespace modules
-declare var DOMPurify;
 declare var droll;
 declare var Handlebars;
 declare var hljs;

 declare global {
   const moment: typeof moment;
+}
-/**
- * Callback data for the `LLM_FUNCTION_TOOL_REGISTER` event type that is triggered when a function tool can be registered.
- */
-interface FunctionToolRegister {
-    /**
-     * The type of generation that is being used
-     */
-    type?: string;
-    /**
-     * Generation data, including messages and sampling parameters
-     */
-    data: Record<string, object>;
-    /**
-     * Callback to register an LLM function tool.
-     */
-    registerFunctionTool: typeof registerFunctionTool;
-}
-/**
- * Callback data for the `LLM_FUNCTION_TOOL_REGISTER` event type that is triggered when a function tool is registered.
- * @param name Name of the function tool to register
- * @param description Description of the function tool
- * @param params JSON schema for the parameters of the function tool
- * @param required Whether the function tool should be forced to be used
- */
-declare function registerFunctionTool(name: string, description: string, params: object, required: boolean): Promise<void>;
-/**
- * Callback data for the `LLM_FUNCTION_TOOL_CALL` event type that is triggered when a function tool is called.
- */
-interface FunctionToolCall {
-    /**
-     * Name of the function tool to call
-     */
-    name: string;
-    /**
-     * JSON object with the parameters to pass to the function tool
-     */
-    arguments: string;
-}

public/index.html+6 -1 Viewed

                                             </div>
                                             <textarea id="claude_assistant_impersonation" class="text_pole textarea_compact autoSetHeight" name="assistant_impersonation" rows="2" data-i18n="[placeholder]Start Claude's answer with..." placeholder="Start Claude's answer with..."></textarea>
                                         </div>
+                                        <div id="claude_function_prefill_warning">
+                                            <i class="fa-solid fa-circle-info"></i>
+                                            <span>Prefills won't work when function calling is enabled and any tools are registered.</span>
+                                        </div>
                                         <label for="claude_use_sysprompt" class="checkbox_label widthFreeExpand">
                                             <input id="claude_use_sysprompt" type="checkbox" />
                                             <span data-i18n="Use system prompt (Claude 2.1+ only)">

                             <h4 data-i18n="Prompt Post-Processing">Prompt Post-Processing</h4>
                             <select id="custom_prompt_post_processing" class="text_pole" title="Applies additional processing to the prompt before sending it to the API." data-i18n="[title]Applies additional processing to the prompt before sending it to the API.">
                                 <option data-i18n="prompt_post_processing_none" value="">None</option>
-                                <option value="claude">Claude</option>
+                                <option value="merge">Merge consecutive roles</option>
+                                <option value="strict">Strict (user first, alternating roles)</option>
                             </select>
                         </form>
                         <div id="01ai_form" data-source="01ai">

public/jsconfig.json+5 -5 Viewed

         "allowSyntheticDefaultImports": true
     },
     "exclude": [
-        "node_modules"
+        "**/node_modules/**",
+        "**/dist/**",
+        "**/.git/**",
+        "lib/**",
+        "**/*.min.js"
     ],
     "typeAcquisition": {
         "include": [
-            "jquery",
             "@popperjs/core",
-            "toastr",
             "showdown",
-            "dompurify",
-            "moment",
             "seedrandom",
             "showdown-katex",
             "droll",

public/script.js+78 -10 Viewed

 import { AbortReason } from './scripts/util/AbortReason.js';
 import { initSystemPrompts } from './scripts/sysprompt.js';
 import { registerExtensionSlashCommands as initExtensionSlashCommands } from './scripts/extensions-slashcommands.js';
+import { ToolManager } from './scripts/tool-calling.js';
 //exporting functions and vars for mods
 export {

     FILE_ATTACHMENT_DELETED: 'file_attachment_deleted',
     WORLDINFO_FORCE_ACTIVATE: 'worldinfo_force_activate',
     OPEN_CHARACTER_LIBRARY: 'open_character_library',
-    LLM_FUNCTION_TOOL_REGISTER: 'llm_function_tool_register',
-    LLM_FUNCTION_TOOL_CALL: 'llm_function_tool_call',
     ONLINE_STATUS_CHANGED: 'online_status_changed',
     IMAGE_SWIPED: 'image_swiped',
     CONNECTION_PROFILE_LOADED: 'connection_profile_loaded',
+    TOOL_CALLS_PERFORMED: 'tool_calls_performed',
+    TOOL_CALLS_RENDERED: 'tool_calls_rendered',
 };
 export const eventSource = new EventEmitter();

     initSystemPrompts();
     initExtensions();
     initExtensionSlashCommands();
+    ToolManager.initToolSlashCommands();
     await initPresetManager();
     await getSystemMessages();
     sendSystemMessage(system_message_types.WELCOME);

                     // Return the original match if no quotes are found
                     return match;
+                }
-            }
+            },
         );
         // Restore double quotes in tags

         newMessage.addClass('smallSysMes');
+    }
+    if (Array.isArray(mes?.extra?.tool_invocations)) {
+        newMessage.addClass('toolCall');
+    }
     //shows or hides the Prompt display button
     let mesIdToFind = type === 'swipe' ? params.mesId - 1 : params.mesId;  //Number(newMessage.attr('mesId'));

         this.swipes = [];
         /** @type {import('./scripts/logprobs.js').TokenLogprobs[]} */
         this.messageLogprobs = [];
+        this.toolCalls = [];
+    }
     #checkDomElements(messageId) {

+        }
+    }
+    #updateMessageBlockVisibility() {
+        if (this.messageDom instanceof HTMLElement && Array.isArray(this.toolCalls) && this.toolCalls.length > 0) {
+            const shouldHide = ['', '...'].includes(this.result);
+            this.messageDom.classList.toggle('displayNone', shouldHide);
+        }
+    }
     showMessageButtons(messageId) {
         if (messageId == -1) {
             return;

+        }
         else {
             this.#checkDomElements(messageId);
+            this.#updateMessageBlockVisibility();
             const currentTime = new Date();
             // Don't waste time calculating token count for streaming
             const currentTokenCount = isFinal && power_user.message_token_count_enabled ? getTokenCount(processedText, 0) : 0;

+    }
     /**
-     * @returns {Generator<{ text: string, swipes: string[], logprobs: import('./scripts/logprobs.js').TokenLogprobs }, void, void>}
+     * @returns {Generator<{ text: string, swipes: string[], logprobs: import('./scripts/logprobs.js').TokenLogprobs, toolCalls: any[] }, void, void>}
      */
     *nullStreamingGeneration() {
         throw new Error('Generation function for streaming is not hooked up');

         try {
             const sw = new Stopwatch(1000 / power_user.streaming_fps);
             const timestamps = [];
-            for await (const { text, swipes, logprobs } of this.generator()) {
+            for await (const { text, swipes, logprobs, toolCalls } of this.generator()) {
                 timestamps.push(Date.now());
                 if (this.isStopped) {
                     return;
+                }
+                this.toolCalls = toolCalls;
                 this.result = text;
                 this.swipes = Array.from(swipes ?? []);
                 if (logprobs) {

  * @param {GenerateOptions} options Generation options
  * @param {boolean} dryRun Whether to actually generate a message or just assemble the prompt
  * @returns {Promise<any>} Returns a promise that resolves when the text is done generating.
- * @typedef {{automatic_trigger?: boolean, force_name2?: boolean, quiet_prompt?: string, quietToLoud?: boolean, skipWIAN?: boolean, force_chid?: number, signal?: AbortSignal, quietImage?: string, quietName?: string }} GenerateOptions
+ * @typedef {{automatic_trigger?: boolean, force_name2?: boolean, quiet_prompt?: string, quietToLoud?: boolean, skipWIAN?: boolean, force_chid?: number, signal?: AbortSignal, quietImage?: string, quietName?: string, depth?: number }} GenerateOptions
  */
-export async function Generate(type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName } = {}, dryRun = false) {
+export async function Generate(type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName, depth = 0 } = {}, dryRun = false) {
     console.log('Generate entered');
     setGenerationProgress(0);
     generation_started = new Date();

@@ -3614,7 +3630,9 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
3614	}	3630	}
3615		3631
3616	// Collect messages with usable content	3632	// Collect messages with usable content
3617	let coreChat = chat.filter(x => !x.is_system);	3633	const canUseTools = ToolManager.isToolCallingSupported();
		3634	const canPerformToolCalls = !dryRun && ToolManager.canPerformToolCalls(type) && depth < ToolManager.RECURSE_LIMIT;
		3635	let coreChat = chat.filter(x => !x.is_system \|\| (canUseTools && Array.isArray(x.extra?.tool_invocations)));
3618	if (type === 'swipe') {	3636	if (type === 'swipe') {
3619	coreChat.pop();	3637	coreChat.pop();
3620	}	3638	}

                 getMessage = continue_mag + getMessage;
+            }
-            if (streamingProcessor && !streamingProcessor.isStopped && streamingProcessor.isFinished) {
+            const isStreamFinished = streamingProcessor && !streamingProcessor.isStopped && streamingProcessor.isFinished;
+            const isStreamWithToolCalls = streamingProcessor && Array.isArray(streamingProcessor.toolCalls) && streamingProcessor.toolCalls.length;
+            if (canPerformToolCalls && isStreamFinished && isStreamWithToolCalls) {
+                const lastMessage = chat[chat.length - 1];
+                const hasToolCalls = ToolManager.hasToolCalls(streamingProcessor.toolCalls);
+                const shouldDeleteMessage = type !== 'swipe' && ['', '...'].includes(lastMessage?.mes) && ['', '...'].includes(streamingProcessor?.result);
+                hasToolCalls && shouldDeleteMessage && await deleteLastMessage();
+                const invocationResult = await ToolManager.invokeFunctionTools(streamingProcessor.toolCalls);
+                if (hasToolCalls) {
+                    if (!invocationResult.invocations.length && shouldDeleteMessage) {
+                        ToolManager.showToolCallError(invocationResult.errors);
+                        unblockGeneration(type);
+                        generatedPromptCache = '';
+                        streamingProcessor = null;
+                        return;
+                    }
+                    streamingProcessor = null;
+                    depth = depth + 1;
+                    await ToolManager.saveFunctionToolInvocations(invocationResult.invocations);
+                    return Generate('normal', { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName, depth }, dryRun);
+                }
+            }
+            if (isStreamFinished) {
                 await streamingProcessor.onFinishStreaming(streamingProcessor.messageId, getMessage);
                 streamingProcessor = null;
                 triggerAutoContinue(messageChunk, isImpersonate);

             parseAndSaveLogprobs(data, continue_mag);
+        }
+        if (canPerformToolCalls) {
+            const hasToolCalls = ToolManager.hasToolCalls(data);
+            const shouldDeleteMessage = type !== 'swipe' && ['', '...'].includes(getMessage);
+            hasToolCalls && shouldDeleteMessage && await deleteLastMessage();
+            const invocationResult = await ToolManager.invokeFunctionTools(data);
+            if (hasToolCalls) {
+                if (!invocationResult.invocations.length && shouldDeleteMessage) {
+                    ToolManager.showToolCallError(invocationResult.errors);
+                    unblockGeneration(type);
+                    generatedPromptCache = '';
+                    return;
+                }
+                depth = depth + 1;
+                await ToolManager.saveFunctionToolInvocations(invocationResult.invocations);
+                return Generate('normal', { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName, depth }, dryRun);
+            }
+        }
         if (type !== 'quiet') {
             playMessageSound();
+        }

     text = getRegexedString(
         text,
         regexPlacement,
-        { characterOverride: mes.extra?.type === 'narrator' ? undefined : mes.name },
+        {
+            characterOverride: mes.extra?.type === 'narrator' ? undefined : mes.name,
+            isEdit: true,
+        },
     );

         registerHelper: () => { },
         registerMacro: MacrosParser.registerMacro.bind(MacrosParser),
         unregisterMacro: MacrosParser.unregisterMacro.bind(MacrosParser),
+        registerFunctionTool: ToolManager.registerFunctionTool.bind(ToolManager),
+        unregisterFunctionTool: ToolManager.unregisterFunctionTool.bind(ToolManager),
+        isToolCallingSupported: ToolManager.isToolCallingSupported.bind(ToolManager),
+        canPerformToolCalls: ToolManager.canPerformToolCalls.bind(ToolManager),
         registerDebugFunction: registerDebugFunction,
         /** @deprecated Use renderExtensionTemplateAsync instead. */
         renderExtensionTemplate: renderExtensionTemplate,

public/scripts/extensions/assets/index.js+9 -0 Viewed

                         const url = isValidUrl(asset['url']) ? asset['url'] : '';
                         const title = assetType === 'extension' ? `Extension repo/guide: ${url}` : 'Preview in browser';
                         const previewIcon = (assetType === 'extension' || assetType === 'character') ? 'fa-arrow-up-right-from-square' : 'fa-headphones-simple';
+                        const toolTag = assetType === 'extension' && asset['tool'];
                         const assetBlock = $('<i></i>')
                             .append(element)

                                             <a class="asset_preview" href="${url}" target="_blank" title="${title}">
                                                 <i class="fa-solid fa-sm ${previewIcon}"></i>
                                             </a>
+                                            ${toolTag ? '<span class="tag" title="Adds a function tool"><i class="fa-solid fa-sm fa-wrench"></i> Tool</span>' : ''}
                                         </span>
                                         <small class="asset-description">
                                             ${description}
                                         </small>
                                      </div>`);
+                        assetBlock.find('.tag').on('click', function (e) {
+                            const a = document.createElement('a');
+                            a.href = 'https://docs.sillytavern.app/for-contributors/function-calling/';
+                            a.target = '_blank';
+                            a.click();
+                        });
                         if (assetType === 'character') {
                             if (asset.highlight) {
                                 assetBlock.find('.asset-name').append('<i class="fa-solid fa-sm fa-trophy"></i>');

public/scripts/extensions/assets/style.css+9 -0 Viewed

     gap: 5px;
     align-items: center;
+}
+.asset-name .tag {
+    gap: 5px;
+    align-items: baseline;
+    font-size: calc(var(--mainFontSize)* 0.8);
+    cursor: pointer;
+    opacity: 0.9;
+    margin-left: 2px;
+}

public/scripts/extensions/expressions/index.js+1 -51 Viewed

 import { SlashCommandParser } from '../../slash-commands/SlashCommandParser.js';
 import { SlashCommand } from '../../slash-commands/SlashCommand.js';
 import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from '../../slash-commands/SlashCommandArgument.js';
-import { isFunctionCallingSupported } from '../../openai.js';
 import { SlashCommandEnumValue, enumTypes } from '../../slash-commands/SlashCommandEnumValue.js';
 import { commonEnumProviders } from '../../slash-commands/SlashCommandCommonEnumsProvider.js';
 import { slashCommandReturnHelper } from '../../slash-commands/SlashCommandReturnHelper.js';

 const STREAMING_UPDATE_INTERVAL = 10000;
 const TALKINGCHECK_UPDATE_INTERVAL = 500;
 const DEFAULT_FALLBACK_EXPRESSION = 'joy';
-const FUNCTION_NAME = 'set_emotion';
 const DEFAULT_LLM_PROMPT = 'Ignore previous instructions. Classify the emotion of the last message. Output just one word, e.g. "joy" or "anger". Choose only one of the following labels: {{labels}}';
 const DEFAULT_EXPRESSIONS = [
     'talkinghead',

         return '';
+    }
-    if (isFunctionCallingSupported()) {
-        return '';
-    }
     const labelsString = labels.map(x => `"${x}"`).join(', ');
     const prompt = substituteParamsExtended(String(extension_settings.expressions.llmPrompt), { labels: labelsString });
     return prompt;

     throw new Error('Could not parse emotion response ' + emotionResponse);
+}
-/**
- * Registers the function tool for the LLM API.
- * @param {FunctionToolRegister} args Function tool register arguments.
- */
-function onFunctionToolRegister(args) {
-    if (inApiCall && extension_settings.expressions.api === EXPRESSION_API.llm && isFunctionCallingSupported()) {
-        // Only trigger on quiet mode
-        if (args.type !== 'quiet') {
-            return;
-        }
-        const emotions = DEFAULT_EXPRESSIONS.filter((e) => e != 'talkinghead');
-        const jsonSchema = {
-            $schema: 'http://json-schema.org/draft-04/schema#',
-            type: 'object',
-            properties: {
-                emotion: {
-                    type: 'string',
-                    enum: emotions,
-                    description: `One of the following: ${JSON.stringify(emotions)}`,
-                },
-            },
-            required: [
-                'emotion',
-            ],
-        };
-        args.registerFunctionTool(
-            FUNCTION_NAME,
-            substituteParams('Sets the label that best describes the current emotional state of {{char}}. Only select one of the enumerated values.'),
-            jsonSchema,
-            true,
-        );
-    }
-}
 function onTextGenSettingsReady(args) {
     // Only call if inside an API call
     if (inApiCall && extension_settings.expressions.api === EXPRESSION_API.llm && isJsonSchemaSupported()) {

                 const expressionsList = await getExpressionsList();
                 const prompt = substituteParamsExtended(customPrompt, { labels: expressionsList }) || await getLlmPrompt(expressionsList);
-                let functionResult = null;
                 eventSource.once(event_types.TEXT_COMPLETION_SETTINGS_READY, onTextGenSettingsReady);
-                eventSource.once(event_types.LLM_FUNCTION_TOOL_REGISTER, onFunctionToolRegister);
-                eventSource.once(event_types.LLM_FUNCTION_TOOL_CALL, (/** @type {FunctionToolCall} */ args) => {
-                    if (args.name !== FUNCTION_NAME) {
-                        return;
-                    }
-                    functionResult = args?.arguments;
-                });
                 const emotionResponse = await generateRaw(text, main_api, false, false, prompt);
-                return parseLlmResponse(functionResult || emotionResponse, expressionsList);
+                return parseLlmResponse(emotionResponse, expressionsList);
+            }
             // Extras
             default: {

public/scripts/extensions/regex/editor.html+1 -1 Viewed

                     <input type="checkbox" name="disabled" />
                     <span data-i18n="Disabled">Disabled</span>
                 </label>
-                <label class="checkbox flex-container">
+                <label class="checkbox flex-container" title="Run the regex script when the message belonging a to specified role(s) is edited.">
                     <input type="checkbox" name="run_on_edit" />
                     <span data-i18n="Run On Edit">Run On Edit</span>
                 </label>

public/scripts/extensions/regex/engine.js+7 -2 Viewed

  * @param {regex_placement} placement The placement of the string
  * @param {RegexParams} params The parameters to use for the regex script
  * @returns {string} The regexed string
- * @typedef {{characterOverride?: string, isMarkdown?: boolean, isPrompt?: boolean, depth?: number }} RegexParams The parameters to use for the regex script
+ * @typedef {{characterOverride?: string, isMarkdown?: boolean, isPrompt?: boolean, isEdit?: boolean, depth?: number }} RegexParams The parameters to use for the regex script
  */
-function getRegexedString(rawString, placement, { characterOverride, isMarkdown, isPrompt, depth } = {}) {
+function getRegexedString(rawString, placement, { characterOverride, isMarkdown, isPrompt, isEdit, depth } = {}) {
     // WTF have you passed me?
     if (typeof rawString !== 'string') {
         console.warn('getRegexedString: rawString is not a string. Returning empty string.');

             // Script applies to all cases when neither "only"s are true, but there's no need to do it when `isMarkdown`, the as source (chat history) should already be changed beforehand
             (!script.markdownOnly && !script.promptOnly && !isMarkdown)
         ) {
+            if (isEdit && !script.runOnEdit) {
+                console.debug(`getRegexedString: Skipping script ${script.scriptName} because it does not run on edit`);
+                return;
+            }
             // Check if the depth is within the min/max depth
             if (typeof depth === 'number' && depth >= 0) {
                 if (!isNaN(script.minDepth) && script.minDepth !== null && script.minDepth >= 0 && depth < script.minDepth) {

public/scripts/extensions/stable-diffusion/index.js+67 -4 Viewed

 import { SlashCommandEnumValue } from '../../slash-commands/SlashCommandEnumValue.js';
 import { POPUP_RESULT, POPUP_TYPE, Popup, callGenericPopup } from '../../popup.js';
 import { commonEnumProviders } from '../../slash-commands/SlashCommandCommonEnumsProvider.js';
+import { ToolManager } from '../../tool-calling.js';
 export { MODULE_NAME };
 const MODULE_NAME = 'sd';

     interactive: 'interactive',
     wand: 'wand',
     swipe: 'swipe',
+    tool: 'tool',
 };
 const generationMode = {
+    TOOL: -2,
     MESSAGE: -1,
     CHARACTER: 0,
     USER: 1,

 };
 const modeLabels = {
+    [generationMode.TOOL]: 'Function Tool Prompt Description',
     [generationMode.MESSAGE]: 'Chat Message Template',
     [generationMode.CHARACTER]: 'Character ("Yourself")',
     [generationMode.FACE]: 'Portrait ("Your Face")',

 };
 const promptTemplates = {
-    // Not really a prompt template, rather an outcome message template
+    // Not really a prompt template, rather an outcome message template and function tool prompt
     [generationMode.MESSAGE]: '[{{char}} sends a picture that contains: {{prompt}}].',
+    [generationMode.TOOL]: [
+        'The text prompt used to generate the image.',
+        'Must represent an exhaustive description of the desired image that will allow an artist or a photographer to perfectly recreate it.',
+    ].join(' '),
     [generationMode.CHARACTER]: 'In the next response I want you to provide only a detailed comma-delimited list of keywords and phrases which describe {{char}}. The list must include all of the following items in this order: name, species and race, gender, age, clothing, occupation, physical features and appearances. Do not include descriptions of non-visual qualities such as personality, movements, scents, mental traits, or anything which could not be seen in a still photograph. Do not write in full sentences. Prefix your description with the phrase \'full body portrait,\'',
     //face-specific prompt
     [generationMode.FACE]: 'In the next response I want you to provide only a detailed comma-delimited list of keywords and phrases which describe {{char}}. The list must include all of the following items in this order: name, species and race, gender, age, facial features and expressions, occupation, hair and hair accessories (if any), what they are wearing on their upper body (if anything). Do not describe anything below their neck. Do not include descriptions of non-visual qualities such as personality, movements, scents, mental traits, or anything which could not be seen in a still photograph. Do not write in full sentences. Prefix your description with the phrase \'close up facial portrait,\'',

     multimodal_captioning: false,
     snap: false,
     free_extend: false,
+    function_tool: false,
     prompts: promptTemplates,

 const writePromptFieldsDebounced = debounce(writePromptFields, debounce_timeout.relaxed);
 function processTriggers(chat, _, abort) {
+    if (extension_settings.sd.function_tool && ToolManager.isToolCallingSupported()) {
+        return;
+    }
     if (!extension_settings.sd.interactive_mode) {
         return;
+    }

     $('#sd_interactive_visible').prop('checked', extension_settings.sd.interactive_visible);
     $('#sd_stability_style_preset').val(extension_settings.sd.stability_style_preset);
     $('#sd_huggingface_model_id').val(extension_settings.sd.huggingface_model_id);
+    $('#sd_function_tool').prop('checked', extension_settings.sd.function_tool);
     for (const style of extension_settings.sd.styles) {
         const option = document.createElement('option');

     toggleSourceControls();
     addPromptTemplates();
+    registerFunctionTool();
     await loadSettingOptions();
+}

             .on('click', () => {
                 textarea.val(promptTemplates[name]);
                 extension_settings.sd.prompts[name] = promptTemplates[name];
+                if (String(name) === String(generationMode.TOOL)) {
+                    registerFunctionTool();
+                }
                 saveSettingsDebounced();
             });
         const container = $('<div></div>')

     await loadSettingOptions();
+}
+function onFunctionToolInput() {
+    extension_settings.sd.function_tool = !!$(this).prop('checked');
+    saveSettingsDebounced();
+    registerFunctionTool();
+}
 async function onOpenAiStyleSelect() {
     extension_settings.sd.openai_style = String($('#sd_openai_style').find(':selected').val());
     saveSettingsDebounced();

@@ -2290,9 +2314,9 @@ async function generatePicture(initiator, args, trigger, message, callback) {
2290	eventSource.emit(event_types.FORCE_SET_BACKGROUND, { url: imgUrl, path: imagePath });	2314	eventSource.emit(event_types.FORCE_SET_BACKGROUND, { url: imgUrl, path: imagePath });
2291		2315
2292	if (typeof callbackOriginal === 'function') {	2316	if (typeof callbackOriginal === 'function') {
2293	callbackOriginal(prompt, imagePath, generationType, negativePromptPrefix, initiator);	2317	await callbackOriginal(prompt, imagePath, generationType, negativePromptPrefix, initiator);
2294	} else {	2318	} else {
2295	sendMessage(prompt, imagePath, generationType, negativePromptPrefix, initiator);	2319	await sendMessage(prompt, imagePath, generationType, negativePromptPrefix, initiator);
2296	}	2320	}
2297	};	2321	};
2298	}	2322	}

     const filename = `${characterName}_${humanizedDateTime()}`;
     const base64Image = await saveBase64AsFile(result.data, characterName, filename, result.format);
-    callback ? callback(prompt, base64Image, generationType, additionalNegativePrefix, initiator) : sendMessage(prompt, base64Image, generationType, additionalNegativePrefix, initiator);
+    callback
+        ? await callback(prompt, base64Image, generationType, additionalNegativePrefix, initiator)
+        : await sendMessage(prompt, base64Image, generationType, additionalNegativePrefix, initiator);
     return base64Image;
+}

     return currentSettings;
+}
+function registerFunctionTool() {
+    if (!extension_settings.sd.function_tool) {
+        return ToolManager.unregisterFunctionTool('GenerateImage');
+    }
+    ToolManager.registerFunctionTool({
+        name: 'GenerateImage',
+        displayName: 'Generate Image',
+        description: [
+            'Generate an image from a given text prompt.',
+            'Use when a user asks for an image, a selfie, to picture a scene, etc.',
+        ].join(' '),
+        parameters: Object.freeze({
+            $schema: 'http://json-schema.org/draft-04/schema#',
+            type: 'object',
+            properties: {
+                prompt: {
+                    type: 'string',
+                    description: extension_settings.sd.prompts[generationMode.TOOL] || promptTemplates[generationMode.TOOL],
+                },
+            },
+            required: [
+                'prompt',
+            ],
+        }),
+        action: async (args) => {
+            if (!isValidState()) throw new Error('Image generation is not configured.');
+            if (!args) throw new Error('Missing arguments');
+            if (!args.prompt) throw new Error('Missing prompt');
+            const url = await generatePicture(initiators.tool, {}, args.prompt);
+            return encodeURI(url);
+        },
+        formatMessage: () => 'Generating an image...',
+    });
+}
 jQuery(async () => {
     await addSDGenButtons();

     $('#sd_stability_key').on('click', onStabilityKeyClick);
     $('#sd_stability_style_preset').on('change', onStabilityStylePresetChange);
     $('#sd_huggingface_model_id').on('input', onHFModelInput);
+    $('#sd_function_tool').on('input', onFunctionToolInput);
     if (!CSS.supports('field-sizing', 'content')) {
         $('.sd_settings .inline-drawer-toggle').on('click', function () {

public/scripts/extensions/stable-diffusion/settings.html+5 -1 Viewed

                 <input id="sd_refine_mode" type="checkbox" />
                 <span data-i18n="sd_refine_mode_txt">Edit prompts before generation</span>
             </label>
+            <label for="sd_function_tool" class="checkbox_label" data-i18n="[title]sd_function_tool" title="Use the function tool to automatically detect intents to generate images.">
+                <input id="sd_function_tool" type="checkbox" />
+                <span data-i18n="sd_function_tool_txt">Use function tool</span>
+            </label>
             <label for="sd_interactive_mode" class="checkbox_label" data-i18n="[title]sd_interactive_mode" title="Automatically generate images when sending messages like 'send me a picture of cat'.">
                 <input id="sd_interactive_mode" type="checkbox" />
-                <span data-i18n="sd_interactive_mode_txt">Interactive mode</span>
+                <span data-i18n="sd_interactive_mode_txt">Use interactive mode</span>
             </label>
             <label for="sd_multimodal_captioning" class="checkbox_label" data-i18n="[title]sd_multimodal_captioning" title="Use multimodal captioning to generate prompts for user and character portraits based on their avatars.">
                 <input id="sd_multimodal_captioning" type="checkbox" />

public/scripts/group-chats.js+1 -1 Viewed

     animation_duration,
     depth_prompt_role_default,
     shouldAutoContinue,
-    this_chid,
 } from '../script.js';
 import { printTagList, createTagMapFromList, applyTagsOnCharacterSelect, tag_map, applyTagsOnGroupSelect } from './tags.js';
 import { FILTER_TYPES, FilterHelper } from './filters.js';
 import { isExternalMediaAllowed } from './chats.js';
 import { POPUP_TYPE, Popup, callGenericPopup } from './popup.js';
+import { t } from './i18n.js';
 export {
     selected_group,

public/scripts/instruct-mode.js+0 -2 Viewed

  * @param {string} name Preset name.
  */
 function selectMatchingContextTemplate(name) {
-    let foundMatch = false;
     for (const context_preset of context_presets) {
         // If context template matches the instruct preset
         if (context_preset.name === name) {
-            foundMatch = true;
             selectContextPreset(context_preset.name, { isAuto: true });
             break;
+        }

public/scripts/kai-settings.js+1 -1 Viewed

@@ -188,7 +188,7 @@ export async function generateKoboldWithStreaming(generate_data, signal) {
188	if (data?.token) {	188	if (data?.token) {
189	text += data.token;	189	text += data.token;
190	}	190	}
191	yield { text, swipes: [] };	191	yield { text, swipes: [], toolCalls: [] };
192	}	192	}
193	};	193	};
194	}	194	}

public/scripts/nai-settings.js+1 -1 Viewed

@@ -746,7 +746,7 @@ export async function generateNovelWithStreaming(generate_data, signal) {
746	text += data.token;	746	text += data.token;
747	}	747	}
748		748
749	yield { text, swipes: [], logprobs: parseNovelAILogprobs(data.logprobs) };	749	yield { text, swipes: [], logprobs: parseNovelAILogprobs(data.logprobs), toolCalls: [] };
750	}	750	}
751	};	751	};
752	}	752	}

public/scripts/openai.js+82 -149 Viewed

 import { SlashCommandEnumValue } from './slash-commands/SlashCommandEnumValue.js';
 import { Popup, POPUP_RESULT } from './popup.js';
 import { t } from './i18n.js';
+import { ToolManager } from './tool-calling.js';
 export {
     openai_messages_count,

 const custom_prompt_post_processing_types = {
     NONE: '',
+    /** @deprecated Use MERGE instead. */
     CLAUDE: 'claude',
+    MERGE: 'merge',
+    STRICT: 'strict',
 };
 const sensitiveFields = [

         if (role == 'user' && oai_settings.wrap_in_quotes) content = `"${content}"`;
         const name = chat[j]['name'];
         const image = chat[j]?.extra?.image;
-        messages[i] = { 'role': role, 'content': content, name: name, 'image': image };
+        const invocations = chat[j]?.extra?.tool_invocations;
+        messages[i] = { 'role': role, 'content': content, name: name, 'image': image, 'invocations': invocations };
         j++;
+    }

+    }
     const imageInlining = isImageInliningSupported();
+    const canUseTools = ToolManager.isToolCallingSupported();
     // Insert chat messages as long as there is budget available
     const chatPool = [...messages].reverse();

             await chatMessage.addImage(chatPrompt.image);
+        }
+        if (canUseTools && Array.isArray(chatPrompt.invocations)) {
+            /** @type {import('./tool-calling.js').ToolInvocation[]} */
+            const invocations = chatPrompt.invocations;
+            const toolCallMessage = new Message(chatMessage.role, undefined, 'toolCall-' + chatMessage.identifier);
+            const toolResultMessages = invocations.slice().reverse().map((invocation) => new Message('tool', invocation.result || '[No content]', invocation.id));
+            toolCallMessage.setToolCalls(invocations);
+            if (chatCompletion.canAffordAll([toolCallMessage, ...toolResultMessages])) {
+                for (const resultMessage of toolResultMessages) {
+                    chatCompletion.insertAtStart(resultMessage, 'chatHistory');
+                }
+                chatCompletion.insertAtStart(toolCallMessage, 'chatHistory');
+            } else {
+                break;
+            }
+            continue;
+        }
         if (chatCompletion.canAfford(chatMessage)) {
             if (type === 'continue' && oai_settings.continue_prefill && chatPrompt === firstNonInjected) {
                 // in case we are using continue_prefill and the latest message is an assistant message, we want to prepend the users assistant prefill on the message

     const eventData = { chat, dryRun };
     await eventSource.emit(event_types.CHAT_COMPLETION_PROMPT_READY, eventData);
-    openai_messages_count = chat.filter(x => x?.role === 'user' || x?.role === 'assistant')?.length || 0;
+    openai_messages_count = chat.filter(x => !x?.tool_calls && (x?.role === 'user' || x?.role === 'assistant'))?.length || 0;
     return [chat, promptManager.tokenHandler.counts];
+}

     messages = messages.filter(msg => msg && typeof msg === 'object');
     let logit_bias = {};
-    const messageId = getNextMessageId(type);
     const isClaude = oai_settings.chat_completion_source == chat_completion_sources.CLAUDE;
     const isOpenRouter = oai_settings.chat_completion_source == chat_completion_sources.OPENROUTER;
     const isScale = oai_settings.chat_completion_source == chat_completion_sources.SCALE;

@@ -1860,8 +1883,8 @@ async function sendOpenAIRequest(type, messages, signal) {
1860	generate_data['seed'] = oai_settings.seed;	1883	generate_data['seed'] = oai_settings.seed;
1861	}	1884	}
1862		1885
1863	if (isFunctionCallingSupported() && !stream) {	1886	if (!canMultiSwipe && ToolManager.canPerformToolCalls(type)) {
1864	await registerFunctionTools(type, generate_data);	1887	await ToolManager.registerFunctionToolsOpenAI(generate_data);
1865	}	1888	}
1866		1889
1867	if (isOAI && oai_settings.openai_model.startsWith('o1-')) {	1890	if (isOAI && oai_settings.openai_model.startsWith('o1-')) {

         return async function* streamData() {
             let text = '';
             const swipes = [];
+            const toolCalls = [];
             while (true) {
                 const { done, value } = await reader.read();
                 if (done) return;

@@ -1923,7 +1947,9 @@ async function sendOpenAIRequest(type, messages, signal) {
1923	text += getStreamingReply(parsed);	1947	text += getStreamingReply(parsed);
1924	}	1948	}
1925		1949
1926	yield { text, swipes: swipes, logprobs: parseChatCompletionLogprobs(parsed) };	1950	ToolManager.parseToolCalls(toolCalls, parsed);
		1951
		1952	yield { text, swipes: swipes, logprobs: parseChatCompletionLogprobs(parsed), toolCalls: toolCalls };
1927	}	1953	}
1928	};	1954	};
1929	}	1955	}

             delay(1).then(() => saveLogprobsForActiveMessage(logprobs, null));
+        }
-        if (isFunctionCallingSupported()) {
-            await checkFunctionToolCalls(data);
-        }
         return data;
+    }
+}
-/**
- * Register function tools for the next chat completion request.
- * @param {string} type Generation type
- * @param {object} data Generation data
- */
-async function registerFunctionTools(type, data) {
-    let toolChoice = 'auto';
-    const tools = [];
-    /**
-     * @type {registerFunctionTool}
-     */
-    const registerFunctionTool = (name, description, parameters, required) => {
-        tools.push({
-            type: 'function',
-            function: {
-                name,
-                description,
-                parameters,
-            },
-        });
-        if (required) {
-            toolChoice = 'required';
-        }
-    };
-    /**
-     * @type {FunctionToolRegister}
-     */
-    const args = {
-        type,
-        data,
-        registerFunctionTool,
-    };
-    await eventSource.emit(event_types.LLM_FUNCTION_TOOL_REGISTER, args);
-    if (tools.length) {
-        console.log('Registered function tools:', tools);
-        data['tools'] = tools;
-        data['tool_choice'] = toolChoice;
-    }
-}
-async function checkFunctionToolCalls(data) {
-    const oaiCompat = [
-        chat_completion_sources.OPENAI,
-        chat_completion_sources.CUSTOM,
-        chat_completion_sources.MISTRALAI,
-        chat_completion_sources.OPENROUTER,
-        chat_completion_sources.GROQ,
-    ];
-    if (oaiCompat.includes(oai_settings.chat_completion_source)) {
-        if (!Array.isArray(data?.choices)) {
-            return;
-        }
-        // Find a choice with 0-index
-        const choice = data.choices.find(choice => choice.index === 0);
-        if (!choice) {
-            return;
-        }
-        const toolCalls = choice.message.tool_calls;
-        if (!Array.isArray(toolCalls)) {
-            return;
-        }
-        for (const toolCall of toolCalls) {
-            if (typeof toolCall.function !== 'object') {
-                continue;
-            }
-            /** @type {FunctionToolCall} */
-            const args = toolCall.function;
-            console.log('Function tool call:', toolCall);
-            await eventSource.emit(event_types.LLM_FUNCTION_TOOL_CALL, args);
-        }
-    }
-    if ([chat_completion_sources.CLAUDE].includes(oai_settings.chat_completion_source)) {
-        if (!Array.isArray(data?.content)) {
-            return;
-        }
-        for (const content of data.content) {
-            if (content.type === 'tool_use') {
-                /** @type {FunctionToolCall} */
-                const args = { name: content.name, arguments: JSON.stringify(content.input) };
-                await eventSource.emit(event_types.LLM_FUNCTION_TOOL_CALL, args);
-            }
-        }
-    }
-    if ([chat_completion_sources.COHERE].includes(oai_settings.chat_completion_source)) {
-        if (!Array.isArray(data?.tool_calls)) {
-            return;
-        }
-        for (const toolCall of data.tool_calls) {
-            /** @type {FunctionToolCall} */
-            const args = { name: toolCall.name, arguments: JSON.stringify(toolCall.parameters) };
-            console.log('Function tool call:', toolCall);
-            await eventSource.emit(event_types.LLM_FUNCTION_TOOL_CALL, args);
-        }
-    }
-}
-export function isFunctionCallingSupported() {
-    if (main_api !== 'openai') {
-        return false;
-    }
-    if (!oai_settings.function_calling) {
-        return false;
-    }
-    const supportedSources = [
-        chat_completion_sources.OPENAI,
-        chat_completion_sources.COHERE,
-        chat_completion_sources.CUSTOM,
-        chat_completion_sources.MISTRALAI,
-        chat_completion_sources.CLAUDE,
-        chat_completion_sources.OPENROUTER,
-        chat_completion_sources.GROQ,
-    ];
-    return supportedSources.includes(oai_settings.chat_completion_source);
-}
 function getStreamingReply(data) {
     if (oai_settings.chat_completion_source == chat_completion_sources.CLAUDE) {
         return data?.delta?.text || '';

     content;
     /** @type {string} */
     name;
+    /** @type {object} */
+    tool_call = null;
     /**
      * @constructor

+        }
+    }
+    /**
+     * Reconstruct the message from a tool invocation.
+     * @param {import('./tool-calling.js').ToolInvocation[]} invocations
+     */
+    setToolCalls(invocations) {
+        this.tool_calls = invocations.map(i => ({
+            id: i.id,
+            type: 'function',
+            function: {
+                arguments: i.parameters,
+                name: i.name,
+            },
+        }));
+        this.tokens = tokenHandler.count({ role: this.role, tool_calls: JSON.stringify(this.tool_calls) });
+    }
     setName(name) {
         this.name = name;
         this.tokens = tokenHandler.count({ role: this.role, content: this.content, name: this.name });

+    }
     /**
-     * Get chat in the format of {role, name, content}.
+     * Get chat in the format of {role, name, content, tool_calls}.
      * @returns {Array} Array of objects with role, name, and content properties.
      */
     getChat() {
         return this.collection.reduce((acc, message) => {
-            const name = message.name;
+            if (message.content || message.tool_calls) {
-            if (message.content) acc.push({ role: message.role, ...(name && { name }), content: message.content });
+                acc.push({
+                    role: message.role,
+                    content: message.content,
+                    ...(message.name && { name: message.name }),
+                    ...(message.tool_calls && { tool_calls: message.tool_calls }),
+                    ...(message.role === 'tool' && { tool_call_id: message.identifier }),
+                });
+            }
             return acc;
         }, []);
+    }

         this.checkTokenBudget(message, message.identifier);
         const index = this.findMessageIndex(identifier);
-        if (message.content) {
+        if (message.content || message.tool_calls) {
             if ('start' === position) this.messages.collection[index].collection.unshift(message);
             else if ('end' === position) this.messages.collection[index].collection.push(message);
             else if (typeof position === 'number') this.messages.collection[index].collection.splice(position, 0, message);

+    }
     /**
+     * Checks if the token budget can afford the tokens of all the specified messages.
+     * @param {Message[]} messages - The messages to check for affordability.
+     * @returns {boolean} True if the budget can afford all the messages, false otherwise.
+     */
+    canAffordAll(messages) {
+        return 0 <= this.tokenBudget - messages.reduce((total, message) => total + message.getTokens(), 0);
+    }
+    /**
      * Checks if a message with the specified identifier exists in the collection.
+     *
      * @param {string} identifier - The identifier to check for existence.

         for (let item of this.messages.collection) {
             if (item instanceof MessageCollection) {
                 chat.push(...item.getChat());
-            } else if (item instanceof Message && item.content) {
+            } else if (item instanceof Message && (item.content || item.tool_calls)) {
-                const message = { role: item.role, content: item.content, ...(item.name ? { name: item.name } : {}) };
+                const message = {
+                    role: item.role,
+                    content: item.content,
+                    ...(item.name ? { name: item.name } : {}),
+                    ...(item.tool_calls ? { tool_calls: item.tool_calls } : {}),
+                    ...(item.role === 'tool' ? { tool_call_id: item.identifier } : {}),
+                };
                 chat.push(message);
             } else {
                 this.log(`Skipping invalid or empty message in collection: ${JSON.stringify(item)}`);

     setNamesBehaviorControls();
     setContinuePostfixControls();
+    if (oai_settings.custom_prompt_post_processing === custom_prompt_post_processing_types.CLAUDE) {
+        oai_settings.custom_prompt_post_processing = custom_prompt_post_processing_types.MERGE;
+    }
     $('#chat_completion_source').val(oai_settings.chat_completion_source).trigger('change');
     $('#oai_max_context_unlocked').prop('checked', oai_settings.max_context_unlocked);
     $('#custom_prompt_post_processing').val(oai_settings.custom_prompt_post_processing);

public/scripts/slash-commands/SlashCommandReturnHelper.js+1 -1 Viewed

             case 'toast-html': {
                 const htmlOrNotHtml = shouldHtml ? DOMPurify.sanitize((new showdown.Converter()).makeHtml(stringValue)) : escapeHtml(stringValue);
-                if (type.startsWith('popup')) await callGenericPopup(htmlOrNotHtml, POPUP_TYPE.TEXT);
+                if (type.startsWith('popup')) await callGenericPopup(htmlOrNotHtml, POPUP_TYPE.TEXT, '', { allowVerticalScrolling: true, wide: true });
                 if (type.startsWith('chat')) sendSystemMessage(system_message_types.GENERIC, htmlOrNotHtml);
                 if (type.startsWith('toast')) toastr.info(htmlOrNotHtml, null, { escapeHtml: !shouldHtml });

public/scripts/textgen-settings.js+2 -1 Viewed

         /** @type {import('./logprobs.js').TokenLogprobs | null} */
         let logprobs = null;
         const swipes = [];
+        const toolCalls = [];
         while (true) {
             const { done, value } = await reader.read();
             if (done) return;

@@ -934,7 +935,7 @@ async function generateTextGenWithStreaming(generate_data, signal) {
934	logprobs = parseTextgenLogprobs(newText, data.choices?.[0]?.logprobs \|\| data?.completion_probabilities);	935	logprobs = parseTextgenLogprobs(newText, data.choices?.[0]?.logprobs \|\| data?.completion_probabilities);
935	}	936	}
936		937
937	yield { text, swipes, logprobs };	938	yield { text, swipes, logprobs, toolCalls };
938	}	939	}
939	};	940	};
940	}	941	}

public/scripts/tool-calling.js+891 -0 Viewed

+import { addOneMessage, chat, event_types, eventSource, main_api, saveChatConditional, system_avatar, systemUserName } from '../script.js';
+import { chat_completion_sources, oai_settings } from './openai.js';
+import { Popup } from './popup.js';
+import { SlashCommand } from './slash-commands/SlashCommand.js';
+import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from './slash-commands/SlashCommandArgument.js';
+import { SlashCommandClosure } from './slash-commands/SlashCommandClosure.js';
+import { enumIcons } from './slash-commands/SlashCommandCommonEnumsProvider.js';
+import { enumTypes, SlashCommandEnumValue } from './slash-commands/SlashCommandEnumValue.js';
+import { SlashCommandParser } from './slash-commands/SlashCommandParser.js';
+import { slashCommandReturnHelper } from './slash-commands/SlashCommandReturnHelper.js';
+/**
+ * @typedef {object} ToolInvocation
+ * @property {string} id - A unique identifier for the tool invocation.
+ * @property {string} displayName - The display name of the tool.
+ * @property {string} name - The name of the tool.
+ * @property {string} parameters - The parameters for the tool invocation.
+ * @property {string} result - The result of the tool invocation.
+ */
+/**
+ * @typedef {object} ToolInvocationResult
+ * @property {ToolInvocation[]} invocations Successful tool invocations
+ * @property {Error[]} errors Errors that occurred during tool invocation
+ */
+/**
+ * @typedef {object} ToolRegistration
+ * @property {string} name - The name of the tool.
+ * @property {string} displayName - The display name of the tool.
+ * @property {string} description - A description of the tool.
+ * @property {object} parameters - The parameters for the tool.
+ * @property {function} action - The action to perform when the tool is invoked.
+ * @property {function} formatMessage - A function to format the tool call message.
+ */
+/**
+ * @typedef {object} ToolDefinitionOpenAI
+ * @property {string} type - The type of the tool.
+ * @property {object} function - The function definition.
+ * @property {string} function.name - The name of the function.
+ * @property {string} function.description - The description of the function.
+ * @property {object} function.parameters - The parameters of the function.
+ * @property {function} toString - A function to convert the tool to a string.
+ */
+/**
+ * Assigns nested variables to a scope.
+ * @param {import('./slash-commands/SlashCommandScope.js').SlashCommandScope} scope The scope to assign variables to.
+ * @param {object} arg Object to assign variables from.
+ * @param {string} prefix Prefix for the variable names.
+ */
+function assignNestedVariables(scope, arg, prefix) {
+    Object.entries(arg).forEach(([key, value]) => {
+        const newPrefix = `${prefix}.${key}`;
+        if (typeof value === 'object' && value !== null) {
+            assignNestedVariables(scope, value, newPrefix);
+        } else {
+            scope.letVariable(newPrefix, value);
+        }
+    });
+}
+/**
+ * Checks if a string is a valid JSON string.
+ * @param {string} str The string to check
+ * @returns {boolean} If the string is a valid JSON string
+ */
+function isJson(str) {
+    try {
+        JSON.parse(str);
+        return true;
+    } catch {
+        return false;
+    }
+}
+/**
+ * Tries to parse a string as JSON, returning the original string if parsing fails.
+ * @param {string} str The string to try to parse
+ * @returns {object|string} Parsed JSON or the original string
+ */
+function tryParse(str) {
+    try {
+        return JSON.parse(str);
+    } catch {
+        return str;
+    }
+}
+/**
+ * Stringifies an object if it is not already a string.
+ * @param {any} obj The object to stringify
+ * @returns {string} A JSON string representation of the object.
+ */
+function stringify(obj) {
+    return typeof obj === 'string' ? obj : JSON.stringify(obj);
+}
+/**
+ * A class that represents a tool definition.
+ */
+class ToolDefinition {
+    /**
+     * A unique name for the tool.
+     * @type {string}
+     */
+    #name;
+    /**
+     * A user-friendly display name for the tool.
+     * @type {string}
+     */
+    #displayName;
+    /**
+     * A description of what the tool does.
+     * @type {string}
+     */
+    #description;
+    /**
+     * A JSON schema for the parameters that the tool accepts.
+     * @type {object}
+     */
+    #parameters;
+    /**
+     * A function that will be called when the tool is executed.
+     * @type {function}
+     */
+    #action;
+    /**
+     * A function that will be called to format the tool call toast.
+     * @type {function}
+     */
+    #formatMessage;
+    /**
+     * Creates a new ToolDefinition.
+     * @param {string} name A unique name for the tool.
+     * @param {string} displayName A user-friendly display name for the tool.
+     * @param {string} description A description of what the tool does.
+     * @param {object} parameters A JSON schema for the parameters that the tool accepts.
+     * @param {function} action A function that will be called when the tool is executed.
+     * @param {function} formatMessage A function that will be called to format the tool call toast.
+     */
+    constructor(name, displayName, description, parameters, action, formatMessage) {
+        this.#name = name;
+        this.#displayName = displayName;
+        this.#description = description;
+        this.#parameters = parameters;
+        this.#action = action;
+        this.#formatMessage = formatMessage;
+    }
+    /**
+     * Converts the ToolDefinition to an OpenAI API representation
+     * @returns {ToolDefinitionOpenAI} OpenAI API representation of the tool.
+     */
+    toFunctionOpenAI() {
+        return {
+            type: 'function',
+            function: {
+                name: this.#name,
+                description: this.#description,
+                parameters: this.#parameters,
+            },
+            toString: function () {
+                return `<div><b>${this.function.name}</b></div><div><small>${this.function.description}</small></div><pre class="justifyLeft wordBreakAll"><code class="flex padding5">${JSON.stringify(this.function.parameters, null, 2)}</code></pre><hr>`;
+            },
+        };
+    }
+    /**
+     * Invokes the tool with the given parameters.
+     * @param {object} parameters The parameters to pass to the tool.
+     * @returns {Promise<any>} The result of the tool's action function.
+     */
+    async invoke(parameters) {
+        return await this.#action(parameters);
+    }
+    /**
+     * Formats a message with the tool invocation.
+     * @param {object} parameters The parameters to pass to the tool.
+     * @returns {string} The formatted message.
+     */
+    formatMessage(parameters) {
+        return typeof this.#formatMessage === 'function'
+            ? this.#formatMessage(parameters)
+            : `Invoking tool: ${this.#displayName || this.#name}`;
+    }
+    get displayName() {
+        return this.#displayName;
+    }
+}
+/**
+ * A class that manages the registration and invocation of tools.
+ */
+export class ToolManager {
+    /**
+     * A map of tool names to tool definitions.
+     * @type {Map<string, ToolDefinition>}
+     */
+    static #tools = new Map();
+    static #INPUT_DELTA_KEY = '__input_json_delta';
+    /**
+     * The maximum number of times to recurse when parsing tool calls.
+     * @type {number}
+     */
+    static RECURSE_LIMIT = 5;
+    /**
+     * Returns an Array of all tools that have been registered.
+     * @type {ToolDefinition[]}
+     */
+    static get tools() {
+        return Array.from(this.#tools.values());
+    }
+    /**
+     * Registers a new tool with the tool registry.
+     * @param {ToolRegistration} tool The tool to register.
+     */
+    static registerFunctionTool({ name, displayName, description, parameters, action, formatMessage }) {
+        // Convert WIP arguments
+        if (typeof arguments[0] !== 'object') {
+            [name, description, parameters, action] = arguments;
+        }
+        if (this.#tools.has(name)) {
+            console.warn(`A tool with the name "${name}" has already been registered. The definition will be overwritten.`);
+        }
+        const definition = new ToolDefinition(name, displayName, description, parameters, action, formatMessage);
+        this.#tools.set(name, definition);
+        console.log('[ToolManager] Registered function tool:', definition);
+    }
+    /**
+     * Removes a tool from the tool registry.
+     * @param {string} name The name of the tool to unregister.
+     */
+    static unregisterFunctionTool(name) {
+        if (!this.#tools.has(name)) {
+            return;
+        }
+        this.#tools.delete(name);
+        console.log(`[ToolManager] Unregistered function tool: ${name}`);
+    }
+    /**
+     * Invokes a tool by name. Returns the result of the tool's action function.
+     * @param {string} name The name of the tool to invoke.
+     * @param {object} parameters Function parameters. For example, if the tool requires a "name" parameter, you would pass {name: "value"}.
+     * @returns {Promise<string|Error>} The result of the tool's action function. If an error occurs, null is returned. Non-string results are JSON-stringified.
+     */
+    static async invokeFunctionTool(name, parameters) {
+        try {
+            if (!this.#tools.has(name)) {
+                throw new Error(`No tool with the name "${name}" has been registered.`);
+            }
+            const invokeParameters = typeof parameters === 'string' ? JSON.parse(parameters) : parameters;
+            const tool = this.#tools.get(name);
+            const result = await tool.invoke(invokeParameters);
+            return typeof result === 'string' ? result : JSON.stringify(result);
+        } catch (error) {
+            console.error(`An error occurred while invoking the tool "${name}":`, error);
+            if (error instanceof Error) {
+                error.cause = name;
+                return error;
+            }
+            return new Error('Unknown error occurred while invoking the tool.', { cause: name });
+        }
+    }
+    /**
+     * Formats a message for a tool call by name.
+     * @param {string} name The name of the tool to format the message for.
+     * @param {object} parameters Function tool call parameters.
+     * @returns {string} The formatted message for the tool call.
+     */
+    static formatToolCallMessage(name, parameters) {
+        if (!this.#tools.has(name)) {
+            return `Invoked unknown tool: ${name}`;
+        }
+        try {
+            const tool = this.#tools.get(name);
+            const formatParameters = typeof parameters === 'string' ? JSON.parse(parameters) : parameters;
+            return tool.formatMessage(formatParameters);
+        } catch (error) {
+            console.error(`An error occurred while formatting the tool call message for "${name}":`, error);
+            return `Invoking tool: ${name}`;
+        }
+    }
+    /**
+     * Gets the display name of a tool by name.
+     * @param {string} name
+     * @returns {string} The display name of the tool.
+     */
+    static getDisplayName(name) {
+        if (!this.#tools.has(name)) {
+            return name;
+        }
+        const tool = this.#tools.get(name);
+        return tool.displayName || name;
+    }
+    /**
+     * Register function tools for the next chat completion request.
+     * @param {object} data Generation data
+     */
+    static async registerFunctionToolsOpenAI(data) {
+        const tools = [];
+        for (const tool of ToolManager.tools) {
+            tools.push(tool.toFunctionOpenAI());
+        }
+        if (tools.length) {
+            console.log('Registered function tools:', tools);
+            data['tools'] = tools;
+            data['tool_choice'] = 'auto';
+        }
+    }
+    /**
+     * Utility function to parse tool calls from a parsed response.
+     * @param {any[]} toolCalls The tool calls to update.
+     * @param {any} parsed The parsed response from the OpenAI API.
+     * @returns {void}
+     */
+    static parseToolCalls(toolCalls, parsed) {
+        if (Array.isArray(parsed?.choices)) {
+            for (const choice of parsed.choices) {
+                const choiceIndex = (typeof choice.index === 'number') ? choice.index : null;
+                const choiceDelta = choice.delta;
+                if (choiceIndex === null || !choiceDelta) {
+                    continue;
+                }
+                const toolCallDeltas = choiceDelta?.tool_calls;
+                if (!Array.isArray(toolCallDeltas)) {
+                    continue;
+                }
+                if (!Array.isArray(toolCalls[choiceIndex])) {
+                    toolCalls[choiceIndex] = [];
+                }
+                for (const toolCallDelta of toolCallDeltas) {
+                    const toolCallIndex = (typeof toolCallDelta?.index === 'number') ? toolCallDelta.index : toolCallDeltas.indexOf(toolCallDelta);
+                    if (isNaN(toolCallIndex) || toolCallIndex < 0) {
+                        continue;
+                    }
+                    if (toolCalls[choiceIndex][toolCallIndex] === undefined) {
+                        toolCalls[choiceIndex][toolCallIndex] = {};
+                    }
+                    const targetToolCall = toolCalls[choiceIndex][toolCallIndex];
+                    ToolManager.#applyToolCallDelta(targetToolCall, toolCallDelta);
+                }
+            }
+        }
+        if (typeof parsed?.content_block === 'object') {
+            const choiceIndex = 0;
+            const toolCallIndex = parsed?.index ?? 0;
+            if (parsed?.content_block?.type === 'tool_use') {
+                if (!Array.isArray(toolCalls[choiceIndex])) {
+                    toolCalls[choiceIndex] = [];
+                }
+                if (toolCalls[choiceIndex][toolCallIndex] === undefined) {
+                    toolCalls[choiceIndex][toolCallIndex] = {};
+                }
+                const targetToolCall = toolCalls[choiceIndex][toolCallIndex];
+                ToolManager.#applyToolCallDelta(targetToolCall, parsed.content_block);
+            }
+        }
+        if (typeof parsed?.delta === 'object') {
+            const choiceIndex = 0;
+            const toolCallIndex = parsed?.index ?? 0;
+            const targetToolCall = toolCalls[choiceIndex]?.[toolCallIndex];
+            if (targetToolCall) {
+                if (parsed?.delta?.type === 'input_json_delta') {
+                    const jsonDelta = parsed?.delta?.partial_json;
+                    if (!targetToolCall[this.#INPUT_DELTA_KEY]) {
+                        targetToolCall[this.#INPUT_DELTA_KEY] = '';
+                    }
+                    targetToolCall[this.#INPUT_DELTA_KEY] += jsonDelta;
+                }
+            }
+        }
+        if (parsed?.type === 'content_block_stop') {
+            const choiceIndex = 0;
+            const toolCallIndex = parsed?.index ?? 0;
+            const targetToolCall = toolCalls[choiceIndex]?.[toolCallIndex];
+            if (targetToolCall) {
+                const jsonDeltaString = targetToolCall[this.#INPUT_DELTA_KEY];
+                if (jsonDeltaString) {
+                    try {
+                        const jsonDelta = { input: JSON.parse(jsonDeltaString) };
+                        delete targetToolCall[this.#INPUT_DELTA_KEY];
+                        ToolManager.#applyToolCallDelta(targetToolCall, jsonDelta);
+                    } catch (error) {
+                        console.warn('Failed to apply input JSON delta:', error);
+                    }
+                }
+            }
+        }
+    }
+    /**
+     * Apply a tool call delta to a target object.
+     * @param {object} target The target object to apply the delta to
+     * @param {object} delta The delta object to apply
+     */
+    static #applyToolCallDelta(target, delta) {
+        for (const key in delta) {
+            if (!Object.prototype.hasOwnProperty.call(delta, key)) continue;
+            if (key === '__proto__' || key === 'constructor') continue;
+            const deltaValue = delta[key];
+            const targetValue = target[key];
+            if (deltaValue === null || deltaValue === undefined) {
+                target[key] = deltaValue;
+                continue;
+            }
+            if (typeof deltaValue === 'string') {
+                if (typeof targetValue === 'string') {
+                    // Concatenate strings
+                    target[key] = targetValue + deltaValue;
+                } else {
+                    target[key] = deltaValue;
+                }
+            } else if (typeof deltaValue === 'object' && !Array.isArray(deltaValue)) {
+                if (typeof targetValue !== 'object' || targetValue === null || Array.isArray(targetValue)) {
+                    target[key] = {};
+                }
+                // Recursively apply deltas to nested objects
+                ToolManager.#applyToolCallDelta(target[key], deltaValue);
+            } else {
+                // Assign other types directly
+                target[key] = deltaValue;
+            }
+        }
+    }
+    /**
+     * Checks if tool calling is supported for the current settings and generation type.
+     * @returns {boolean} Whether tool calling is supported for the given type
+     */
+    static isToolCallingSupported() {
+        if (main_api !== 'openai' || !oai_settings.function_calling) {
+            return false;
+        }
+        const supportedSources = [
+            chat_completion_sources.OPENAI,
+            chat_completion_sources.CUSTOM,
+            chat_completion_sources.MISTRALAI,
+            chat_completion_sources.CLAUDE,
+            chat_completion_sources.OPENROUTER,
+            chat_completion_sources.GROQ,
+        ];
+        return supportedSources.includes(oai_settings.chat_completion_source);
+    }
+    /**
+     * Checks if tool calls can be performed for the current settings and generation type.
+     * @param {string} type Generation type
+     * @returns {boolean} Whether tool calls can be performed for the given type
+     */
+    static canPerformToolCalls(type) {
+        const noToolCallTypes = ['impersonate', 'quiet', 'continue'];
+        const isSupported = ToolManager.isToolCallingSupported();
+        return isSupported && !noToolCallTypes.includes(type);
+    }
+    /**
+     * Utility function to get tool calls from the response data.
+     * @param {any} data Response data
+     * @returns {any[]} Tool calls from the response data
+     */
+    static #getToolCallsFromData(data) {
+        const isClaudeToolCall = c => Array.isArray(c) ? c.filter(x => x).every(isClaudeToolCall) : c?.input && c?.name && c?.id;
+        const convertClaudeToolCall = c => ({ id: c.id, function: { name: c.name, arguments: c.input } });
+        // Parsed tool calls from streaming data
+        if (Array.isArray(data) && data.length > 0 && Array.isArray(data[0])) {
+            return isClaudeToolCall(data[0]) ? data[0].filter(x => x).map(convertClaudeToolCall) : data[0];
+        }
+        // Parsed tool calls from non-streaming data
+        if (Array.isArray(data?.choices)) {
+            // Find a choice with 0-index
+            const choice = data.choices.find(choice => choice.index === 0);
+            if (choice) {
+                return choice.message.tool_calls;
+            }
+        }
+        // Claude tool calls to OpenAI tool calls
+        if (Array.isArray(data?.content)) {
+            const content = data.content.filter(c => c.type === 'tool_use').map(convertClaudeToolCall);
+            if (content) {
+                return content;
+            }
+        }
+    }
+    /**
+     * Checks if the response data contains tool calls.
+     * @param {object} data Response data
+     * @returns {boolean} Whether the response data contains tool calls
+     */
+    static hasToolCalls(data) {
+        const toolCalls = ToolManager.#getToolCallsFromData(data);
+        return Array.isArray(toolCalls) && toolCalls.length > 0;
+    }
+    /**
+     * Check for function tool calls in the response data and invoke them.
+     * @param {any} data Reply data
+     * @returns {Promise<ToolInvocationResult>} Successful tool invocations
+     */
+    static async invokeFunctionTools(data) {
+        /** @type {ToolInvocationResult} */
+        const result = {
+            invocations: [],
+            errors: [],
+        };
+        const toolCalls = ToolManager.#getToolCallsFromData(data);
+        if (!Array.isArray(toolCalls)) {
+            return result;
+        }
+        for (const toolCall of toolCalls) {
+            if (typeof toolCall.function !== 'object') {
+                continue;
+            }
+            console.log('Function tool call:', toolCall);
+            const id = toolCall.id;
+            const parameters = toolCall.function.arguments;
+            const name = toolCall.function.name;
+            const displayName = ToolManager.getDisplayName(name);
+            const message = ToolManager.formatToolCallMessage(name, parameters);
+            const toast = message && toastr.info(message, 'Tool Calling', { timeOut: 0 });
+            const toolResult = await ToolManager.invokeFunctionTool(name, parameters);
+            toastr.clear(toast);
+            console.log('Function tool result:', result);
+            // Save a successful invocation
+            if (toolResult instanceof Error) {
+                result.errors.push(toolResult);
+                continue;
+            }
+            const invocation = {
+                id,
+                displayName,
+                name,
+                parameters: stringify(parameters),
+                result: toolResult,
+            };
+            result.invocations.push(invocation);
+        }
+        return result;
+    }
+    /**
+     * Groups tool names by count.
+     * @param {string[]} toolNames Tool names
+     * @returns {string} Grouped tool names
+     */
+    static #groupToolNames(toolNames) {
+        const toolCounts = toolNames.reduce((acc, name) => {
+            acc[name] = (acc[name] || 0) + 1;
+            return acc;
+        }, {});
+        return Object.entries(toolCounts).map(([name, count]) => count > 1 ? `${name} (${count})` : name).join(', ');
+    }
+    /**
+     * Formats a message with tool invocations.
+     * @param {ToolInvocation[]} invocations Tool invocations.
+     * @returns {string} Formatted message with tool invocations.
+     */
+    static #formatToolInvocationMessage(invocations) {
+        const data = structuredClone(invocations);
+        const detailsElement = document.createElement('details');
+        const summaryElement = document.createElement('summary');
+        const preElement = document.createElement('pre');
+        const codeElement = document.createElement('code');
+        codeElement.classList.add('language-json');
+        data.forEach(i => {
+            i.parameters = tryParse(i.parameters);
+            i.result = tryParse(i.result);
+        });
+        codeElement.textContent = JSON.stringify(data, null, 2);
+        const toolNames = data.map(i => i.displayName || i.name);
+        summaryElement.textContent = `Tool calls: ${this.#groupToolNames(toolNames)}`;
+        preElement.append(codeElement);
+        detailsElement.append(summaryElement, preElement);
+        return detailsElement.outerHTML;
+    }
+    /**
+     * Saves function tool invocations to the last user chat message extra metadata.
+     * @param {ToolInvocation[]} invocations Successful tool invocations
+     */
+    static async saveFunctionToolInvocations(invocations) {
+        if (!Array.isArray(invocations) || invocations.length === 0) {
+            return;
+        }
+        const message = {
+            name: systemUserName,
+            force_avatar: system_avatar,
+            is_system: true,
+            is_user: false,
+            mes: ToolManager.#formatToolInvocationMessage(invocations),
+            extra: {
+                isSmallSys: true,
+                tool_invocations: invocations,
+            },
+        };
+        chat.push(message);
+        await eventSource.emit(event_types.TOOL_CALLS_PERFORMED, invocations);
+        addOneMessage(message);
+        await eventSource.emit(event_types.TOOL_CALLS_RENDERED, invocations);
+        await saveChatConditional();
+    }
+    /**
+     * Shows an error message for tool calls.
+     * @param {Error[]} errors Errors that occurred during tool invocation
+     * @returns {void}
+     */
+    static showToolCallError(errors) {
+        toastr.error('An error occurred while invoking function tools. Click here for more details.', 'Tool Calling', {
+            onclick: () => Popup.show.text('Tool Calling Errors', DOMPurify.sanitize(errors.map(e => `${e.cause}: ${e.message}`).join('<br>'))),
+            timeOut: 5000,
+        });
+    }
+    static initToolSlashCommands() {
+        const toolsEnumProvider = () => ToolManager.tools.map(tool => {
+            const toolOpenAI = tool.toFunctionOpenAI();
+            return new SlashCommandEnumValue(toolOpenAI.function.name, toolOpenAI.function.description, enumTypes.enum, enumIcons.closure);
+        });
+        SlashCommandParser.addCommandObject(SlashCommand.fromProps({
+            name: 'tools-list',
+            aliases: ['tool-list'],
+            helpString: 'Gets a list of all registered tools in the OpenAI function JSON format. Use the <code>return</code> argument to specify the return value type.',
+            returns: 'A list of all registered tools.',
+            namedArgumentList: [
+                SlashCommandNamedArgument.fromProps({
+                    name: 'return',
+                    description: 'The way how you want the return value to be provided',
+                    typeList: [ARGUMENT_TYPE.STRING],
+                    defaultValue: 'none',
+                    enumList: slashCommandReturnHelper.enumList({ allowObject: true }),
+                    forceEnum: true,
+                }),
+            ],
+            callback: async (args) => {
+                /** @type {any} */
+                const returnType = String(args?.return ?? 'popup-html').trim().toLowerCase();
+                const objectToStringFunc = (tools) => Array.isArray(tools) ? tools.map(x => x.toString()).join('\n\n') : tools.toString();
+                const tools = ToolManager.tools.map(tool => tool.toFunctionOpenAI());
+                return await slashCommandReturnHelper.doReturn(returnType ?? 'popup-html', tools ?? [], { objectToStringFunc });
+            },
+        }));
+        SlashCommandParser.addCommandObject(SlashCommand.fromProps({
+            name: 'tools-invoke',
+            aliases: ['tool-invoke'],
+            helpString: 'Invokes a registered tool by name. The <code>parameters</code> argument MUST be a JSON-serialized object.',
+            namedArgumentList: [
+                SlashCommandNamedArgument.fromProps({
+                    name: 'parameters',
+                    description: 'The parameters to pass to the tool.',
+                    typeList: [ARGUMENT_TYPE.DICTIONARY],
+                    isRequired: true,
+                    acceptsMultiple: false,
+                }),
+            ],
+            unnamedArgumentList: [
+                SlashCommandArgument.fromProps({
+                    description: 'The name of the tool to invoke.',
+                    typeList: [ARGUMENT_TYPE.STRING],
+                    isRequired: true,
+                    acceptsMultiple: false,
+                    forceEnum: true,
+                    enumProvider: toolsEnumProvider,
+                }),
+            ],
+            callback: async (args, name) => {
+                const { parameters } = args;
+                const result = await ToolManager.invokeFunctionTool(String(name), parameters);
+                if (result instanceof Error) {
+                    throw result;
+                }
+                return result;
+            },
+        }));
+        SlashCommandParser.addCommandObject(SlashCommand.fromProps({
+            name: 'tools-register',
+            aliases: ['tool-register'],
+            helpString: `<div>Registers a new tool with the tool registry.</div>
+                <ul>
+                    <li>The <code>parameters</code> argument MUST be a JSON-serialized object with a valid JSON schema.</li>
+                    <li>The unnamed argument MUST be a closure that accepts the function parameters as local script variables.</li>
+                </ul>
+                <div>See <a target="_blank" href="https://json-schema.org/learn/">json-schema.org</a> and <a target="_blank" href="https://platform.openai.com/docs/guides/function-calling">OpenAI Function Calling</a> for more information.</div>
+                <div>Example:</div>
+                <pre><code>/let key=echoSchema
+{
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "type": "object",
+    "properties": {
+        "message": {
+            "type": "string",
+            "description": "The message to echo."
+        }
+    },
+    "required": [
+        "message"
+    ]
+}
+||
+/tools-register name=Echo description="Echoes a message. Call when the user is asking to repeat something" parameters={{var::echoSchema}} {: /echo {{var::arg.message}} :}</code></pre>`,
+            namedArgumentList: [
+                SlashCommandNamedArgument.fromProps({
+                    name: 'name',
+                    description: 'The name of the tool.',
+                    typeList: [ARGUMENT_TYPE.STRING],
+                    isRequired: true,
+                    acceptsMultiple: false,
+                }),
+                SlashCommandNamedArgument.fromProps({
+                    name: 'description',
+                    description: 'A description of what the tool does.',
+                    typeList: [ARGUMENT_TYPE.STRING],
+                    isRequired: true,
+                    acceptsMultiple: false,
+                }),
+                SlashCommandNamedArgument.fromProps({
+                    name: 'parameters',
+                    description: 'The parameters for the tool.',
+                    typeList: [ARGUMENT_TYPE.DICTIONARY],
+                    isRequired: true,
+                    acceptsMultiple: false,
+                }),
+                SlashCommandNamedArgument.fromProps({
+                    name: 'displayName',
+                    description: 'The display name of the tool.',
+                    typeList: [ARGUMENT_TYPE.STRING],
+                    isRequired: false,
+                    acceptsMultiple: false,
+                }),
+                SlashCommandNamedArgument.fromProps({
+                    name: 'formatMessage',
+                    description: 'The closure to be executed to format the tool call message. Must return a string.',
+                    typeList: [ARGUMENT_TYPE.CLOSURE],
+                    isRequired: true,
+                    acceptsMultiple: false,
+                }),
+            ],
+            unnamedArgumentList: [
+                SlashCommandArgument.fromProps({
+                    description: 'The closure to be executed when the tool is invoked.',
+                    typeList: [ARGUMENT_TYPE.CLOSURE],
+                    isRequired: true,
+                    acceptsMultiple: false,
+                }),
+            ],
+            callback: async (args, action) => {
+                /**
+                 * Converts a slash command closure to a function.
+                 * @param {SlashCommandClosure} action Closure to convert to a function
+                 * @returns {function} Function that executes the closure
+                 */
+                function closureToFunction(action) {
+                    return async (args) => {
+                        const localClosure = action.getCopy();
+                        localClosure.onProgress = () => { };
+                        const scope = localClosure.scope;
+                        if (typeof args === 'object' && args !== null) {
+                            assignNestedVariables(scope, args, 'arg');
+                        } else if (typeof args !== 'undefined') {
+                            scope.letVariable('arg', args);
+                        }
+                        const result = await localClosure.execute();
+                        return result.pipe;
+                    };
+                }
+                const { name, displayName, description, parameters, formatMessage } = args;
+                if (!(action instanceof SlashCommandClosure)) {
+                    throw new Error('The unnamed argument must be a closure.');
+                }
+                if (typeof name !== 'string' || !name) {
+                    throw new Error('The "name" argument must be a non-empty string.');
+                }
+                if (typeof description !== 'string' || !description) {
+                    throw new Error('The "description" argument must be a non-empty string.');
+                }
+                if (typeof parameters !== 'string' || !isJson(parameters)) {
+                    throw new Error('The "parameters" argument must be a JSON-serialized object.');
+                }
+                if (displayName && typeof displayName !== 'string') {
+                    throw new Error('The "displayName" argument must be a string.');
+                }
+                if (formatMessage && !(formatMessage instanceof SlashCommandClosure)) {
+                    throw new Error('The "formatMessage" argument must be a closure.');
+                }
+                const actionFunc = closureToFunction(action);
+                const formatMessageFunc = formatMessage instanceof SlashCommandClosure ? closureToFunction(formatMessage) : null;
+                ToolManager.registerFunctionTool({
+                    name: String(name ?? ''),
+                    displayName: String(displayName ?? ''),
+                    description: String(description ?? ''),
+                    parameters: JSON.parse(parameters ?? '{}'),
+                    action: actionFunc,
+                    formatMessage: formatMessageFunc,
+                });
+                return '';
+            },
+        }));
+        SlashCommandParser.addCommandObject(SlashCommand.fromProps({
+            name: 'tools-unregister',
+            aliases: ['tool-unregister'],
+            helpString: 'Unregisters a tool from the tool registry.',
+            unnamedArgumentList: [
+                SlashCommandArgument.fromProps({
+                    description: 'The name of the tool to unregister.',
+                    typeList: [ARGUMENT_TYPE.STRING],
+                    isRequired: true,
+                    acceptsMultiple: false,
+                    forceEnum: true,
+                    enumProvider: toolsEnumProvider,
+                }),
+            ],
+            callback: async (name) => {
+                if (typeof name !== 'string' || !name) {
+                    throw new Error('The unnamed argument must be a non-empty string.');
+                }
+                ToolManager.unregisterFunctionTool(name);
+                return '';
+            },
+        }));
+    }
+}

public/style.css+10 -0 Viewed

     text-align: center;
+}
+.mes.smallSysMes pre {
+    text-align: initial;
+    word-break: break-all;
+    margin-top: 5px;
+}
+.mes.smallSysMes summary {
+    cursor: pointer;
+}
 .mes.smallSysMes .mes_text p:last-child {
     margin: 0;
+}

src/endpoints/backends/chat-completions.js+23 -31 Viewed

 const { jsonParser } = require('../../express-common');
 const { CHAT_COMPLETION_SOURCES, GEMINI_SAFETY, BISON_SAFETY, OPENROUTER_HEADERS } = require('../../constants');
 const { forwardFetchResponse, getConfigValue, tryParse, uuidv4, mergeObjectWithYaml, excludeKeysByYaml, color } = require('../../util');
-const { convertClaudeMessages, convertGooglePrompt, convertTextCompletionPrompt, convertCohereMessages, convertMistralMessages, convertCohereTools, convertAI21Messages } = require('../../prompt-converters');
+const { convertClaudeMessages, convertGooglePrompt, convertTextCompletionPrompt, convertCohereMessages, convertMistralMessages, convertAI21Messages, mergeMessages } = require('../../prompt-converters');
 const CohereStream = require('../../cohere-stream');
 const { readSecret, SECRET_KEYS } = require('../secrets');

  */
 function postProcessPrompt(messages, type, charName, userName) {
     switch (type) {
+        case 'merge':
         case 'claude':
-            return convertClaudeMessages(messages, '', false, '', charName, userName).messages;
+            return mergeMessages(messages, charName, userName, false);
+        case 'strict':
+            return mergeMessages(messages, charName, userName, true);
         default:
             return messages;
+    }

     const apiUrl = new URL(request.body.reverse_proxy || API_CLAUDE).toString();
     const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.CLAUDE);
     const divider = '-'.repeat(process.stdout.columns);
-    const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false);
+    const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false) && request.body.model.startsWith('claude-3');
     if (!apiKey) {
         console.log(color.red(`Claude API key is missing.\n${divider}`));

             controller.abort();
         });
         const additionalHeaders = {};
+        const useTools = request.body.model.startsWith('claude-3') && Array.isArray(request.body.tools) && request.body.tools.length > 0;
         const useSystemPrompt = (request.body.model.startsWith('claude-2') || request.body.model.startsWith('claude-3')) && request.body.claude_use_sysprompt;
-        const convertedPrompt = convertClaudeMessages(request.body.messages, request.body.assistant_prefill, useSystemPrompt, request.body.human_sysprompt_message, request.body.char_name, request.body.user_name);
+        const convertedPrompt = convertClaudeMessages(request.body.messages, request.body.assistant_prefill, useSystemPrompt, useTools, request.body.human_sysprompt_message, request.body.char_name, request.body.user_name);
         // Add custom stop sequences
         const stopSequences = [];
         if (Array.isArray(request.body.stop)) {

+        }
         const requestBody = {
-            /** @type {any} */ system: '',
+            /** @type {any} */ system: [],
             messages: convertedPrompt.messages,
             model: request.body.model,
             max_tokens: request.body.max_tokens,

             stream: request.body.stream,
         };
         if (useSystemPrompt) {
-            requestBody.system = enableSystemPromptCache
+            if (enableSystemPromptCache && Array.isArray(convertedPrompt.systemPrompt) && convertedPrompt.systemPrompt.length) {
-                ? [{ type: 'text', text: convertedPrompt.systemPrompt, cache_control: { type: 'ephemeral' } }]
+                convertedPrompt.systemPrompt[convertedPrompt.systemPrompt.length - 1]['cache_control'] = { type: 'ephemeral' };
-                : convertedPrompt.systemPrompt;
+            }
+            requestBody.system = convertedPrompt.systemPrompt;
         } else {
             delete requestBody.system;
+        }
-        if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
+        if (useTools) {
             // Claude doesn't do prefills on function calls, and doesn't allow empty messages
             if (convertedPrompt.messages.length && convertedPrompt.messages[convertedPrompt.messages.length - 1].role === 'assistant') {
                 convertedPrompt.messages.push({ role: 'user', content: '.' });
+            }
             additionalHeaders['anthropic-beta'] = 'tools-2024-05-16';
-            requestBody.tool_choice = { type: request.body.tool_choice === 'required' ? 'any' : 'auto' };
+            requestBody.tool_choice = { type: request.body.tool_choice };
             requestBody.tools = request.body.tools
                 .filter(tool => tool.type === 'function')
                 .map(tool => tool.function)
                 .map(fn => ({ name: fn.name, description: fn.description, input_schema: fn.parameters }));
+            if (enableSystemPromptCache && requestBody.tools.length) {
+                requestBody.tools[requestBody.tools.length - 1]['cache_control'] = { type: 'ephemeral' };
+            }
+        }
         if (enableSystemPromptCache) {
             additionalHeaders['anthropic-beta'] = 'prompt-caching-2024-07-31';

         if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
             requestBody['tools'] = request.body.tools;
-            requestBody['tool_choice'] = request.body.tool_choice === 'required' ? 'any' : 'auto';
+            requestBody['tool_choice'] = request.body.tool_choice;
+        }
         const config = {

             });
+        }
-        if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
-            tools.push(...convertCohereTools(request.body.tools));
-            // Can't have both connectors and tools in the same request
-            connectors.splice(0, connectors.length);
-        }
         // https://docs.cohere.com/reference/chat
         const requestBody = {
             stream: Boolean(request.body.stream),

         apiKey = readSecret(request.user.directories, SECRET_KEYS.PERPLEXITY);
         headers = {};
         bodyParams = {};
-        request.body.messages = postProcessPrompt(request.body.messages, 'claude', request.body.char_name, request.body.user_name);
+        request.body.messages = postProcessPrompt(request.body.messages, 'strict', request.body.char_name, request.body.user_name);
     } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.GROQ) {
         apiUrl = API_GROQ;
         apiKey = readSecret(request.user.directories, SECRET_KEYS.GROQ);
         headers = {};
         bodyParams = {};
-        // 'required' tool choice is not supported by Groq
-        if (request.body.tool_choice === 'required') {
-            if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
-                request.body.tool_choice = request.body.tools.length > 1
-                    ? 'auto' :
-                    { type: 'function', function: { name: request.body.tools[0]?.function?.name } };
-            } else {
-                request.body.tool_choice = 'none';
-            }
-        }
     } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.ZEROONEAI) {
         apiUrl = API_01AI;
         apiKey = readSecret(request.user.directories, SECRET_KEYS.ZEROONEAI);

         controller.abort();
     });
-    if (!isTextCompletion) {
+    if (!isTextCompletion && Array.isArray(request.body.tools) && request.body.tools.length > 0) {
         bodyParams['tools'] = request.body.tools;
         bodyParams['tool_choice'] = request.body.tool_choice;
+    }

src/endpoints/novelai.js+1 -1 Viewed

 , 1, 11, 13, 25, 198, 12, 9, 8, 279, 264, 459, 323, 477, 539, 912, 374, 574, 1051, 1550, 1587, 4536, 5828, 15058,
 , 3250, 1461, 1077, 813, 11074, 872, 1202, 1436, 7846, 1288, 13434, 1053, 8434, 617, 9167, 1047, 19117, 706,
 , 649, 4250, 527, 7784, 690, 2834, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 1210, 1359, 608, 220, 596, 956,
-, 44886, 4265, 3358, 2351, 2846, 311, 389, 315, 304, 520, 505, 430
+, 44886, 4265, 3358, 2351, 2846, 311, 389, 315, 304, 520, 505, 430,
 ];
 // Ban the dinkus and asterism

src/endpoints/search.js+70 -49 Viewed

     'Sec-Fetch-User': '?1',
 };
-router.post('/serpapi', jsonParser, async (request, response) => {
-    try {
-        const key = readSecret(request.user.directories, SECRET_KEYS.SERPAPI);
-        if (!key) {
-            console.log('No SerpApi key found');
-            return response.sendStatus(400);
-        }
-        const { query } = request.body;
-        const result = await fetch(`https://serpapi.com/search.json?q=${encodeURIComponent(query)}&api_key=${key}`);
-        console.log('SerpApi query', query);
-        if (!result.ok) {
-            const text = await result.text();
-            console.log('SerpApi request failed', result.statusText, text);
-            return response.status(500).send(text);
-        }
-        const data = await result.json();
-        return response.json(data);
-    } catch (error) {
-        console.log(error);
-        return response.sendStatus(500);
-    }
-});
 /**
- * Get the transcript of a YouTube video
+ * Extract the transcript of a YouTube video
- * @copyright https://github.com/Kakulukian/youtube-transcript (MIT License)
+ * @param {string} videoPageBody HTML of the video page
+ * @param {string} lang Language code
+ * @returns {Promise<string>} Transcript text
  */
-router.post('/transcript', jsonParser, async (request, response) => {
+async function extractTranscript(videoPageBody, lang) {
-    try {
     const he = require('he');
     const RE_XML_TRANSCRIPT = /<text start="([^"]*)" dur="([^"]*)">([^<]*)<\/text>/g;
-        const id = request.body.id;
-        const lang = request.body.lang;
-        if (!id) {
-            console.log('Id is required for /transcript');
-            return response.sendStatus(400);
-        }
-        const videoPageResponse = await fetch(`https://www.youtube.com/watch?v=${id}`, {
-            headers: {
-                ...(lang && { 'Accept-Language': lang }),
-                'User-Agent': visitHeaders['User-Agent'],
-            },
-        });
-        const videoPageBody = await videoPageResponse.text();
     const splittedHTML = videoPageBody.split('"captions":');
     if (splittedHTML.length <= 1) {

     }));
     // The text is double-encoded
     const transcriptText = transcript.map((line) => he.decode(he.decode(line.text))).join(' ');
+    return transcriptText;
+}
+router.post('/serpapi', jsonParser, async (request, response) => {
+    try {
+        const key = readSecret(request.user.directories, SECRET_KEYS.SERPAPI);
+        if (!key) {
+            console.log('No SerpApi key found');
+            return response.sendStatus(400);
+        }
+        const { query } = request.body;
+        const result = await fetch(`https://serpapi.com/search.json?q=${encodeURIComponent(query)}&api_key=${key}`);
+        console.log('SerpApi query', query);
+        if (!result.ok) {
+            const text = await result.text();
+            console.log('SerpApi request failed', result.statusText, text);
+            return response.status(500).send(text);
+        }
+        const data = await result.json();
+        return response.json(data);
+    } catch (error) {
+        console.log(error);
+        return response.sendStatus(500);
+    }
+});
+/**
+ * Get the transcript of a YouTube video
+ * @copyright https://github.com/Kakulukian/youtube-transcript (MIT License)
+ */
+router.post('/transcript', jsonParser, async (request, response) => {
+    try {
+        const id = request.body.id;
+        const lang = request.body.lang;
+        const json = request.body.json;
+        if (!id) {
+            console.log('Id is required for /transcript');
+            return response.sendStatus(400);
+        }
-        return response.send(transcriptText);
+        const videoPageResponse = await fetch(`https://www.youtube.com/watch?v=${id}`, {
+            headers: {
+                ...(lang && { 'Accept-Language': lang }),
+                'User-Agent': visitHeaders['User-Agent'],
+            },
+        });
+        const videoPageBody = await videoPageResponse.text();
+        try {
+            const transcriptText = await extractTranscript(videoPageBody, lang);
+            return json
+                ? response.json({ transcript: transcriptText, html: videoPageBody })
+                : response.send(transcriptText);
+        } catch (error) {
+            if (json) {
+                return response.json({ html: videoPageBody, transcript: '' });
+            }
+            throw error;
+        }
     } catch (error) {
         console.log(error);
         return response.sendStatus(500);

src/prompt-converters.js+237 -119 Viewed

 require('./polyfill.js');
 const { getConfigValue } = require('./util.js');
+const crypto = require('crypto');
+const PROMPT_PLACEHOLDER = getConfigValue('promptPlaceholder', 'Let\'s get started.');
 /**
  * Convert a prompt from the ChatML objects to the format used by Claude.

     //Prepare messages for claude.
     //When 'Exclude Human/Assistant prefixes' checked, setting messages role to the 'system'(last message is exception).
     if (messages.length > 0) {
+        messages.forEach((m) => {
+            if (!m.content) {
+                m.content = '';
+            }
+            if (m.tool_calls) {
+                m.content += JSON.stringify(m.tool_calls);
+            }
+        });
         if (excludePrefixes) {
             messages.slice(0, -1).forEach(message => message.role = 'system');
         } else {

  * @param {object[]} messages Array of messages
  * @param {string}   prefillString User determined prefill string
  * @param {boolean}  useSysPrompt See if we want to use a system prompt
+ * @param {boolean}  useTools See if we want to use tools
  * @param {string}   humanMsgFix Add Human message between system prompt and assistant.
  * @param {string}   charName Character name
  * @param {string}   userName User name
  */
-function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFix, charName = '', userName = '') {
+function convertClaudeMessages(messages, prefillString, useSysPrompt, useTools, humanMsgFix, charName = '', userName = '') {
-    let systemPrompt = '';
+    let systemPrompt = [];
     if (useSysPrompt) {
         // Collect all the system messages up until the first instance of a non-system message, and then remove them from the messages array.
         let i;

@@ -104,7 +116,7 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFi
104	messages[i].content = `${charName}: ${messages[i].content}`;	116	messages[i].content = `${charName}: ${messages[i].content}`;
105	}	117	}
106	}	118	}
107	systemPrompt += `${messages[i].content}\n\n`;	119	systemPrompt.push({ type: 'text', text: messages[i].content });
108	}	120	}
109		121
110	messages.splice(0, i);	122	messages.splice(0, i);

         if (messages.length === 0 || (messages.length > 0 && messages[0].role !== 'user')) {
             messages.unshift({
                 role: 'user',
-                content: humanMsgFix || '[Start a new chat]',
+                content: humanMsgFix || PROMPT_PLACEHOLDER,
             });
+        }
+    }
     // Now replace all further messages that have the role 'system' with the role 'user'. (or all if we're not using one)
+    const parse = (str) => typeof str === 'string' ? JSON.parse(str) : str;
     messages.forEach((message) => {
+        if (message.role === 'assistant' && message.tool_calls) {
+            message.content = message.tool_calls.map((tc) => ({
+                type: 'tool_use',
+                id: tc.id,
+                name: tc.function.name,
+                input: parse(tc.function.arguments),
+            }));
+        }
+        if (message.role === 'tool') {
+            message.role = 'user';
+            message.content = [{
+                type: 'tool_result',
+                tool_use_id: message.tool_call_id,
+                content: message.content,
+            }];
+        }
         if (message.role === 'system') {
             if (userName && message.name === 'example_user') {
                 message.content = `${userName}: ${message.content}`;

                 message.content = `${charName}: ${message.content}`;
+            }
             message.role = 'user';
+            // Delete name here so it doesn't get added later
+            delete message.name;
+        }
-    });
-    // Shouldn't be conditional anymore, messages api expects the last role to be user unless we're explicitly prefilling
+        // Convert everything to an array of it would be easier to work with
-    if (prefillString) {
+        if (typeof message.content === 'string') {
-        messages.push({
+            // Take care of name properties since claude messages don't support them
-            role: 'assistant',
+            if (message.name) {
-            content: prefillString.trimEnd(),
+                message.content = `${message.name}: ${message.content}`;
-        });
+            }
-    // Since the messaging endpoint only supports user assistant roles in turns, we have to merge messages with the same role if they follow eachother
+            message.content = [{ type: 'text', text: message.content }];
-    // Also handle multi-modality, holy slop.
+        } else if (Array.isArray(message.content)) {
-    let mergedMessages = [];
+            message.content = message.content.map((content) => {
-    messages.forEach((message) => {
+                if (content.type === 'image_url') {
-        const imageEntry = message.content?.[1]?.image_url;
+                    const imageEntry = content?.image_url;
                     const imageData = imageEntry?.url;
                     const mimeType = imageData?.split(';')?.[0].split(':')?.[1];
                     const base64Data = imageData?.split(',')?.[1];
-        // Take care of name properties since claude messages don't support them
+                    return {
+                        type: 'image',
+                        source: {
+                            type: 'base64',
+                            media_type: mimeType,
+                            data: base64Data,
+                        },
+                    };
+                }
+                if (content.type === 'text') {
                     if (message.name) {
-            if (Array.isArray(message.content)) {
+                        content.text = `${message.name}: ${content.text}`;
-                message.content[0].text = `${message.name}: ${message.content[0].text}`;
+                    }
-            } else {
-                message.content = `${message.name}: ${message.content}`;
+                    return content;
+                }
+                return content;
+            });
+        }
+        // Remove offending properties
         delete message.name;
+        delete message.tool_calls;
+        delete message.tool_call_id;
+    });
+    // Images in assistant messages should be moved to the next user message
+    for (let i = 0; i < messages.length; i++) {
+        if (messages[i].role === 'assistant' && messages[i].content.some(c => c.type === 'image')) {
+            // Find the next user message
+            let j = i + 1;
+            while (j < messages.length && messages[j].role !== 'user') {
+                j++;
+            }
-        if (mergedMessages.length > 0 && mergedMessages[mergedMessages.length - 1].role === message.role) {
+            // Move the images
-            if (Array.isArray(message.content)) {
+            if (j >= messages.length) {
-                if (Array.isArray(mergedMessages[mergedMessages.length - 1].content)) {
+                // If there is no user message after the assistant message, add a new one
-                    mergedMessages[mergedMessages.length - 1].content[0].text += '\n\n' + message.content[0].text;
+                messages.splice(i + 1, 0, { role: 'user', content: [] });
-                } else {
-                    mergedMessages[mergedMessages.length - 1].content += '\n\n' + message.content[0].text;
+            }
-            } else {
-                if (Array.isArray(mergedMessages[mergedMessages.length - 1].content)) {
+            messages[j].content.push(...messages[i].content.filter(c => c.type === 'image'));
-                    mergedMessages[mergedMessages.length - 1].content[0].text += '\n\n' + message.content;
+            messages[i].content = messages[i].content.filter(c => c.type !== 'image');
-                } else {
-                    mergedMessages[mergedMessages.length - 1].content += '\n\n' + message.content;
+        }
+    }
+    // Shouldn't be conditional anymore, messages api expects the last role to be user unless we're explicitly prefilling
+    if (prefillString) {
+        messages.push({
+            role: 'assistant',
+            // Dangling whitespace are not allowed for prefilling
+            content: [{ type: 'text', text: prefillString.trimEnd() }],
+        });
+    }
+    // Since the messaging endpoint only supports user assistant roles in turns, we have to merge messages with the same role if they follow eachother
+    // Also handle multi-modality, holy slop.
+    let mergedMessages = [];
+    messages.forEach((message) => {
+        if (mergedMessages.length > 0 && mergedMessages[mergedMessages.length - 1].role === message.role) {
+            mergedMessages[mergedMessages.length - 1].content.push(...message.content);
         } else {
             mergedMessages.push(message);
+        }
-        if (imageData) {
+    });
-            mergedMessages[mergedMessages.length - 1].content = [
-                { type: 'text', text: mergedMessages[mergedMessages.length - 1].content[0]?.text || mergedMessages[mergedMessages.length - 1].content },
+    if (!useTools) {
-                {
+        mergedMessages.forEach((message) => {
-                    type: 'image', source: {
+            message.content.forEach((content) => {
-                        type: 'base64',
+                if (content.type === 'tool_use') {
-                        media_type: mimeType,
+                    content.type = 'text';
-                        data: base64Data,
+                    content.text = JSON.stringify(content.input);
-                    },
+                    delete content.id;
-                },
+                    delete content.name;
-            ];
+                    delete content.input;
+                }
+                if (content.type === 'tool_result') {
+                    content.type = 'text';
+                    content.text = content.content;
+                    delete content.tool_use_id;
+                    delete content.content;
+                }
             });
+        });
+    }
-    return { messages: mergedMessages, systemPrompt: systemPrompt.trim() };
+    return { messages: mergedMessages, systemPrompt: systemPrompt };
+}
 /**

         'user': 'USER',
         'assistant': 'CHATBOT',
     };
-    const placeholder = '[Start a new chat]';
     let systemPrompt = '';
     // Collect all the system messages up until the first instance of a non-system message, and then remove them from the messages array.

     if (messages.length === 0) {
         messages.unshift({
             role: 'user',
-            content: placeholder,
+            content: PROMPT_PLACEHOLDER,
         });
+    }
     const lastNonSystemMessageIndex = messages.findLastIndex(msg => msg.role === 'user' || msg.role === 'assistant');
-    const userPrompt = messages.slice(lastNonSystemMessageIndex).map(msg => msg.content).join('\n\n') || placeholder;
+    const userPrompt = messages.slice(lastNonSystemMessageIndex).map(msg => msg.content).join('\n\n') || PROMPT_PLACEHOLDER;
     const chatHistory = messages.slice(0, lastNonSystemMessageIndex).map(msg => {
         return {

     if (messages.length === 0) {
         messages.unshift({
             role: 'user',
-            content: '[Start a new chat]',
+            content: PROMPT_PLACEHOLDER,
         });
+    }

         lastMsg.prefix = true;
+    }
+    const sanitizeToolId = (id) => crypto.createHash('sha512').update(id).digest('hex').slice(0, 9);
     // Doesn't support completion names, so prepend if not already done by the frontend (e.g. for group chats).
     messages.forEach(msg => {
+        if ('tool_calls' in msg && Array.isArray(msg.tool_calls)) {
+            msg.tool_calls.forEach(tool => {
+                tool.id = sanitizeToolId(tool.id);
+            });
+        }
+        if ('tool_call_id' in msg && msg.role === 'tool') {
+            msg.tool_call_id = sanitizeToolId(msg.tool_call_id);
+        }
         if (msg.role === 'system' && msg.name === 'example_assistant') {
             if (charName && !msg.content.startsWith(`${charName}: `)) {
                 msg.content = `${charName}: ${msg.content}`;

+        }
     });
+    // If user role message immediately follows a tool message, append it to the last user message
+    const fixToolMessages = () => {
+        let rerun = true;
+        while (rerun) {
+            rerun = false;
+            messages.forEach((message, i) => {
+                if (i === messages.length - 1) {
+                    return;
+                }
+                if (message.role === 'tool' && messages[i + 1].role === 'user') {
+                    const lastUserMessage = messages.slice(0, i).findLastIndex(m => m.role === 'user' && m.content);
+                    if (lastUserMessage !== -1) {
+                        messages[lastUserMessage].content += '\n\n' + messages[i + 1].content;
+                        messages.splice(i + 1, 1);
+                        rerun = true;
+                    }
+                }
+            });
+        }
+    };
+    fixToolMessages();
     // If system role message immediately follows an assistant message, change its role to user
     for (let i = 0; i < messages.length - 1; i++) {
         if (messages[i].role === 'assistant' && messages[i + 1].role === 'system') {

+}
 /**
+ * Merge messages with the same consecutive role, removing names if they exist.
+ * @param {any[]} messages Messages to merge
+ * @param {string} charName Character name
+ * @param {string} userName User name
+ * @param {boolean} strict Enable strict mode: only allow one system message at the start, force user first message
+ * @returns {any[]} Merged messages
+ */
+function mergeMessages(messages, charName, userName, strict) {
+    let mergedMessages = [];
+    // Remove names from the messages
+    messages.forEach((message) => {
+        if (!message.content) {
+            message.content = '';
+        }
+        if (message.role === 'system' && message.name === 'example_assistant') {
+            if (charName && !message.content.startsWith(`${charName}: `)) {
+                message.content = `${charName}: ${message.content}`;
+            }
+        }
+        if (message.role === 'system' && message.name === 'example_user') {
+            if (userName && !message.content.startsWith(`${userName}: `)) {
+                message.content = `${userName}: ${message.content}`;
+            }
+        }
+        if (message.name && message.role !== 'system') {
+            if (!message.content.startsWith(`${message.name}: `)) {
+                message.content = `${message.name}: ${message.content}`;
+            }
+        }
+        if (message.role === 'tool') {
+            message.role = 'user';
+        }
+        delete message.name;
+        delete message.tool_calls;
+        delete message.tool_call_id;
+    });
+    // Squash consecutive messages with the same role
+    messages.forEach((message) => {
+        if (mergedMessages.length > 0 && mergedMessages[mergedMessages.length - 1].role === message.role && message.content) {
+            mergedMessages[mergedMessages.length - 1].content += '\n\n' + message.content;
+        } else {
+            mergedMessages.push(message);
+        }
+    });
+    // Prevent erroring out if the messages array is empty.
+    if (messages.length === 0) {
+        messages.unshift({
+            role: 'user',
+            content: PROMPT_PLACEHOLDER,
+        });
+    }
+    if (strict) {
+        for (let i = 0; i < mergedMessages.length; i++) {
+            // Force mid-prompt system messages to be user messages
+            if (i > 0 && mergedMessages[i].role === 'system') {
+                mergedMessages[i].role = 'user';
+            }
+        }
+        if (mergedMessages.length) {
+            if (mergedMessages[0].role === 'system' && (mergedMessages.length === 1 || mergedMessages[1].role !== 'user')) {
+                mergedMessages.splice(1, 0, { role: 'user', content: PROMPT_PLACEHOLDER });
+            }
+            else if (mergedMessages[0].role !== 'system' && mergedMessages[0].role !== 'user') {
+                mergedMessages.unshift({ role: 'user', content: PROMPT_PLACEHOLDER });
+            }
+        }
+        return mergeMessages(mergedMessages, charName, userName, false);
+    }
+    return mergedMessages;
+}
+/**
  * Convert a prompt from the ChatML objects to the format used by Text Completion API.
  * @param {object[]} messages Array of messages
  * @returns {string} Prompt for Text Completion API

     return messageStrings.join('\n') + '\nassistant:';
+}
-/**
- * Convert OpenAI Chat Completion tools to the format used by Cohere.
- * @param {object[]} tools OpenAI Chat Completion tool definitions
- */
-function convertCohereTools(tools) {
-    if (!Array.isArray(tools) || tools.length === 0) {
-        return [];
-    }
-    const jsonSchemaToPythonTypes = {
-        'string': 'str',
-        'number': 'float',
-        'integer': 'int',
-        'boolean': 'bool',
-        'array': 'list',
-        'object': 'dict',
-    };
-    const cohereTools = [];
-    for (const tool of tools) {
-        if (tool?.type !== 'function') {
-            console.log(`Unsupported tool type: ${tool.type}`);
-            continue;
-        }
-        const name = tool?.function?.name;
-        const description = tool?.function?.description;
-        const properties = tool?.function?.parameters?.properties;
-        const required = tool?.function?.parameters?.required;
-        const parameters = {};
-        if (!name) {
-            console.log('Tool name is missing');
-            continue;
-        }
-        if (!description) {
-            console.log('Tool description is missing');
-        }
-        if (!properties || typeof properties !== 'object') {
-            console.log(`No properties found for tool: ${tool?.function?.name}`);
-            continue;
-        }
-        for (const property in properties) {
-            const parameterDefinition = properties[property];
-            const description = parameterDefinition.description || (parameterDefinition.enum ? JSON.stringify(parameterDefinition.enum) : '');
-            const type = jsonSchemaToPythonTypes[parameterDefinition.type] || 'str';
-            const isRequired = Array.isArray(required) && required.includes(property);
-            parameters[property] = {
-                description: description,
-                type: type,
-                required: isRequired,
-            };
-        }
-        const cohereTool = {
-            name: tool.function.name,
-            description: tool.function.description,
-            parameter_definitions: parameters,
-        };
-        cohereTools.push(cohereTool);
-    }
-    return cohereTools;
-}
 module.exports = {
     convertClaudePrompt,
     convertClaudeMessages,

     convertTextCompletionPrompt,
     convertCohereMessages,
     convertMistralMessages,
-    convertCohereTools,
     convertAI21Messages,
+    mergeMessages,
 };