New tool calling framework

8006795897c54180823323357ad5a74c8660fe25

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: b3e88c82b83566dc632137c81955d07d15933a18

public/global.d.ts+0 -41 Viewed

 declare global {
   const moment: typeof moment;
+}
+-
 -/**
 - * Callback data for the `LLM_FUNCTION_TOOL_REGISTER` event type that is triggered when a function tool can be registered.
 - */
 -interface FunctionToolRegister {
 -    /**
 -     * The type of generation that is being used
 -     */
 -    type?: string;
 -    /**
 -     * Generation data, including messages and sampling parameters
 -     */
 -    data: Record<string, object>;
 -    /**
 -     * Callback to register an LLM function tool.
 -     */
 -    registerFunctionTool: typeof registerFunctionTool;
 -}
+-
 -/**
 - * Callback data for the `LLM_FUNCTION_TOOL_REGISTER` event type that is triggered when a function tool is registered.
 - * @param name Name of the function tool to register
 - * @param description Description of the function tool
 - * @param params JSON schema for the parameters of the function tool
 - * @param required Whether the function tool should be forced to be used
 - */
 -declare function registerFunctionTool(name: string, description: string, params: object, required: boolean): Promise<void>;
+-
 -/**
 - * Callback data for the `LLM_FUNCTION_TOOL_CALL` event type that is triggered when a function tool is called.
 - */
 -interface FunctionToolCall {
 -    /**
 -     * Name of the function tool to call
 -     */
 -    name: string;
 -    /**
 -     * JSON object with the parameters to pass to the function tool
 -     */
 -    arguments: string;
 -}

public/script.js+30 -5 Viewed

 import { AbortReason } from './scripts/util/AbortReason.js';
 import { initSystemPrompts } from './scripts/sysprompt.js';
 import { registerExtensionSlashCommands as initExtensionSlashCommands } from './scripts/extensions-slashcommands.js';
 +import { ToolManager } from './scripts/tool-calling.js';
 //exporting functions and vars for mods
 export {

     FILE_ATTACHMENT_DELETED: 'file_attachment_deleted',
     WORLDINFO_FORCE_ACTIVATE: 'worldinfo_force_activate',
     OPEN_CHARACTER_LIBRARY: 'open_character_library',
 -    LLM_FUNCTION_TOOL_REGISTER: 'llm_function_tool_register',
 -    LLM_FUNCTION_TOOL_CALL: 'llm_function_tool_call',
     ONLINE_STATUS_CHANGED: 'online_status_changed',
     IMAGE_SWIPED: 'image_swiped',
     CONNECTION_PROFILE_LOADED: 'connection_profile_loaded',

         this.swipes = [];
         /** @type {import('./scripts/logprobs.js').TokenLogprobs[]} */
         this.messageLogprobs = [];
 +        this.toolCalls = [];
+    }
     #checkDomElements(messageId) {

+    }
     /**
      * @returns {Generator<{ text: string, swipes: string[], logprobs: import('./scripts/logprobs.js').TokenLogprobs, toolCalls: any[] }, void, void>}
      */
     *nullStreamingGeneration() {
         throw new Error('Generation function for streaming is not hooked up');

         try {
             const sw = new Stopwatch(1000 / power_user.streaming_fps);
             const timestamps = [];
             for await (const { text, swipes, logprobs, toolCalls } of this.generator()) {
                 timestamps.push(Date.now());
                 if (this.isStopped) {
                     return;
+                }
 +                this.toolCalls = toolCalls;
                 this.result = text;
                 this.swipes = Array.from(swipes ?? []);
                 if (logprobs) {

                 getMessage = continue_mag + getMessage;
+            }
 +            if (ToolManager.isFunctionCallingSupported() && Array.isArray(streamingProcessor.toolCalls) && streamingProcessor.toolCalls.length) {
 +                const invocations = await ToolManager.checkFunctionToolCalls(streamingProcessor.toolCalls);
 +                if (invocations.length) {
 +                    const lastMessage = chat[chat.length - 1];
 +                    const shouldDeleteMessage = ['', '...'].includes(lastMessage?.mes) && ['', '...'].includes(streamingProcessor.result);
 +                    if (shouldDeleteMessage) {
 +                        await deleteLastMessage();
 +                        streamingProcessor = null;
 +                    }
 +                    ToolManager.saveFunctionToolInvocations(invocations);
 +                    return Generate(type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName }, dryRun);
 +                }
 +            }
++
             if (streamingProcessor && !streamingProcessor.isStopped && streamingProcessor.isFinished) {
                 await streamingProcessor.onFinishStreaming(streamingProcessor.messageId, getMessage);
                 streamingProcessor = null;

             throw new Error(data?.response);
+        }
 +        if (ToolManager.isFunctionCallingSupported()) {
 +            const invocations = await ToolManager.checkFunctionToolCalls(data);
 +            if (invocations.length) {
 +                ToolManager.saveFunctionToolInvocations(invocations);
 +                return Generate(type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName }, dryRun);
 +            }
 +        }
++
         //const getData = await response.json();
         let getMessage = extractMessageFromData(data);
         let title = extractTitleFromData(data);

             if (menu_type !== 'create') {
                 await createOrEditCharacter();
+            }
         },
     });
     for (let index = 0; index < getArray().length; index++) {

         registerHelper: () => { },
         registerMacro: MacrosParser.registerMacro.bind(MacrosParser),
         unregisterMacro: MacrosParser.unregisterMacro.bind(MacrosParser),
 +        registerFunctionTool: ToolManager.registerFunctionTool.bind(ToolManager),
 +        unregisterFunctionTool: ToolManager.unregisterFunctionTool.bind(ToolManager),
         registerDebugFunction: registerDebugFunction,
         /** @deprecated Use renderExtensionTemplateAsync instead. */
         renderExtensionTemplate: renderExtensionTemplate,

public/scripts/extensions/expressions/index.js+1 -51 Viewed

 import { SlashCommandParser } from '../../slash-commands/SlashCommandParser.js';
 import { SlashCommand } from '../../slash-commands/SlashCommand.js';
 import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from '../../slash-commands/SlashCommandArgument.js';
 -import { isFunctionCallingSupported } from '../../openai.js';
 import { SlashCommandEnumValue, enumTypes } from '../../slash-commands/SlashCommandEnumValue.js';
 import { commonEnumProviders } from '../../slash-commands/SlashCommandCommonEnumsProvider.js';
 import { slashCommandReturnHelper } from '../../slash-commands/SlashCommandReturnHelper.js';

 const STREAMING_UPDATE_INTERVAL = 10000;
 const TALKINGCHECK_UPDATE_INTERVAL = 500;
 const DEFAULT_FALLBACK_EXPRESSION = 'joy';
 -const FUNCTION_NAME = 'set_emotion';
 const DEFAULT_LLM_PROMPT = 'Ignore previous instructions. Classify the emotion of the last message. Output just one word, e.g. "joy" or "anger". Choose only one of the following labels: {{labels}}';
 const DEFAULT_EXPRESSIONS = [
     'talkinghead',

         return '';
+    }
 -    if (isFunctionCallingSupported()) {
 -        return '';
 -    }
+-
     const labelsString = labels.map(x => `"${x}"`).join(', ');
     const prompt = substituteParamsExtended(String(extension_settings.expressions.llmPrompt), { labels: labelsString });
     return prompt;

     throw new Error('Could not parse emotion response ' + emotionResponse);
+}
 -/**
 - * Registers the function tool for the LLM API.
 - * @param {FunctionToolRegister} args Function tool register arguments.
 - */
 -function onFunctionToolRegister(args) {
 -    if (inApiCall && extension_settings.expressions.api === EXPRESSION_API.llm && isFunctionCallingSupported()) {
 -        // Only trigger on quiet mode
 -        if (args.type !== 'quiet') {
 -            return;
 -        }
+-
 -        const emotions = DEFAULT_EXPRESSIONS.filter((e) => e != 'talkinghead');
 -        const jsonSchema = {
 -            $schema: 'http://json-schema.org/draft-04/schema#',
 -            type: 'object',
 -            properties: {
 -                emotion: {
 -                    type: 'string',
 -                    enum: emotions,
 -                    description: `One of the following: ${JSON.stringify(emotions)}`,
 -                },
 -            },
 -            required: [
 -                'emotion',
 -            ],
 -        };
 -        args.registerFunctionTool(
 -            FUNCTION_NAME,
 -            substituteParams('Sets the label that best describes the current emotional state of {{char}}. Only select one of the enumerated values.'),
 -            jsonSchema,
 -            true,
 -        );
 -    }
 -}
+-
 function onTextGenSettingsReady(args) {
     // Only call if inside an API call
     if (inApiCall && extension_settings.expressions.api === EXPRESSION_API.llm && isJsonSchemaSupported()) {

                 const expressionsList = await getExpressionsList();
                 const prompt = substituteParamsExtended(customPrompt, { labels: expressionsList }) || await getLlmPrompt(expressionsList);
 -                let functionResult = null;
                 eventSource.once(event_types.TEXT_COMPLETION_SETTINGS_READY, onTextGenSettingsReady);
 -                eventSource.once(event_types.LLM_FUNCTION_TOOL_REGISTER, onFunctionToolRegister);
 -                eventSource.once(event_types.LLM_FUNCTION_TOOL_CALL, (/** @type {FunctionToolCall} */ args) => {
 -                    if (args.name !== FUNCTION_NAME) {
 -                        return;
 -                    }
+-
 -                    functionResult = args?.arguments;
 -                });
                 const emotionResponse = await generateRaw(text, main_api, false, false, prompt);
                 return parseLlmResponse(functionResult || emotionResponse, expressionsList);
+            }
             // Extras
             default: {

public/scripts/openai.js+8 -141 Viewed

 import { SlashCommandEnumValue } from './slash-commands/SlashCommandEnumValue.js';
 import { Popup, POPUP_RESULT } from './popup.js';
 import { t } from './i18n.js';
 +import { ToolManager } from './tool-calling.js';
 export {
     openai_messages_count,

@@ -1863,8 +1864,8 @@ async function sendOpenAIRequest(type, messages, signal) {
1863	1864	generate_data['seed'] = oai_settings.seed;
1864	1865	}
1865	1866
1866	1867	if (isFunctionCallingSupported()!canMultiSwipe && !streamToolManager.isFunctionCallingSupported()) {
1867	1868	await registerFunctionToolsToolManager.registerFunctionToolsOpenAI(type, generate_data);
1868	1869	}
1869	1870
1870	1871	if (isOAI && oai_settings.openai_model.startsWith('o1-')) {

         return async function* streamData() {
             let text = '';
             const swipes = [];
 +            const toolCalls = [];
             while (true) {
                 const { done, value } = await reader.read();
                 if (done) return;

@@ -1926,7 +1928,9 @@ async function sendOpenAIRequest(type, messages, signal) {
1926	1928	text += getStreamingReply(parsed);
1927	1929	}
1928	1930
1929		- yield { text, swipes: swipes, logprobs: parseChatCompletionLogprobs(parsed) };
	1931	+ ToolManager.parseToolCalls(toolCalls, parsed);
	1932	+
	1933	+ yield { text, swipes: swipes, logprobs: parseChatCompletionLogprobs(parsed), toolCalls: toolCalls };
1930	1934	}
1931	1935	};
1932	1936	}

             delay(1).then(() => saveLogprobsForActiveMessage(logprobs, null));
+        }
 -        if (isFunctionCallingSupported()) {
 -            await checkFunctionToolCalls(data);
 -        }
+-
         return data;
+    }
+}
 -/**
 - * Register function tools for the next chat completion request.
 - * @param {string} type Generation type
 - * @param {object} data Generation data
 - */
 -async function registerFunctionTools(type, data) {
 -    let toolChoice = 'auto';
 -    const tools = [];
+-
 -    /**
 -     * @type {registerFunctionTool}
 -     */
 -    const registerFunctionTool = (name, description, parameters, required) => {
 -        tools.push({
 -            type: 'function',
 -            function: {
 -                name,
 -                description,
 -                parameters,
 -            },
 -        });
+-
 -        if (required) {
 -            toolChoice = 'required';
 -        }
 -    };
+-
 -    /**
 -     * @type {FunctionToolRegister}
 -     */
 -    const args = {
 -        type,
 -        data,
 -        registerFunctionTool,
 -    };
+-
 -    await eventSource.emit(event_types.LLM_FUNCTION_TOOL_REGISTER, args);
+-
 -    if (tools.length) {
 -        console.log('Registered function tools:', tools);
+-
 -        data['tools'] = tools;
 -        data['tool_choice'] = toolChoice;
 -    }
 -}
+-
 -async function checkFunctionToolCalls(data) {
 -    const oaiCompat = [
 -        chat_completion_sources.OPENAI,
 -        chat_completion_sources.CUSTOM,
 -        chat_completion_sources.MISTRALAI,
 -        chat_completion_sources.OPENROUTER,
 -        chat_completion_sources.GROQ,
 -    ];
 -    if (oaiCompat.includes(oai_settings.chat_completion_source)) {
 -        if (!Array.isArray(data?.choices)) {
 -            return;
 -        }
+-
 -        // Find a choice with 0-index
 -        const choice = data.choices.find(choice => choice.index === 0);
+-
 -        if (!choice) {
 -            return;
 -        }
+-
 -        const toolCalls = choice.message.tool_calls;
+-
 -        if (!Array.isArray(toolCalls)) {
 -            return;
 -        }
+-
 -        for (const toolCall of toolCalls) {
 -            if (typeof toolCall.function !== 'object') {
 -                continue;
 -            }
+-
 -            /** @type {FunctionToolCall} */
 -            const args = toolCall.function;
 -            console.log('Function tool call:', toolCall);
 -            await eventSource.emit(event_types.LLM_FUNCTION_TOOL_CALL, args);
 -        }
 -    }
+-
 -    if ([chat_completion_sources.CLAUDE].includes(oai_settings.chat_completion_source)) {
 -        if (!Array.isArray(data?.content)) {
 -            return;
 -        }
+-
 -        for (const content of data.content) {
 -            if (content.type === 'tool_use') {
 -                /** @type {FunctionToolCall} */
 -                const args = { name: content.name, arguments: JSON.stringify(content.input) };
 -                await eventSource.emit(event_types.LLM_FUNCTION_TOOL_CALL, args);
 -            }
 -        }
 -    }
+-
 -    if ([chat_completion_sources.COHERE].includes(oai_settings.chat_completion_source)) {
 -        if (!Array.isArray(data?.tool_calls)) {
 -            return;
 -        }
+-
 -        for (const toolCall of data.tool_calls) {
 -            /** @type {FunctionToolCall} */
 -            const args = { name: toolCall.name, arguments: JSON.stringify(toolCall.parameters) };
 -            console.log('Function tool call:', toolCall);
 -            await eventSource.emit(event_types.LLM_FUNCTION_TOOL_CALL, args);
 -        }
 -    }
 -}
+-
 -export function isFunctionCallingSupported() {
 -    if (main_api !== 'openai') {
 -        return false;
 -    }
+-
 -    if (!oai_settings.function_calling) {
 -        return false;
 -    }
+-
 -    const supportedSources = [
 -        chat_completion_sources.OPENAI,
 -        chat_completion_sources.COHERE,
 -        chat_completion_sources.CUSTOM,
 -        chat_completion_sources.MISTRALAI,
 -        chat_completion_sources.CLAUDE,
 -        chat_completion_sources.OPENROUTER,
 -        chat_completion_sources.GROQ,
 -    ];
 -    return supportedSources.includes(oai_settings.chat_completion_source);
 -}
+-
 function getStreamingReply(data) {
     if (oai_settings.chat_completion_source == chat_completion_sources.CLAUDE) {
         return data?.delta?.text || '';

             $('#openai_max_context').attr('max', max_32k);
         } else if (value === 'text-bison-001') {
             $('#openai_max_context').attr('max', max_8k);
                     // The ultra endpoints are possibly dead:
         } else if (value.includes('gemini-1.0-ultra') || value === 'gemini-ultra') {
             $('#openai_max_context').attr('max', max_32k);
         } else {

public/scripts/tool-calling.js+381 -0 Viewed

 +import { chat, main_api } from '../script.js';
 +import { chat_completion_sources, oai_settings } from './openai.js';
++
 +/**
 + * @typedef {object} ToolInvocation
 + * @property {string} id - A unique identifier for the tool invocation.
 + * @property {string} name - The name of the tool.
 + * @property {string} parameters - The parameters for the tool invocation.
 + * @property {string} result - The result of the tool invocation.
 + */
++
 +/**
 + * A class that represents a tool definition.
 + */
 +class ToolDefinition {
 +    /**
 +     * A unique name for the tool.
 +     * @type {string}
 +     */
 +    #name;
++
 +    /**
 +     * A description of what the tool does.
 +     * @type {string}
 +     */
 +    #description;
++
 +    /**
 +     * A JSON schema for the parameters that the tool accepts.
 +     * @type {object}
 +     */
 +    #parameters;
++
 +    /**
 +     * A function that will be called when the tool is executed.
 +     * @type {function}
 +     */
 +    #action;
++
 +    /**
 +     * Creates a new ToolDefinition.
 +     * @param {string} name A unique name for the tool.
 +     * @param {string} description A description of what the tool does.
 +     * @param {object} parameters A JSON schema for the parameters that the tool accepts.
 +     * @param {function} action A function that will be called when the tool is executed.
 +     */
 +    constructor(name, description, parameters, action) {
 +        this.#name = name;
 +        this.#description = description;
 +        this.#parameters = parameters;
 +        this.#action = action;
 +    }
++
 +    /**
 +     * Converts the ToolDefinition to an OpenAI API representation
 +     * @returns {object} OpenAI API representation of the tool.
 +     */
 +    toFunctionOpenAI() {
 +        return {
 +            type: 'function',
 +            function: {
 +                name: this.#name,
 +                description: this.#description,
 +                parameters: this.#parameters,
 +            },
 +        };
 +    }
++
 +    /**
 +     * Invokes the tool with the given parameters.
 +     * @param {object} parameters The parameters to pass to the tool.
 +     * @returns {Promise<any>} The result of the tool's action function.
 +     */
 +    async invoke(parameters) {
 +        return await this.#action(parameters);
 +    }
 +}
++
 +/**
 + * A class that manages the registration and invocation of tools.
 + */
 +export class ToolManager {
 +    /**
 +     * A map of tool names to tool definitions.
 +     * @type {Map<string, ToolDefinition>}
 +     */
 +    static #tools = new Map();
++
 +    /**
 +     * Returns an Array of all tools that have been registered.
 +     * @type {ToolDefinition[]}
 +     */
 +    static get tools() {
 +        return Array.from(this.#tools.values());
 +    }
++
 +    /**
 +     * Registers a new tool with the tool registry.
 +     * @param {string} name The name of the tool.
 +     * @param {string} description A description of what the tool does.
 +     * @param {object} parameters A JSON schema for the parameters that the tool accepts.
 +     * @param {function} action A function that will be called when the tool is executed.
 +     */
 +    static registerFunctionTool(name, description, parameters, action) {
 +        if (this.#tools.has(name)) {
 +            console.warn(`A tool with the name "${name}" has already been registered. The definition will be overwritten.`);
 +        }
++
 +        const definition = new ToolDefinition(name, description, parameters, action);
 +        this.#tools.set(name, definition);
 +    }
++
 +    /**
 +     * Removes a tool from the tool registry.
 +     * @param {string} name The name of the tool to unregister.
 +     */
 +    static unregisterFunctionTool(name) {
 +        if (!this.#tools.has(name)) {
 +            console.warn(`No tool with the name "${name}" has been registered.`);
 +            return;
 +        }
++
 +        this.#tools.delete(name);
 +    }
++
 +    /**
 +     * Invokes a tool by name. Returns the result of the tool's action function.
 +     * @param {string} name The name of the tool to invoke.
 +     * @param {object} parameters Function parameters. For example, if the tool requires a "name" parameter, you would pass {name: "value"}.
 +     * @returns {Promise<string|null>} The result of the tool's action function. If an error occurs, null is returned. Non-string results are JSON-stringified.
 +     */
 +    static async invokeFunctionTool(name, parameters) {
 +        try {
 +            if (!this.#tools.has(name)) {
 +                throw new Error(`No tool with the name "${name}" has been registered.`);
 +            }
++
 +            const invokeParameters = typeof parameters === 'string' ? JSON.parse(parameters) : parameters;
 +            const tool = this.#tools.get(name);
 +            const result = await tool.invoke(invokeParameters);
 +            return typeof result === 'string' ? result : JSON.stringify(result);
 +        } catch (error) {
 +            console.error(`An error occurred while invoking the tool "${name}":`, error);
 +            return null;
 +        }
 +    }
++
 +    /**
 +     * Register function tools for the next chat completion request.
 +     * @param {object} data Generation data
 +     */
 +    static async registerFunctionToolsOpenAI(data) {
 +        const tools = [];
++
 +        for (const tool of ToolManager.tools) {
 +            tools.push(tool.toFunctionOpenAI());
 +        }
++
 +        if (tools.length) {
 +            console.log('Registered function tools:', tools);
++
 +            data['tools'] = tools;
 +            data['tool_choice'] = 'auto';
 +        }
 +    }
++
 +    /**
 +     * Utility function to parse tool calls from a parsed response.
 +     * @param {any[]} toolCalls The tool calls to update.
 +     * @param {any} parsed The parsed response from the OpenAI API.
 +     * @returns {void}
 +     */
 +    static parseToolCalls(toolCalls, parsed) {
 +        if (!Array.isArray(parsed?.choices)) {
 +            return;
 +        }
 +        for (const choice of parsed.choices) {
 +            const choiceIndex = (typeof choice.index === 'number') ? choice.index : null;
 +            const choiceDelta = choice.delta;
++
 +            if (choiceIndex === null || !choiceDelta) {
 +                continue;
 +            }
++
 +            const toolCallDeltas = choiceDelta?.tool_calls;
++
 +            if (!Array.isArray(toolCallDeltas)) {
 +                continue;
 +            }
++
 +            if (!Array.isArray(toolCalls[choiceIndex])) {
 +                toolCalls[choiceIndex] = [];
 +            }
++
 +            for (const toolCallDelta of toolCallDeltas) {
 +                const toolCallIndex = (typeof toolCallDelta?.index === 'number') ? toolCallDelta.index : null;
++
 +                if (toolCallIndex === null) {
 +                    continue;
 +                }
++
 +                if (toolCalls[choiceIndex][toolCallIndex] === undefined) {
 +                    toolCalls[choiceIndex][toolCallIndex] = {};
 +                }
++
 +                const targetToolCall = toolCalls[choiceIndex][toolCallIndex];
++
 +                ToolManager.#applyToolCallDelta(targetToolCall, toolCallDelta);
 +            }
 +        }
 +    }
++
 +    static #applyToolCallDelta(target, delta) {
 +        for (const key in delta) {
 +            if (!delta.hasOwnProperty(key)) continue;
++
 +            const deltaValue = delta[key];
 +            const targetValue = target[key];
++
 +            if (deltaValue === null || deltaValue === undefined) {
 +                target[key] = deltaValue;
 +                continue;
 +            }
++
 +            if (typeof deltaValue === 'string') {
 +                if (typeof targetValue === 'string') {
 +                    // Concatenate strings
 +                    target[key] = targetValue + deltaValue;
 +                } else {
 +                    target[key] = deltaValue;
 +                }
 +            } else if (typeof deltaValue === 'object' && !Array.isArray(deltaValue)) {
 +                if (typeof targetValue !== 'object' || targetValue === null || Array.isArray(targetValue)) {
 +                    target[key] = {};
 +                }
 +                // Recursively apply deltas to nested objects
 +                ToolManager.#applyToolCallDelta(target[key], deltaValue);
 +            } else {
 +                // Assign other types directly
 +                target[key] = deltaValue;
 +            }
 +        }
 +    }
++
 +    static isFunctionCallingSupported() {
 +        if (main_api !== 'openai') {
 +            return false;
 +        }
++
 +        if (!oai_settings.function_calling) {
 +            return false;
 +        }
++
 +        const supportedSources = [
 +            chat_completion_sources.OPENAI,
 +            //chat_completion_sources.COHERE,
 +            chat_completion_sources.CUSTOM,
 +            chat_completion_sources.MISTRALAI,
 +            //chat_completion_sources.CLAUDE,
 +            chat_completion_sources.OPENROUTER,
 +            chat_completion_sources.GROQ,
 +        ];
 +        return supportedSources.includes(oai_settings.chat_completion_source);
 +    }
++
 +    static #getToolCallsFromData(data) {
 +        // Parsed tool calls from streaming data
 +        if (Array.isArray(data) && data.length > 0) {
 +            return data[0];
 +        }
++
 +        // Parsed tool calls from non-streaming data
 +        if (!Array.isArray(data?.choices)) {
 +            return;
 +        }
++
 +        // Find a choice with 0-index
 +        const choice = data.choices.find(choice => choice.index === 0);
++
 +        if (!choice) {
 +            return;
 +        }
++
 +        return choice.message.tool_calls;
 +    }
++
 +    /**
 +     * Check for function tool calls in the response data and invoke them.
 +     * @param {any} data Reply data
 +     * @returns {Promise<ToolInvocation[]>} Successful tool invocations
 +     */
 +    static async checkFunctionToolCalls(data) {
 +        if (!ToolManager.isFunctionCallingSupported()) {
 +            return [];
 +        }
++
 +        /** @type {ToolInvocation[]} */
 +        const invocations = [];
 +        const toolCalls = ToolManager.#getToolCallsFromData(data);
 +        const oaiCompat = [
 +            chat_completion_sources.OPENAI,
 +            chat_completion_sources.CUSTOM,
 +            chat_completion_sources.MISTRALAI,
 +            chat_completion_sources.OPENROUTER,
 +            chat_completion_sources.GROQ,
 +        ];
++
 +        if (oaiCompat.includes(oai_settings.chat_completion_source)) {
 +            if (!Array.isArray(toolCalls)) {
 +                return;
 +            }
++
 +            for (const toolCall of toolCalls) {
 +                if (typeof toolCall.function !== 'object') {
 +                    continue;
 +                }
++
 +                console.log('Function tool call:', toolCall);
 +                const id = toolCall.id;
 +                const parameters = toolCall.function.arguments;
 +                const name = toolCall.function.name;
++
 +                toastr.info('Invoking function tool: ' + name);
 +                const result = await ToolManager.invokeFunctionTool(name, parameters);
 +                toastr.info('Function tool result: ' + result);
++
 +                // Save a successful invocation
 +                if (result) {
 +                    invocations.push({ id, name, result, parameters });
 +                }
 +            }
 +        }
++
 +        /*
 +        if ([chat_completion_sources.CLAUDE].includes(oai_settings.chat_completion_source)) {
 +            if (!Array.isArray(data?.content)) {
 +                return;
 +            }
++
 +            for (const content of data.content) {
 +                if (content.type === 'tool_use') {
 +                    const args = { name: content.name, arguments: JSON.stringify(content.input) };
 +                }
 +            }
 +        }
 +        */
++
 +        /*
 +        if ([chat_completion_sources.COHERE].includes(oai_settings.chat_completion_source)) {
 +            if (!Array.isArray(data?.tool_calls)) {
 +                return;
 +            }
++
 +            for (const toolCall of data.tool_calls) {
 +                const args = { name: toolCall.name, arguments: JSON.stringify(toolCall.parameters) };
 +                console.log('Function tool call:', toolCall);
 +            }
 +        }
 +        */
++
 +        return invocations;
 +    }
++
 +    /**
 +     * Saves function tool invocations to the last user chat message extra metadata.
 +     * @param {ToolInvocation[]} invocations
 +     */
 +    static saveFunctionToolInvocations(invocations) {
 +        for (let index = chat.length - 1; index >= 0; index--) {
 +            const message = chat[index];
 +            if (message.is_user) {
 +                if (!message.extra || typeof message.extra !== 'object') {
 +                    message.extra = {};
 +                }
 +                message.extra.tool_invocations = invocations;
 +                debugger;
 +                break;
 +            }
 +        }
 +    }
 +}

src/endpoints/backends/chat-completions.js+7 -15 Viewed

                 ? [{ type: 'text', text: convertedPrompt.systemPrompt, cache_control: { type: 'ephemeral' } }]
                 : convertedPrompt.systemPrompt;
+        }
 +        /*
         if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
             // Claude doesn't do prefills on function calls, and doesn't allow empty messages
             if (convertedPrompt.messages.length && convertedPrompt.messages[convertedPrompt.messages.length - 1].role === 'assistant') {
                 convertedPrompt.messages.push({ role: 'user', content: '.' });
+            }
             additionalHeaders['anthropic-beta'] = 'tools-2024-05-16';
             requestBody.tool_choice = { type: request.body.tool_choice === 'required' ? 'any' : 'auto' };
             requestBody.tools = request.body.tools
                 .filter(tool => tool.type === 'function')
                 .map(tool => tool.function)
                 .map(fn => ({ name: fn.name, description: fn.description, input_schema: fn.parameters }));
+        }
 +        */
         if (enableSystemPromptCache) {
             additionalHeaders['anthropic-beta'] = 'prompt-caching-2024-07-31';
+        }

         if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
             requestBody['tools'] = request.body.tools;
             requestBody['tool_choice'] = request.body.tool_choice === 'required' ? 'any' : 'auto';
+        }
         const config = {

             });
+        }
 +        /*
         if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
             tools.push(...convertCohereTools(request.body.tools));
             // Can't have both connectors and tools in the same request
             connectors.splice(0, connectors.length);
+        }
 +        */
         // https://docs.cohere.com/reference/chat
         const requestBody = {

         apiKey = readSecret(request.user.directories, SECRET_KEYS.GROQ);
         headers = {};
         bodyParams = {};
+-
 -        // 'required' tool choice is not supported by Groq
 -        if (request.body.tool_choice === 'required') {
 -            if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
 -                request.body.tool_choice = request.body.tools.length > 1
 -                    ? 'auto' :
 -                    { type: 'function', function: { name: request.body.tools[0]?.function?.name } };
+-
 -            } else {
 -                request.body.tool_choice = 'none';
 -            }
 -        }
     } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.ZEROONEAI) {
         apiUrl = API_01AI;
         apiKey = readSecret(request.user.directories, SECRET_KEYS.ZEROONEAI);

         controller.abort();
     });
 -    if (!isTextCompletion) {
 +    if (!isTextCompletion && Array.isArray(request.body.tools) && request.body.tools.length > 0) {
         bodyParams['tools'] = request.body.tools;
         bodyParams['tool_choice'] = request.body.tool_choice;
+    }