Implement function tool calling for OpenAI

c94c06ed4dac3c4895e4afb5b0e6c1524829e49b

Cohee <18619528+Cohee1207@users.noreply.github.com>

public/script.js+2 -2 Viewed

             if (ToolManager.isFunctionCallingSupported() && Array.isArray(streamingProcessor.toolCalls) && streamingProcessor.toolCalls.length) {
                 const invocations = await ToolManager.checkFunctionToolCalls(streamingProcessor.toolCalls);
                 if (Array.isArray(invocations) && invocations.length) {
                     const lastMessage = chat[chat.length - 1];
                     const shouldDeleteMessage = ['', '...'].includes(lastMessage?.mes) && ['', '...'].includes(streamingProcessor.result);
                     if (shouldDeleteMessage) {

         if (ToolManager.isFunctionCallingSupported()) {
             const invocations = await ToolManager.checkFunctionToolCalls(data);
             if (Array.isArray(invocations) && invocations.length) {
                 ToolManager.saveFunctionToolInvocations(invocations);
                 return Generate(type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName }, dryRun);
+            }

public/scripts/openai.js+48 -4 Viewed

         if (role == 'user' && oai_settings.wrap_in_quotes) content = `"${content}"`;
         const name = chat[j]['name'];
         const image = chat[j]?.extra?.image;
 -        messages[i] = { 'role': role, 'content': content, name: name, 'image': image };
 +        const invocations = chat[j]?.extra?.tool_invocations;
 +        messages[i] = { 'role': role, 'content': content, name: name, 'image': image, 'invocations': invocations };
         j++;
+    }

+    }
     const imageInlining = isImageInliningSupported();
 +    const toolCalling = ToolManager.isFunctionCallingSupported();
     // Insert chat messages as long as there is budget available
     const chatPool = [...messages].reverse();

             await chatMessage.addImage(chatPrompt.image);
+        }
 +        if (toolCalling && Array.isArray(chatPrompt.invocations)) {
 +            /** @type {import('./tool-calling.js').ToolInvocation[]} */
 +            const invocations = chatPrompt.invocations.slice().reverse();
 +            const toolCallMessage = new Message('assistant', undefined, 'toolCall-' + chatMessage.identifier);
 +            toolCallMessage.setToolCalls(invocations);
 +            if (chatCompletion.canAfford(toolCallMessage)) {
 +                for (const invocation of invocations) {
 +                    const toolResultMessage = new Message('tool', invocation.result, invocation.id);
 +                    const canAfford = chatCompletion.canAfford(toolResultMessage) && chatCompletion.canAfford(toolCallMessage);
 +                    if (!canAfford) {
 +                        break;
 +                    }
 +                    chatCompletion.insertAtStart(toolResultMessage, 'chatHistory');
 +                }
 +                chatCompletion.insertAtStart(toolCallMessage, 'chatHistory');
 +            }
 +        }
++
         if (chatCompletion.canAfford(chatMessage)) {
             if (type === 'continue' && oai_settings.continue_prefill && chatPrompt === firstNonInjected) {
                 // in case we are using continue_prefill and the latest message is an assistant message, we want to prepend the users assistant prefill on the message

     content;
     /** @type {string} */
     name;
 +    /** @type {object} */
 +    tool_call = null;
     /**
      * @constructor

+        }
+    }
 +    /**
 +     * Reconstruct the message from a tool invocation.
 +     * @param {import('./tool-calling.js').ToolInvocation[]} invocations
 +     */
 +    setToolCalls(invocations) {
 +        this.tool_calls = invocations.map(i => ({
 +            id: i.id,
 +            type: 'function',
 +            function: {
 +                arguments: i.parameters,
 +                name: i.name,
 +            },
 +        }));
 +        this.tokens = tokenHandler.count({ role: this.role, tool_calls: JSON.stringify(this.tool_calls) });
 +    }
++
     setName(name) {
         this.name = name;
         this.tokens = tokenHandler.count({ role: this.role, content: this.content, name: this.name });

         this.checkTokenBudget(message, message.identifier);
         const index = this.findMessageIndex(identifier);
         if (message.content || message.tool_calls) {
             if ('start' === position) this.messages.collection[index].collection.unshift(message);
             else if ('end' === position) this.messages.collection[index].collection.push(message);
             else if (typeof position === 'number') this.messages.collection[index].collection.splice(position, 0, message);

         for (let item of this.messages.collection) {
             if (item instanceof MessageCollection) {
                 chat.push(...item.getChat());
             } else if (item instanceof Message && (item.content || item.tool_calls)) {
 -                const message = { role: item.role, content: item.content, ...(item.name ? { name: item.name } : {}) };
 +                const message = {
 +                    role: item.role,
 +                    content: item.content,
 +                    ...(item.name ? { name: item.name } : {}),
 +                    ...(item.tool_calls ? { tool_calls: item.tool_calls } : {}),
 +                    ...(item.role === 'tool' ? { tool_call_id: item.identifier } : {}),
 +                };
                 chat.push(message);
             } else {
                 this.log(`Skipping invalid or empty message in collection: ${JSON.stringify(item)}`);

public/scripts/tool-calling.js+2 -3 Viewed

         if (oaiCompat.includes(oai_settings.chat_completion_source)) {
             if (!Array.isArray(toolCalls)) {
                 return [];
+            }
             for (const toolCall of toolCalls) {

     /**
      * Saves function tool invocations to the last user chat message extra metadata.
      * @param {ToolInvocation[]} invocations Successful tool invocations
      */
     static saveFunctionToolInvocations(invocations) {
         for (let index = chat.length - 1; index >= 0; index--) {

@@ -373,7 +373,6 @@ export class ToolManager {
373	373	message.extra = {};
374	374	}
375	375	message.extra.tool_invocations = invocations;
376		- debugger;
377	376	break;
378	377	}
379	378	}