Claude: new prompt converter + non-streaming tools

c3c10a629e000f3d7479f9a50fbb6d28316acaa0

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: 559f1b81f79b2c4e2dc68538d29ead31ed738c36

public/scripts/tool-calling.js+54 -57 Viewed

         const supportedSources = [
             chat_completion_sources.OPENAI,
 -            //chat_completion_sources.COHERE,
             chat_completion_sources.CUSTOM,
             chat_completion_sources.MISTRALAI,
             //chat_completion_sources.CLAUDE,
             chat_completion_sources.OPENROUTER,
             chat_completion_sources.GROQ,
         ];

+        }
         // Parsed tool calls from non-streaming data
         if (!Array.isArray(data?.choices)) {
 -            return;
 +            // Find a choice with 0-index
 +            const choice = data.choices.find(choice => choice.index === 0);
++
 +            if (choice) {
 +                return choice.message.tool_calls;
 +            }
+        }
 -        // Find a choice with 0-index
 +        if (Array.isArray(data?.content)) {
 -        const choice = data.choices.find(choice => choice.index === 0);
 +            // Claude tool calls to OpenAI tool calls
 +            const content = data.content.filter(c => c.type === 'tool_use').map(c => {
 +                return {
 +                    id: c.id,
 +                    function: {
 +                        name: c.name,
 +                        arguments: c.input,
 +                    },
 +                };
 +            });
 -        if (!choice) {
 +            return content;
 -            return;
+        }
+-
 -        return choice.message.tool_calls;
+    }
     /**

             chat_completion_sources.GROQ,
         ];
         if (oaiCompatibleSources!Array.includesisArray(oai_settings.chat_completion_sourcetoolCalls)) {
 -            if (!Array.isArray(toolCalls)) {
 +            return result;
 -                return result;
 -            }
+-
 -            for (const toolCall of toolCalls) {
 -                if (typeof toolCall.function !== 'object') {
 -                    continue;
 -                }
+-
 -                console.log('Function tool call:', toolCall);
 -                const id = toolCall.id;
 -                const parameters = toolCall.function.arguments;
 -                const name = toolCall.function.name;
 -                const displayName = ToolManager.getDisplayName(name);
 -                result.hadToolCalls = true;
+-
 -                const message = ToolManager.formatToolCallMessage(name, parameters);
 -                const toast = message && toastr.info(message, 'Tool Calling', { timeOut: 0 });
 -                const toolResult = await ToolManager.invokeFunctionTool(name, parameters);
 -                toastr.clear(toast);
 -                console.log('Function tool result:', result);
+-
 -                // Save a successful invocation
 -                if (toolResult instanceof Error) {
 -                    result.errors.push(toolResult);
 -                    continue;
 -                }
+-
 -                const invocation = {
 -                    id,
 -                    displayName,
 -                    name,
 -                    parameters,
 -                    result: toolResult,
 -                };
 -                result.invocations.push(invocation);
 -            }
+        }
 -        /*
 +        for (const toolCall of toolCalls) {
 -        if ([chat_completion_sources.CLAUDE].includes(oai_settings.chat_completion_source)) {
 +            if (typeof toolCall.function !== 'object') {
 -            if (!Array.isArray(data?.content)) {
 +                continue;
 -                return;
+            }
 -            for (const content of data.content) {
 +            console.log('Function tool call:', toolCall);
 -                if (content.type === 'tool_use') {
 +            const id = toolCall.id;
 -                    const args = { name: content.name, arguments: JSON.stringify(content.input) };
 +            const parameters = toolCall.function.arguments;
 -                }
 +            const name = toolCall.function.name;
 +            const displayName = ToolManager.getDisplayName(name);
 +            result.hadToolCalls = true;
++
 +            const message = ToolManager.formatToolCallMessage(name, parameters);
 +            const toast = message && toastr.info(message, 'Tool Calling', { timeOut: 0 });
 +            const toolResult = await ToolManager.invokeFunctionTool(name, parameters);
 +            toastr.clear(toast);
 +            console.log('Function tool result:', result);
++
 +            // Save a successful invocation
 +            if (toolResult instanceof Error) {
 +                result.errors.push(toolResult);
 +                continue;
+            }
++
 +            const invocation = {
 +                id,
 +                displayName,
 +                name,
 +                parameters,
 +                result: toolResult,
 +            };
 +            result.invocations.push(invocation);
+        }
 -        */
         return result;
+    }

      * @param {ToolInvocation[]} invocations Successful tool invocations
      */
     static saveFunctionToolInvocations(invocations) {
 +        if (!Array.isArray(invocations) || invocations.length === 0) {
 +            return;
 +        }
         const message = {
             name: systemUserName,
             force_avatar: system_avatar,

src/endpoints/backends/chat-completions.js+0 -2 Viewed

         } else {
             delete requestBody.system;
+        }
 -        /*
         if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
             // Claude doesn't do prefills on function calls, and doesn't allow empty messages
             if (convertedPrompt.messages.length && convertedPrompt.messages[convertedPrompt.messages.length - 1].role === 'assistant') {

@@ -137,7 +136,6 @@ async function sendClaudeRequest(request, response) {
137	136	.map(tool => tool.function)
138	137	.map(fn => ({ name: fn.name, description: fn.description, input_schema: fn.parameters }));
139	138	}
140		- */
141	139	if (enableSystemPromptCache) {
142	140	additionalHeaders['anthropic-beta'] = 'prompt-caching-2024-07-31';
143	141	}

src/prompt-converters.js+87 -40 Viewed

             });
+        }
+    }
++
     // Now replace all further messages that have the role 'system' with the role 'user'. (or all if we're not using one)
     messages.forEach((message) => {
 +        if (message.role === 'assistant' && message.tool_calls) {
 +            message.content = message.tool_calls.map((tc) => ({
 +                type: 'tool_use',
 +                id: tc.id,
 +                name: tc.function.name,
 +                input: tc.function.arguments,
 +            }));
 +        }
++
 +        if (message.role === 'tool') {
 +            message.role = 'user';
 +            message.content = [{
 +                type: 'tool_result',
 +                tool_use_id: message.tool_call_id,
 +                content: message.content,
 +            }];
 +        }
++
         if (message.role === 'system') {
             if (userName && message.name === 'example_user') {
                 message.content = `${userName}: ${message.content}`;

                 message.content = `${charName}: ${message.content}`;
+            }
             message.role = 'user';
++
 +            // Delete name here so it doesn't get added later
 +            delete message.name;
 +        }
++
 +        // Convert everything to an array of it would be easier to work with
 +        if (typeof message.content === 'string') {
 +            // Take care of name properties since claude messages don't support them
 +            if (message.name) {
 +                message.content = `${message.name}: ${message.content}`;
 +            }
++
 +            message.content = [{ type: 'text', text: message.content }];
 +        } else if (Array.isArray(message.content)) {
 +            message.content = message.content.map((content) => {
 +                if (content.type === 'image_url') {
 +                    const imageEntry = content?.image_url;
 +                    const imageData = imageEntry?.url;
 +                    const mimeType = imageData?.split(';')?.[0].split(':')?.[1];
 +                    const base64Data = imageData?.split(',')?.[1];
++
 +                    return {
 +                        type: 'image',
 +                        source: {
 +                            type: 'base64',
 +                            media_type: mimeType,
 +                            data: base64Data,
 +                        },
 +                    };
 +                }
++
 +                if (content.type === 'text') {
 +                    if (message.name) {
 +                        content.text = `${message.name}: ${content.text}`;
 +                    }
++
 +                    return content;
 +                }
++
 +                return content;
 +            });
+        }
++
 +        // Remove offending properties
 +        delete message.name;
 +        delete message.tool_calls;
 +        delete message.tool_call_id;
     });
 +    // Images in assistant messages should be moved to the next user message
 +    for (let i = 0; i < messages.length; i++) {
 +        if (messages[i].role === 'assistant' && messages[i].content.some(c => c.type === 'image')) {
 +            // Find the next user message
 +            let j = i + 1;
 +            while (j < messages.length && messages[j].role !== 'user') {
 +                j++;
 +            }
++
 +            // Move the images
 +            if (j >= messages.length) {
 +                // If there is no user message after the assistant message, add a new one
 +                messages.splice(i + 1, 0, { role: 'user', content: [] });
 +            }
++
 +            messages[j].content.push(...messages[i].content.filter(c => c.type === 'image'));
 +            messages[i].content = messages[i].content.filter(c => c.type !== 'image');
 +        }
 +    }
++
     // Shouldn't be conditional anymore, messages api expects the last role to be user unless we're explicitly prefilling
     if (prefillString) {
         messages.push({
             role: 'assistant',
 +            // Dangling whitespace are not allowed for prefilling
             content: prefillString.trimEnd(),
         });
+    }

     // Also handle multi-modality, holy slop.
     let mergedMessages = [];
     messages.forEach((message) => {
 -        const imageEntry = message.content?.[1]?.image_url;
 -        const imageData = imageEntry?.url;
 -        const mimeType = imageData?.split(';')?.[0].split(':')?.[1];
 -        const base64Data = imageData?.split(',')?.[1];
+-
 -        // Take care of name properties since claude messages don't support them
 -        if (message.name) {
 -            if (Array.isArray(message.content)) {
 -                message.content[0].text = `${message.name}: ${message.content[0].text}`;
 -            } else {
 -                message.content = `${message.name}: ${message.content}`;
 -            }
 -            delete message.name;
 -        }
+-
         if (mergedMessages.length > 0 && mergedMessages[mergedMessages.length - 1].role === message.role) {
 -            if (Array.isArray(message.content)) {
 +            mergedMessages[mergedMessages.length - 1].content.push(...message.content);
 -                if (Array.isArray(mergedMessages[mergedMessages.length - 1].content)) {
 -                    mergedMessages[mergedMessages.length - 1].content[0].text += '\n\n' + message.content[0].text;
 -                } else {
 -                    mergedMessages[mergedMessages.length - 1].content += '\n\n' + message.content[0].text;
 -                }
 -            } else {
 -                if (Array.isArray(mergedMessages[mergedMessages.length - 1].content)) {
 -                    mergedMessages[mergedMessages.length - 1].content[0].text += '\n\n' + message.content;
 -                } else {
 -                    mergedMessages[mergedMessages.length - 1].content += '\n\n' + message.content;
 -                }
 -            }
         } else {
             mergedMessages.push(message);
+        }
 -        if (imageData) {
 -            mergedMessages[mergedMessages.length - 1].content = [
 -                { type: 'text', text: mergedMessages[mergedMessages.length - 1].content[0]?.text || mergedMessages[mergedMessages.length - 1].content },
 -                {
 -                    type: 'image', source: {
 -                        type: 'base64',
 -                        media_type: mimeType,
 -                        data: base64Data,
 -                    },
 -                },
 -            ];
 -        }
     });
     return { messages: mergedMessages, systemPrompt: systemPrompt.trim() };