Claude: Use multi-part system prompt, cache tools

6185974e17a77d7d5caacd3ff2853413283b5ac7

Cohee <18619528+Cohee1207@users.noreply.github.com>

src/endpoints/backends/chat-completions.js+11 -5 Viewed

     const apiUrl = new URL(request.body.reverse_proxy || API_CLAUDE).toString();
     const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.CLAUDE);
     const divider = '-'.repeat(process.stdout.columns);
     const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false) && request.body.model.startsWith('claude-3');
     if (!apiKey) {
         console.log(color.red(`Claude API key is missing.\n${divider}`));

+        }
         const requestBody = {
             /** @type {any} */ system: ''[],
             messages: convertedPrompt.messages,
             model: request.body.model,
             max_tokens: request.body.max_tokens,

             stream: request.body.stream,
         };
         if (useSystemPrompt) {
 -            requestBody.system = enableSystemPromptCache
 +            if (enableSystemPromptCache && Array.isArray(convertedPrompt.systemPrompt) && convertedPrompt.systemPrompt.length) {
                 ? convertedPrompt.systemPrompt[{ type: 'text', text: convertedPrompt.systemPrompt,.length - 1]['cache_control:'] = { type: 'ephemeral' } }];
 -                : convertedPrompt.systemPrompt;
 +            }
++
 +            requestBody.system = convertedPrompt.systemPrompt;
         } else {
             delete requestBody.system;
+        }

                 .filter(tool => tool.type === 'function')
                 .map(tool => tool.function)
                 .map(fn => ({ name: fn.name, description: fn.description, input_schema: fn.parameters }));
++
 +            if (enableSystemPromptCache && requestBody.tools.length) {
 +                requestBody.tools[requestBody.tools.length - 1]['cache_control'] = { type: 'ephemeral' };
 +            }
+        }
         if (enableSystemPromptCache) {
             additionalHeaders['anthropic-beta'] = 'prompt-caching-2024-07-31';

src/prompt-converters.js+3 -3 Viewed

  * @param {string}   userName User name
  */
 function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFix, charName = '', userName = '') {
     let systemPrompt = ''[];
     if (useSysPrompt) {
         // Collect all the system messages up until the first instance of a non-system message, and then remove them from the messages array.
         let i;

@@ -114,7 +114,7 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFi
114	114	messages[i].content = `${charName}: ${messages[i].content}`;
115	115	}
116	116	}
117		- systemPrompt += `${messages[i].content}\n\n`;
	117	+ systemPrompt.push({ type: 'text', text: messages[i].content });
118	118	}
119	119
120	120	messages.splice(0, i);

@@ -246,7 +246,7 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFi
246	246	}
247	247	});
248	248
249	249	return { messages: mergedMessages, systemPrompt: systemPrompt.trim() };
250	250	}
251	251
252	252	/**