Claude: Use multi-part system prompt, cache tools

6185974e17a77d7d5caacd3ff2853413283b5ac7

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: 3b4a455ef845ca6682cbd07e6c97be199a8a1399
2 files changed, +14 -8Showing whitespace changes
src/endpoints/backends/chat-completions.js+11 -5
@@ -87,7 +87,7 @@ async function sendClaudeRequest(request, response) {
87 const apiUrl = new URL(request.body.reverse_proxy || API_CLAUDE).toString();87 const apiUrl = new URL(request.body.reverse_proxy || API_CLAUDE).toString();
88 const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.CLAUDE);88 const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.CLAUDE);
89 const divider = '-'.repeat(process.stdout.columns);89 const divider = '-'.repeat(process.stdout.columns);
90 const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false);90 const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false) && request.body.model.startsWith('claude-3');
9191
92 if (!apiKey) {92 if (!apiKey) {
93 console.log(color.red(`Claude API key is missing.\n${divider}`));93 console.log(color.red(`Claude API key is missing.\n${divider}`));
@@ -110,7 +110,7 @@ async function sendClaudeRequest(request, response) {
110 }110 }
111111
112 const requestBody = {112 const requestBody = {
113 /** @type {any} */ system: '',113 /** @type {any} */ system: [],
114 messages: convertedPrompt.messages,114 messages: convertedPrompt.messages,
115 model: request.body.model,115 model: request.body.model,
116 max_tokens: request.body.max_tokens,116 max_tokens: request.body.max_tokens,
@@ -121,9 +121,11 @@ async function sendClaudeRequest(request, response) {
121 stream: request.body.stream,121 stream: request.body.stream,
122 };122 };
123 if (useSystemPrompt) {123 if (useSystemPrompt) {
124 requestBody.system = enableSystemPromptCache124 if (enableSystemPromptCache && Array.isArray(convertedPrompt.systemPrompt) && convertedPrompt.systemPrompt.length) {
125 ? [{ type: 'text', text: convertedPrompt.systemPrompt, cache_control: { type: 'ephemeral' } }]125 convertedPrompt.systemPrompt[convertedPrompt.systemPrompt.length - 1]['cache_control'] = { type: 'ephemeral' };
126 : convertedPrompt.systemPrompt;126 }
127
128 requestBody.system = convertedPrompt.systemPrompt;
127 } else {129 } else {
128 delete requestBody.system;130 delete requestBody.system;
129 }131 }
@@ -138,6 +140,10 @@ async function sendClaudeRequest(request, response) {
138 .filter(tool => tool.type === 'function')140 .filter(tool => tool.type === 'function')
139 .map(tool => tool.function)141 .map(tool => tool.function)
140 .map(fn => ({ name: fn.name, description: fn.description, input_schema: fn.parameters }));142 .map(fn => ({ name: fn.name, description: fn.description, input_schema: fn.parameters }));
143
144 if (enableSystemPromptCache && requestBody.tools.length) {
145 requestBody.tools[requestBody.tools.length - 1]['cache_control'] = { type: 'ephemeral' };
146 }
141 }147 }
142 if (enableSystemPromptCache) {148 if (enableSystemPromptCache) {
143 additionalHeaders['anthropic-beta'] = 'prompt-caching-2024-07-31';149 additionalHeaders['anthropic-beta'] = 'prompt-caching-2024-07-31';
src/prompt-converters.js+3 -3
@@ -95,7 +95,7 @@ function convertClaudePrompt(messages, addAssistantPostfix, addAssistantPrefill,
95 * @param {string} userName User name95 * @param {string} userName User name
96 */96 */
97function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFix, charName = '', userName = '') {97function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFix, charName = '', userName = '') {
98 let systemPrompt = '';98 let systemPrompt = [];
99 if (useSysPrompt) {99 if (useSysPrompt) {
100 // Collect all the system messages up until the first instance of a non-system message, and then remove them from the messages array.100 // Collect all the system messages up until the first instance of a non-system message, and then remove them from the messages array.
101 let i;101 let i;
@@ -114,7 +114,7 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFi
114 messages[i].content = `${charName}: ${messages[i].content}`;114 messages[i].content = `${charName}: ${messages[i].content}`;
115 }115 }
116 }116 }
117 systemPrompt += `${messages[i].content}\n\n`;117 systemPrompt.push({ type: 'text', text: messages[i].content });
118 }118 }
119119
120 messages.splice(0, i);120 messages.splice(0, i);
@@ -246,7 +246,7 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFi
246 }246 }
247 });247 });
248248
249 return { messages: mergedMessages, systemPrompt: systemPrompt.trim() };249 return { messages: mergedMessages, systemPrompt: systemPrompt };
250}250}
251251
252/**252/**