Claude: control cache TTL with config

560c6e8ff113d12f83d29c72cfaf5ce01e84c2f5

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: ed2e6fff6e13f909abb4ce98136fcdf40509e2bb
3 files changed, +16 -5Showing whitespace changes
default/config.yaml+4 -0
@@ -234,6 +234,10 @@ claude:
234 # should be ideal for most use cases.234 # should be ideal for most use cases.
235 # Any value other than a non-negative integer will be ignored and caching at depth will not be enabled.235 # Any value other than a non-negative integer will be ignored and caching at depth will not be enabled.
236 cachingAtDepth: -1236 cachingAtDepth: -1
237 # Use 1h TTL instead of the default 5m.
238 ## 5m: base price x 1.25
239 ## 1h: base price x 2
240 extendedTTL: false
237# -- GOOGLE GEMINI API CONFIGURATION --241# -- GOOGLE GEMINI API CONFIGURATION --
238gemini:242gemini:
239 # API endpoint version ("v1beta" or "v1alpha")243 # API endpoint version ("v1beta" or "v1alpha")
src/endpoints/backends/chat-completions.js+4 -3
@@ -154,6 +154,7 @@ async function sendClaudeRequest(request, response) {
154 const convertedPrompt = convertClaudeMessages(request.body.messages, request.body.assistant_prefill, useSystemPrompt, useTools, getPromptNames(request));154 const convertedPrompt = convertClaudeMessages(request.body.messages, request.body.assistant_prefill, useSystemPrompt, useTools, getPromptNames(request));
155 const useThinking = /^claude-(3-7|opus-4|sonnet-4)/.test(request.body.model) && Boolean(request.body.include_reasoning);155 const useThinking = /^claude-(3-7|opus-4|sonnet-4)/.test(request.body.model) && Boolean(request.body.include_reasoning);
156 const useWebSearch = /^claude-(3-5|3-7|opus-4|sonnet-4)/.test(request.body.model) && Boolean(request.body.enable_web_search);156 const useWebSearch = /^claude-(3-5|3-7|opus-4|sonnet-4)/.test(request.body.model) && Boolean(request.body.enable_web_search);
157 const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';
157 let fixThinkingPrefill = false;158 let fixThinkingPrefill = false;
158 // Add custom stop sequences159 // Add custom stop sequences
159 const stopSequences = [];160 const stopSequences = [];
@@ -174,7 +175,7 @@ async function sendClaudeRequest(request, response) {
174 };175 };
175 if (useSystemPrompt) {176 if (useSystemPrompt) {
176 if (enableSystemPromptCache && Array.isArray(convertedPrompt.systemPrompt) && convertedPrompt.systemPrompt.length) {177 if (enableSystemPromptCache && Array.isArray(convertedPrompt.systemPrompt) && convertedPrompt.systemPrompt.length) {
177 convertedPrompt.systemPrompt[convertedPrompt.systemPrompt.length - 1]['cache_control'] = { type: 'ephemeral', ttl: '1h' };178 convertedPrompt.systemPrompt[convertedPrompt.systemPrompt.length - 1]['cache_control'] = { type: 'ephemeral', ttl: cacheTTL };
178 }179 }
179180
180 requestBody.system = convertedPrompt.systemPrompt;181 requestBody.system = convertedPrompt.systemPrompt;
@@ -190,7 +191,7 @@ async function sendClaudeRequest(request, response) {
190 .map(fn => ({ name: fn.name, description: fn.description, input_schema: fn.parameters }));191 .map(fn => ({ name: fn.name, description: fn.description, input_schema: fn.parameters }));
191192
192 if (enableSystemPromptCache && requestBody.tools.length) {193 if (enableSystemPromptCache && requestBody.tools.length) {
193 requestBody.tools[requestBody.tools.length - 1]['cache_control'] = { type: 'ephemeral', ttl: '1h' };194 requestBody.tools[requestBody.tools.length - 1]['cache_control'] = { type: 'ephemeral', ttl: cacheTTL };
194 }195 }
195 }196 }
196197
@@ -203,7 +204,7 @@ async function sendClaudeRequest(request, response) {
203 }204 }
204205
205 if (cachingAtDepth !== -1) {206 if (cachingAtDepth !== -1) {
206 cachingAtDepthForClaude(convertedPrompt.messages, cachingAtDepth);207 cachingAtDepthForClaude(convertedPrompt.messages, cachingAtDepth, cacheTTL);
207 }208 }
208209
209 if (enableSystemPromptCache || cachingAtDepth !== -1) {210 if (enableSystemPromptCache || cachingAtDepth !== -1) {
src/prompt-converters.js+8 -2
@@ -854,7 +854,13 @@ export function convertTextCompletionPrompt(messages) {
854 return messageStrings.join('\n') + '\nassistant:';854 return messageStrings.join('\n') + '\nassistant:';
855}855}
856856
857export function cachingAtDepthForClaude(messages, cachingAtDepth) {857/**
858 * Append cache_control object to a Claude messages at depth. Directly modifies the messages array.
859 * @param {any[]} messages Messages to modify
860 * @param {number} cachingAtDepth Depth at which caching is supposed to occur
861 * @param {string} ttl TTL value
862 */
863export function cachingAtDepthForClaude(messages, cachingAtDepth, ttl) {
858 let passedThePrefill = false;864 let passedThePrefill = false;
859 let depth = 0;865 let depth = 0;
860 let previousRoleName = '';866 let previousRoleName = '';
@@ -869,7 +875,7 @@ export function cachingAtDepthForClaude(messages, cachingAtDepth) {
869 if (messages[i].role !== previousRoleName) {875 if (messages[i].role !== previousRoleName) {
870 if (depth === cachingAtDepth || depth === cachingAtDepth + 2) {876 if (depth === cachingAtDepth || depth === cachingAtDepth + 2) {
871 const content = messages[i].content;877 const content = messages[i].content;
872 content[content.length - 1].cache_control = { type: 'ephemeral', ttl: '1h' };878 content[content.length - 1].cache_control = { type: 'ephemeral', ttl: ttl };
873 }879 }
874880
875 if (depth === cachingAtDepth + 2) {881 if (depth === cachingAtDepth + 2) {