Claude: control cache TTL with config

560c6e8ff113d12f83d29c72cfaf5ce01e84c2f5

Cohee <18619528+Cohee1207@users.noreply.github.com>

default/config.yaml+4 -0 Viewed

   # should be ideal for most use cases.
   # Any value other than a non-negative integer will be ignored and caching at depth will not be enabled.
   cachingAtDepth: -1
 +  # Use 1h TTL instead of the default 5m.
 +  ## 5m: base price x 1.25
 +  ## 1h: base price x 2
 +  extendedTTL: false
 # -- GOOGLE GEMINI API CONFIGURATION --
 gemini:
   # API endpoint version ("v1beta" or "v1alpha")

src/endpoints/backends/chat-completions.js+4 -3 Viewed

         const convertedPrompt = convertClaudeMessages(request.body.messages, request.body.assistant_prefill, useSystemPrompt, useTools, getPromptNames(request));
         const useThinking = /^claude-(3-7|opus-4|sonnet-4)/.test(request.body.model) && Boolean(request.body.include_reasoning);
         const useWebSearch = /^claude-(3-5|3-7|opus-4|sonnet-4)/.test(request.body.model) && Boolean(request.body.enable_web_search);
 +        const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';
         let fixThinkingPrefill = false;
         // Add custom stop sequences
         const stopSequences = [];

         };
         if (useSystemPrompt) {
             if (enableSystemPromptCache && Array.isArray(convertedPrompt.systemPrompt) && convertedPrompt.systemPrompt.length) {
                 convertedPrompt.systemPrompt[convertedPrompt.systemPrompt.length - 1]['cache_control'] = { type: 'ephemeral', ttl: '1h'cacheTTL };
+            }
             requestBody.system = convertedPrompt.systemPrompt;

@@ -190,7 +191,7 @@ async function sendClaudeRequest(request, response) {
190	191	.map(fn => ({ name: fn.name, description: fn.description, input_schema: fn.parameters }));
191	192
192	193	if (enableSystemPromptCache && requestBody.tools.length) {
193	194	requestBody.tools[requestBody.tools.length - 1]['cache_control'] = { type: 'ephemeral', ttl: '1h'cacheTTL };
194	195	}
195	196	}
196	197

@@ -203,7 +204,7 @@ async function sendClaudeRequest(request, response) {
203	204	}
204	205
205	206	if (cachingAtDepth !== -1) {
206	207	cachingAtDepthForClaude(convertedPrompt.messages, cachingAtDepth, cacheTTL);
207	208	}
208	209
209	210	if (enableSystemPromptCache \|\| cachingAtDepth !== -1) {

src/prompt-converters.js+8 -2 Viewed

     return messageStrings.join('\n') + '\nassistant:';
+}
 -export function cachingAtDepthForClaude(messages, cachingAtDepth) {
 +/**
 + * Append cache_control object to a Claude messages at depth. Directly modifies the messages array.
 + * @param {any[]} messages Messages to modify
 + * @param {number} cachingAtDepth Depth at which caching is supposed to occur
 + * @param {string} ttl TTL value
 + */
 +export function cachingAtDepthForClaude(messages, cachingAtDepth, ttl) {
     let passedThePrefill = false;
     let depth = 0;
     let previousRoleName = '';

         if (messages[i].role !== previousRoleName) {
             if (depth === cachingAtDepth || depth === cachingAtDepth + 2) {
                 const content = messages[i].content;
                 content[content.length - 1].cache_control = { type: 'ephemeral', ttl: '1h'ttl };
+            }
             if (depth === cachingAtDepth + 2) {