fix: caching system prompt functionality for OpenRouter Claude (#4872)

6fdeaa2cd9d6aedb40c36a7f4217a2433b247d46

Chanho Chung <hello@chungchan.dev>

Signed

Parents: cd9de2f1b08fdf49d2dc15b45e22aaf67dcddde8

src/endpoints/backends/chat-completions.js+11 -2 Viewed

     addAssistantPrefix,
     embedOpenRouterMedia,
     addReasoningContentToToolCalls,
+    cachingSystemPromptForOpenRouterClaude,
 } from '../../prompt-converters.js';
 import { readSecret, SECRET_KEYS } from '../secrets.js';

             };
+        }
+        const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false, 'boolean');
         const cachingAtDepth = getConfigValue('claude.cachingAtDepth', -1, 'number');
         const isClaude3or4 = /anthropic\/claude-(3|opus-4|sonnet-4|haiku-4)/.test(request.body.model);
         const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';
         if (Array.isArray(request.body.messages)) {
             embedOpenRouterMedia(request.body.messages);
-            if (Number.isInteger(cachingAtDepth) && cachingAtDepth >= 0 && isClaude3or4) {
-                cachingAtDepthForOpenRouterClaude(request.body.messages, cachingAtDepth, cacheTTL);
+            if (isClaude3or4) {
+                if (enableSystemPromptCache) {
+                    cachingSystemPromptForOpenRouterClaude(request.body.messages, cacheTTL);
+                }
+                if (Number.isInteger(cachingAtDepth) && cachingAtDepth >= 0) {
+                    cachingAtDepthForOpenRouterClaude(request.body.messages, cachingAtDepth, cacheTTL);
+                }
+            }
+        }

src/prompt-converters.js+45 -0 Viewed

+}
 /**
+ * Adds cache_control to the system prompt for OpenRouter Claude requests.
+ *
+ * @param {object[]} messages Array of messages
+ * @param {string} ttl TTL value
+ */
+export function cachingSystemPromptForOpenRouterClaude(messages, ttl) {
+    if (!Array.isArray(messages) || messages.length === 0) {
+        return;
+    }
+    // Find the first system message
+    const systemMessage = messages.find(msg => msg.role === 'system');
+    if (!systemMessage) {
+        return;
+    }
+    // Check if it already has cache_control (at message level)
+    if (systemMessage.cache_control) {
+        return;
+    }
+    if (Array.isArray(systemMessage.content)) {
+        const hasExistingCacheControl = systemMessage.content.some(part => part?.cache_control);
+        if (hasExistingCacheControl) {
+            return;
+        }
+        for (let i = systemMessage.content.length - 1; i >= 0; i--) {
+            if (systemMessage.content[i]?.type === 'text') {
+                systemMessage.content[i].cache_control = { type: 'ephemeral', ttl };
+                return;
+            }
+        }
+    } else if (typeof systemMessage.content === 'string') {
+        systemMessage.content = [
+            {
+                type: 'text',
+                text: systemMessage.content,
+                cache_control: { type: 'ephemeral', ttl },
+            },
+        ];
+    }
+}
+/**
  * Calculate the Claude budget tokens for a given reasoning effort.
  * @param {number} maxTokens Maximum tokens
  * @param {string} reasoningEffort Reasoning effort