fix: caching system prompt functionality for OpenRouter Claude (#4872)

6fdeaa2cd9d6aedb40c36a7f4217a2433b247d46

Chanho Chung <hello@chungchan.dev>

Signed

Parents: cd9de2f1b08fdf49d2dc15b45e22aaf67dcddde8

src/endpoints/backends/chat-completions.js+10 -1 Viewed

     addAssistantPrefix,
     embedOpenRouterMedia,
     addReasoningContentToToolCalls,
 +    cachingSystemPromptForOpenRouterClaude,
 } from '../../prompt-converters.js';
 import { readSecret, SECRET_KEYS } from '../secrets.js';

             };
+        }
 +        const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false, 'boolean');
         const cachingAtDepth = getConfigValue('claude.cachingAtDepth', -1, 'number');
         const isClaude3or4 = /anthropic\/claude-(3|opus-4|sonnet-4|haiku-4)/.test(request.body.model);
         const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';
         if (Array.isArray(request.body.messages)) {
             embedOpenRouterMedia(request.body.messages);
 -            if (Number.isInteger(cachingAtDepth) && cachingAtDepth >= 0 && isClaude3or4) {
++
 +            if (isClaude3or4) {
 +                if (enableSystemPromptCache) {
 +                    cachingSystemPromptForOpenRouterClaude(request.body.messages, cacheTTL);
 +                }
++
 +                if (Number.isInteger(cachingAtDepth) && cachingAtDepth >= 0) {
                     cachingAtDepthForOpenRouterClaude(request.body.messages, cachingAtDepth, cacheTTL);
+                }
+            }
 +        }
         const isGemini = /google\/gemini/.test(request.body.model);
         if (isGemini) {

src/prompt-converters.js+45 -0 Viewed

+}
 /**
 + * Adds cache_control to the system prompt for OpenRouter Claude requests.
 + *
 + * @param {object[]} messages Array of messages
 + * @param {string} ttl TTL value
 + */
 +export function cachingSystemPromptForOpenRouterClaude(messages, ttl) {
 +    if (!Array.isArray(messages) || messages.length === 0) {
 +        return;
 +    }
++
 +    // Find the first system message
 +    const systemMessage = messages.find(msg => msg.role === 'system');
 +    if (!systemMessage) {
 +        return;
 +    }
++
 +    // Check if it already has cache_control (at message level)
 +    if (systemMessage.cache_control) {
 +        return;
 +    }
++
 +    if (Array.isArray(systemMessage.content)) {
 +        const hasExistingCacheControl = systemMessage.content.some(part => part?.cache_control);
 +        if (hasExistingCacheControl) {
 +            return;
 +        }
++
 +        for (let i = systemMessage.content.length - 1; i >= 0; i--) {
 +            if (systemMessage.content[i]?.type === 'text') {
 +                systemMessage.content[i].cache_control = { type: 'ephemeral', ttl };
 +                return;
 +            }
 +        }
 +    } else if (typeof systemMessage.content === 'string') {
 +        systemMessage.content = [
 +            {
 +                type: 'text',
 +                text: systemMessage.content,
 +                cache_control: { type: 'ephemeral', ttl },
 +            },
 +        ];
 +    }
 +}
++
 +/**
  * Calculate the Claude budget tokens for a given reasoning effort.
  * @param {number} maxTokens Maximum tokens
  * @param {string} reasoningEffort Reasoning effort