Add caching system prompt feature for OpenRouter Gemini (#4903) * feat: add caching system prompt for OpenRouter Gemini * fix: resolve reviews

ca4379679566a29561d0502488667edfaf7e85db

Chanho Chung <hello@chungchan.dev>

Signed

Parents: c75df956c598f995d543e383f55917d50ce2eaf5

default/config.yaml+6 -0 Viewed

 gemini:
   # API endpoint version ("v1beta" or "v1alpha")
   apiVersion: 'v1beta'
+  # Enables caching of the system prompt (if supported). Only for OpenRouter.
+  # -- IMPORTANT! --
+  # Use only when the prompt before the chat history is static and doesn't change between requests
+  # (e.g {{random}} macro or lorebooks not as in-chat injections).
+  # Otherwise, you'll just waste money on cache misses.
+  enableSystemPromptCache: false
   # https://ai.google.dev/gemini-api/docs/imagen#imagen-configuration
   image:
     # Leave empty to use the API-default value.

src/endpoints/backends/chat-completions.js+63 -3 Viewed

     addAssistantPrefix,
     embedOpenRouterMedia,
     addReasoningContentToToolCalls,
-    cachingSystemPromptForOpenRouterClaude,
+    cachingSystemPromptForOpenRouter,
     addOpenRouterSignatures,
 } from '../../prompt-converters.js';

 const API_ZAI_COMMON = 'https://api.z.ai/api/paas/v4';
 const API_ZAI_CODING = 'https://api.z.ai/api/coding/paas/v4';
 const API_SILICONFLOW = 'https://api.siliconflow.com/v1';
+const API_OPENROUTER = 'https://openrouter.ai/api/v1';
+/**
+ * Cache for cacheable (writing) OpenRouter model IDs.
+ * @type {string[]}
+ */
+const openRouterCacheableModels = [];
+/**
+ * Checks if an OpenRouter model supports prompt cache writing.
+ * Uses a cache to avoid repeated API calls.
+ * @param {string} modelId - The OpenRouter model ID
+ * @returns {Promise<boolean>} `true` if the model supports writing cache
+ */
+async function isOpenRouterModelCacheable(modelId) {
+    if (openRouterCacheableModels.includes(modelId)) {
+        return true;
+    }
+    try {
+        const response = await fetch(`${API_OPENROUTER}/models`, {
+            method: 'GET',
+            headers: { 'Accept': 'application/json' },
+            signal: AbortSignal.timeout(5000),
+        });
+        if (!response.ok) {
+            console.warn(`OpenRouter models API returned ${response.status}: ${response.statusText}`);
+            return false;
+        }
+        /** @type {any} */
+        const data = await response.json();
+        if (!Array.isArray(data?.data)) {
+            console.warn('OpenRouter API response format unexpected');
+            return false;
+        }
+        const model = data.data.find(m => m.id === modelId);
+        const supportsCache = model?.pricing?.input_cache_write != null;
+        if (supportsCache) {
+            openRouterCacheableModels.push(modelId);
+        }
+        return supportsCache;
+    } catch (error) {
+        console.warn(`Failed to check OpenRouter cache support for ${modelId}:`, error.message);
+        return false;
+    }
+}
 /**
  * Gets OpenRouter transforms based on the request.

             const cachingAtDepth = getConfigValue('claude.cachingAtDepth', -1, 'number');
             const isClaude3or4 = /anthropic\/claude-(3|opus-4|sonnet-4|haiku-4)/.test(request.body.model);
             const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';
+            const isGemini = /google\/gemini/.test(request.body.model);
+            const isCacheableGemini = isGemini && await isOpenRouterModelCacheable(request.body.model);
+            const enableGeminiSystemPromptCache = getConfigValue('gemini.enableSystemPromptCache', false, 'boolean');
             if (Array.isArray(request.body.messages)) {
                 embedOpenRouterMedia(request.body.messages);
                 addOpenRouterSignatures(request.body.messages, request.body.model);
                 if (isClaude3or4) {
                     if (enableSystemPromptCache) {
-                        cachingSystemPromptForOpenRouterClaude(request.body.messages, cacheTTL);
+                        cachingSystemPromptForOpenRouter(request.body.messages, cacheTTL);
+                    }
                     if (Number.isInteger(cachingAtDepth) && cachingAtDepth >= 0) {
                         cachingAtDepthForOpenRouterClaude(request.body.messages, cachingAtDepth, cacheTTL);
+                    }
+                }
+                if (isCacheableGemini && enableGeminiSystemPromptCache) {
+                    cachingSystemPromptForOpenRouter(request.body.messages);
+                }
+            }
-            const isGemini = /google\/gemini/.test(request.body.model);
             if (isGemini) {
                 bodyParams['safety_settings'] = GEMINI_SAFETY;
+            }

src/prompt-converters.js+9 -5 Viewed

@@ -1057,12 +1057,12 @@ export function cachingAtDepthForOpenRouterClaude(messages, cachingAtDepth, ttl)
1057	}	1057	}
1058		1058
1059	/**	1059	/**
1060	* Adds cache_control to the system prompt for OpenRouter Claude requests.	1060	* Adds cache_control to the system prompt for OpenRouter requests.
1061	*	1061	*
1062	* @param {object[]} messages Array of messages	1062	* @param {object[]} messages Array of messages
1063	* @param {string} ttl TTL value	1063	* @param {string} [ttl] TTL value (optional)
1064	*/	1064	*/
1065	export function cachingSystemPromptForOpenRouterClaude(messages, ttl) {	1065	export function cachingSystemPromptForOpenRouter(messages, ttl = undefined) {
1066	if (!Array.isArray(messages) \|\| messages.length === 0) {	1066	if (!Array.isArray(messages) \|\| messages.length === 0) {
1067	return;	1067	return;
1068	}	1068	}

         return;
+    }
+    const cacheControl = ttl
+        ? { type: 'ephemeral', ttl }
+        : { type: 'ephemeral' };
     if (Array.isArray(systemMessage.content)) {
         const hasExistingCacheControl = systemMessage.content.some(part => part?.cache_control);
         if (hasExistingCacheControl) {

@@ -1086,7 +1090,7 @@ export function cachingSystemPromptForOpenRouterClaude(messages, ttl) {
1086		1090
1087	for (let i = systemMessage.content.length - 1; i >= 0; i--) {	1091	for (let i = systemMessage.content.length - 1; i >= 0; i--) {
1088	if (systemMessage.content[i]?.type === 'text') {	1092	if (systemMessage.content[i]?.type === 'text') {
1089	systemMessage.content[i].cache_control = { type: 'ephemeral', ttl };	1093	systemMessage.content[i].cache_control = cacheControl;
1090	return;	1094	return;
1091	}	1095	}
1092	}	1096	}

@@ -1095,7 +1099,7 @@ export function cachingSystemPromptForOpenRouterClaude(messages, ttl) {
1095	{	1099	{
1096	type: 'text',	1100	type: 'text',
1097	text: systemMessage.content,	1101	text: systemMessage.content,
1098	cache_control: { type: 'ephemeral', ttl },	1102	cache_control: cacheControl,
1099	},	1103	},
1100	];	1104	];
1101	}	1105	}