Add caching system prompt feature for OpenRouter Gemini (#4903) * feat: add caching system prompt for OpenRouter Gemini * fix: resolve reviews

ca4379679566a29561d0502488667edfaf7e85db

Chanho Chung <hello@chungchan.dev>

Signed
Parents: c75df956c598f995d543e383f55917d50ce2eaf5
3 files changed, +78 -8Showing whitespace changes
default/config.yaml+6 -0
@@ -287,6 +287,12 @@ claude:
287287gemini:
288288 # API endpoint version ("v1beta" or "v1alpha")
289289 apiVersion: 'v1beta'
290+ # Enables caching of the system prompt (if supported). Only for OpenRouter.
291+ # -- IMPORTANT! --
292+ # Use only when the prompt before the chat history is static and doesn't change between requests
293+ # (e.g {{random}} macro or lorebooks not as in-chat injections).
294+ # Otherwise, you'll just waste money on cache misses.
295+ enableSystemPromptCache: false
290296 # https://ai.google.dev/gemini-api/docs/imagen#imagen-configuration
291297 image:
292298 # Leave empty to use the API-default value.
src/endpoints/backends/chat-completions.js+63 -3
@@ -45,7 +45,7 @@ import {
4545 addAssistantPrefix,
4646 embedOpenRouterMedia,
4747 addReasoningContentToToolCalls,
4848 cachingSystemPromptForOpenRouterClaudecachingSystemPromptForOpenRouter,
4949 addOpenRouterSignatures,
5050} from '../../prompt-converters.js';
5151
@@ -84,6 +84,58 @@ const API_COMETAPI = 'https://api.cometapi.com/v1';
8484const API_ZAI_COMMON = 'https://api.z.ai/api/paas/v4';
8585const API_ZAI_CODING = 'https://api.z.ai/api/coding/paas/v4';
8686const API_SILICONFLOW = 'https://api.siliconflow.com/v1';
87+const API_OPENROUTER = 'https://openrouter.ai/api/v1';
88+
89+/**
90+ * Cache for cacheable (writing) OpenRouter model IDs.
91+ * @type {string[]}
92+ */
93+const openRouterCacheableModels = [];
94+
95+/**
96+ * Checks if an OpenRouter model supports prompt cache writing.
97+ * Uses a cache to avoid repeated API calls.
98+ * @param {string} modelId - The OpenRouter model ID
99+ * @returns {Promise<boolean>} `true` if the model supports writing cache
100+ */
101+async function isOpenRouterModelCacheable(modelId) {
102+ if (openRouterCacheableModels.includes(modelId)) {
103+ return true;
104+ }
105+
106+ try {
107+ const response = await fetch(`${API_OPENROUTER}/models`, {
108+ method: 'GET',
109+ headers: { 'Accept': 'application/json' },
110+ signal: AbortSignal.timeout(5000),
111+ });
112+
113+ if (!response.ok) {
114+ console.warn(`OpenRouter models API returned ${response.status}: ${response.statusText}`);
115+ return false;
116+ }
117+
118+ /** @type {any} */
119+ const data = await response.json();
120+
121+ if (!Array.isArray(data?.data)) {
122+ console.warn('OpenRouter API response format unexpected');
123+ return false;
124+ }
125+
126+ const model = data.data.find(m => m.id === modelId);
127+ const supportsCache = model?.pricing?.input_cache_write != null;
128+
129+ if (supportsCache) {
130+ openRouterCacheableModels.push(modelId);
131+ }
132+
133+ return supportsCache;
134+ } catch (error) {
135+ console.warn(`Failed to check OpenRouter cache support for ${modelId}:`, error.message);
136+ return false;
137+ }
138+}
87139
88140/**
89141 * Gets OpenRouter transforms based on the request.
@@ -2051,22 +2103,30 @@ router.post('/generate', async function (request, response) {
20512103 const cachingAtDepth = getConfigValue('claude.cachingAtDepth', -1, 'number');
20522104 const isClaude3or4 = /anthropic\/claude-(3|opus-4|sonnet-4|haiku-4)/.test(request.body.model);
20532105 const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';
2106+
2107+ const isGemini = /google\/gemini/.test(request.body.model);
2108+ const isCacheableGemini = isGemini && await isOpenRouterModelCacheable(request.body.model);
2109+ const enableGeminiSystemPromptCache = getConfigValue('gemini.enableSystemPromptCache', false, 'boolean');
2110+
20542111 if (Array.isArray(request.body.messages)) {
20552112 embedOpenRouterMedia(request.body.messages);
20562113 addOpenRouterSignatures(request.body.messages, request.body.model);
20572114
20582115 if (isClaude3or4) {
20592116 if (enableSystemPromptCache) {
20602117 cachingSystemPromptForOpenRouterClaudecachingSystemPromptForOpenRouter(request.body.messages, cacheTTL);
20612118 }
20622119
20632120 if (Number.isInteger(cachingAtDepth) && cachingAtDepth >= 0) {
20642121 cachingAtDepthForOpenRouterClaude(request.body.messages, cachingAtDepth, cacheTTL);
20652122 }
20662123 }
2124+
2125+ if (isCacheableGemini && enableGeminiSystemPromptCache) {
2126+ cachingSystemPromptForOpenRouter(request.body.messages);
2127+ }
20672128 }
20682129
2069- const isGemini = /google\/gemini/.test(request.body.model);
20702130 if (isGemini) {
20712131 bodyParams['safety_settings'] = GEMINI_SAFETY;
20722132 }
src/prompt-converters.js+9 -5
@@ -1057,12 +1057,12 @@ export function cachingAtDepthForOpenRouterClaude(messages, cachingAtDepth, ttl)
10571057}
10581058
10591059/**
10601060 * Adds cache_control to the system prompt for OpenRouter Claude requests.
10611061 *
10621062 * @param {object[]} messages Array of messages
10631063 * @param {string} [ttl] TTL value (optional)
10641064 */
10651065export function cachingSystemPromptForOpenRouterClaudecachingSystemPromptForOpenRouter(messages, ttl = undefined) {
10661066 if (!Array.isArray(messages) || messages.length === 0) {
10671067 return;
10681068 }
@@ -1078,6 +1078,10 @@ export function cachingSystemPromptForOpenRouterClaude(messages, ttl) {
10781078 return;
10791079 }
10801080
1081+ const cacheControl = ttl
1082+ ? { type: 'ephemeral', ttl }
1083+ : { type: 'ephemeral' };
1084+
10811085 if (Array.isArray(systemMessage.content)) {
10821086 const hasExistingCacheControl = systemMessage.content.some(part => part?.cache_control);
10831087 if (hasExistingCacheControl) {
@@ -1086,7 +1090,7 @@ export function cachingSystemPromptForOpenRouterClaude(messages, ttl) {
10861090
10871091 for (let i = systemMessage.content.length - 1; i >= 0; i--) {
10881092 if (systemMessage.content[i]?.type === 'text') {
10891093 systemMessage.content[i].cache_control = { type: 'ephemeral', ttl }cacheControl;
10901094 return;
10911095 }
10921096 }
@@ -1095,7 +1099,7 @@ export function cachingSystemPromptForOpenRouterClaude(messages, ttl) {
10951099 {
10961100 type: 'text',
10971101 text: systemMessage.content,
1098- cache_control: { type: 'ephemeral', ttl },
1102+ cache_control: cacheControl,
10991103 },
11001104 ];
11011105 }