[Electron Hub] Prompt Caching Support for Claude models (#4918) * Prompt Caching support Claude models * Prompt Caching support Claude models * Diff clean-up --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>

829db7f2d059daa5e9b95696e3029e87d4ba3ddf

Ngo Dinh Gia Bao <112873281+snowby666@users.noreply.github.com>

Signed
Parents: bb53da4c09209a49cf4ab83ba1f9970d6d852664
1 files changed, +28 -18Ignore whitespace
src/endpoints/backends/chat-completions.js+28 -18
@@ -87,6 +87,16 @@ const API_SILICONFLOW = 'https://api.siliconflow.com/v1';
8787const API_OPENROUTER = 'https://openrouter.ai/api/v1';
8888
8989/**
90+ * Module-scoped Claude caching configuration values.
91+ */
92+const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';
93+const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false, 'boolean');
94+const cachingAtDepth = (() => {
95+ const value = getConfigValue('claude.cachingAtDepth', -1, 'number');
96+ return Number.isInteger(value) && value >= 0 ? value : -1;
97+})();
98+
99+/**
90100 * Cache for cacheable (writing) OpenRouter model IDs.
91101 * @type {string[]}
92102 */
@@ -194,12 +204,6 @@ async function sendClaudeRequest(request, response) {
194204 const apiUrl = new URL(request.body.reverse_proxy || API_CLAUDE).toString();
195205 const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.CLAUDE);
196206 const divider = '-'.repeat(process.stdout.columns);
197- const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false, 'boolean');
198- let cachingAtDepth = getConfigValue('claude.cachingAtDepth', -1, 'number');
199- // Disabled if not an integer or negative
200- if (!Number.isInteger(cachingAtDepth) || cachingAtDepth < 0) {
201- cachingAtDepth = -1;
202- }
203207
204208 if (!apiKey) {
205209 console.warn(color.red(`Claude API key is missing.\n${divider}`));
@@ -221,7 +225,6 @@ async function sendClaudeRequest(request, response) {
221225 const useWebSearch = /^claude-(3-5|3-7|opus-4|sonnet-4|haiku-4-5|opus-4-5)/.test(request.body.model) && Boolean(request.body.enable_web_search);
222226 const isLimitedSampling = /^claude-(opus-4-1|sonnet-4-5|haiku-4-5|opus-4-5)/.test(request.body.model);
223227 const useVerbosity = /^claude-(opus-4-5)/.test(request.body.model);
224- const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';
225228 let fixThinkingPrefill = false;
226229 // Add custom stop sequences
227230 const stopSequences = [];
@@ -1365,6 +1368,18 @@ async function sendElectronHubRequest(request, response) {
13651368 };
13661369 }
13671370
1371+ const isClaude = /^claude-/.test(request.body.model);
1372+
1373+ if (Array.isArray(request.body.messages) && isClaude) {
1374+ if (enableSystemPromptCache) {
1375+ cachingSystemPromptForOpenRouter(request.body.messages, cacheTTL);
1376+ }
1377+
1378+ if (cachingAtDepth !== -1) {
1379+ cachingAtDepthForOpenRouterClaude(request.body.messages, cachingAtDepth, cacheTTL);
1380+ }
1381+ }
1382+
13681383 const requestBody = {
13691384 'messages': request.body.messages,
13701385 'model': request.body.model,
@@ -2101,11 +2116,7 @@ router.post('/generate', async function (request, response) {
21012116 };
21022117 }
21032118
2104- const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false, 'boolean');
2119+ const isClaude = /^anthropic\/claude/.test(request.body.model);
2105- const cachingAtDepth = getConfigValue('claude.cachingAtDepth', -1, 'number');
2106- const isClaude3or4 = /anthropic\/claude-(3|opus-4|sonnet-4|haiku-4)/.test(request.body.model);
2107- const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';
2108-
21092120 const isGemini = /google\/gemini/.test(request.body.model);
21102121 const isCacheableGemini = isGemini && await isOpenRouterModelCacheable(request.body.model);
21112122 const enableGeminiSystemPromptCache = getConfigValue('gemini.enableSystemPromptCache', false, 'boolean');
@@ -2114,12 +2125,12 @@ router.post('/generate', async function (request, response) {
21142125 embedOpenRouterMedia(request.body.messages);
21152126 addOpenRouterSignatures(request.body.messages, request.body.model);
21162127
21172128 if (isClaude3or4isClaude) {
21182129 if (enableSystemPromptCache) {
21192130 cachingSystemPromptForOpenRouter(request.body.messages, cacheTTL);
21202131 }
21212132
21222133 if (Number.isInteger(cachingAtDepth) && cachingAtDepth >!== 0-1) {
21232134 cachingAtDepthForOpenRouterClaude(request.body.messages, cachingAtDepth, cacheTTL);
21242135 }
21252136 }
@@ -2214,10 +2225,9 @@ router.post('/generate', async function (request, response) {
22142225 if (request.body.repetition_penalty !== undefined) {
22152226 bodyParams['repetition_penalty'] = request.body.repetition_penalty;
22162227 }
2217- const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false, 'boolean');
2228+
22182229 const isClaude3or4isClaude = /^claude-(3|opus-4|sonnet-4|haiku-4)/.test(request.body.model);
2219- const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';
2230+ if (enableSystemPromptCache && isClaude) {
2220- if (enableSystemPromptCache && isClaude3or4) {
22212231 bodyParams['cache_control'] = {
22222232 'enabled': true,
22232233 'ttl': cacheTTL,