NanoGPT: Enable prompt cache for Claude models

57cd00122110d2895ddd524c94ba454320163c5e

Cohee <18619528+Cohee1207@users.noreply.github.com>

1 files changed, +11 -1Showing whitespace changes
src/endpoints/backends/chat-completions.js+11 -1
@@ -1642,7 +1642,17 @@ router.post('/generate', function (request, response) {
16421642 if (request.body.enable_web_search && !/:online$/.test(request.body.model)) {
16431643 request.body.model = `${request.body.model}:online`;
16441644 }
1645- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.POLLINATIONS) {
1645+ const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false, 'boolean');
1646+ const isClaude3or4 = /claude-(3|opus-4|sonnet-4)/.test(request.body.model);
1647+ const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';
1648+ if (enableSystemPromptCache && isClaude3or4) {
1649+ bodyParams['cache_control'] = {
1650+ 'enabled': true,
1651+ 'ttl': cacheTTL,
1652+ };
1653+ }
1654+ }
1655+ else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.POLLINATIONS) {
16461656 apiUrl = API_POLLINATIONS;
16471657 apiKey = 'NONE';
16481658 headers = {