NanoGPT: Enable prompt cache for Claude models

57cd00122110d2895ddd524c94ba454320163c5e

Cohee <18619528+Cohee1207@users.noreply.github.com>

1 files changed, +11 -1Showing whitespace changes
src/endpoints/backends/chat-completions.js+11 -1
@@ -1642,7 +1642,17 @@ router.post('/generate', function (request, response) {
1642 if (request.body.enable_web_search && !/:online$/.test(request.body.model)) {1642 if (request.body.enable_web_search && !/:online$/.test(request.body.model)) {
1643 request.body.model = `${request.body.model}:online`;1643 request.body.model = `${request.body.model}:online`;
1644 }1644 }
1645 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.POLLINATIONS) {1645 const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false, 'boolean');
1646 const isClaude3or4 = /claude-(3|opus-4|sonnet-4)/.test(request.body.model);
1647 const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';
1648 if (enableSystemPromptCache && isClaude3or4) {
1649 bodyParams['cache_control'] = {
1650 'enabled': true,
1651 'ttl': cacheTTL,
1652 };
1653 }
1654 }
1655 else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.POLLINATIONS) {
1646 apiUrl = API_POLLINATIONS;1656 apiUrl = API_POLLINATIONS;
1647 apiKey = 'NONE';1657 apiKey = 'NONE';
1648 headers = {1658 headers = {