Add optional Claude system prompt cache.

7322dd1954fa20c08a0d370515407ea002bf3fea

Cohee <18619528+Cohee1207@users.noreply.github.com>

default/config.yaml+14 -5 Viewed

 enableUserAccounts: false
 # Enable discreet login mode: hides user list on the login screen
 enableDiscreetLogin: false
 +# User session timeout *in seconds* (defaults to 24 hours).
 +## Set to a positive number to expire session after a certain time of inactivity
 +## Set to 0 to expire session when the browser is closed
 +## Set to a negative number to disable session expiration
 +sessionTimeout: 86400
 # Used to sign session cookies. Will be auto-generated if not set
 cookieSecret: ''
 # Disable CSRF protection - NOT RECOMMENDED

   # * 0: Unload the model immediately after the request
   # * 5m: Keep the model loaded for 5 minutes after the request. Accepts duration strings (e.g. 5h30m40s)
   keepAlive: -1
 +# -- ANTHROPIC CLAUDE API CONFIGURATION --
 +claude:
 +  # Enables caching of the system prompt (if supported).
 +  # https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching
 +  # -- IMPORTANT! --
 +  # Use only when the prompt before the chat history is static and doesn't change between requests
 +  # (e.g {{random}} macro or lorebooks not as in-chat injections).
 +  # Otherwise, you'll just waste money on cache misses.
 +  enableSystemPromptCache: false
 # -- SERVER PLUGIN CONFIGURATION --
 enableServerPlugins: false
 -# User session timeout *in seconds* (defaults to 24 hours).
 -## Set to a positive number to expire session after a certain time of inactivity
 -## Set to 0 to expire session when the browser is closed
 -## Set to a negative number to disable session expiration
 -sessionTimeout: 86400

src/endpoints/backends/chat-completions.js+13 -7 Viewed

     const apiUrl = new URL(request.body.reverse_proxy || API_CLAUDE).toString();
     const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.CLAUDE);
     const divider = '-'.repeat(process.stdout.columns);
 +    const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false);
     if (!apiKey) {
         console.log(color.red(`Claude API key is missing.\n${divider}`));

             controller.abort();
         });
         const additionalHeaders = {};
         letconst use_system_promptuseSystemPrompt = (request.body.model.startsWith('claude-2') || request.body.model.startsWith('claude-3')) && request.body.claude_use_sysprompt;
         letconst converted_promptconvertedPrompt = convertClaudeMessages(request.body.messages, request.body.assistant_prefill, use_system_promptuseSystemPrompt, request.body.human_sysprompt_message, request.body.char_name, request.body.user_name);
         // Add custom stop sequences
         const stopSequences = [];
         if (Array.isArray(request.body.stop)) {

+        }
         const requestBody = {
             messages: converted_promptconvertedPrompt.messages,
             model: request.body.model,
             max_tokens: request.body.max_tokens,
             stop_sequences: stopSequences,

             top_k: request.body.top_k,
             stream: request.body.stream,
         };
         if (use_system_promptuseSystemPrompt) {
             requestBody.system = converted_prompt.systemPrompt;enableSystemPromptCache
 +                ? [{ type: 'text', text: convertedPrompt.systemPrompt, cache_control: { type: 'ephemeral' } }]
 +                : convertedPrompt.systemPrompt;
+        }
         if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
             // Claude doesn't do prefills on function calls, and doesn't allow empty messages
             if (converted_promptconvertedPrompt.messages.length && converted_promptconvertedPrompt.messages[converted_promptconvertedPrompt.messages.length - 1].role === 'assistant') {
                 converted_promptconvertedPrompt.messages.push({ role: 'user', content: '.' });
+            }
             additionalHeaders['anthropic-beta'] = 'tools-2024-05-16';
             requestBody.tool_choice = { type: request.body.tool_choice === 'required' ? 'any' : 'auto' };

                 .map(tool => tool.function)
                 .map(fn => ({ name: fn.name, description: fn.description, input_schema: fn.parameters }));
+        }
 +        if (enableSystemPromptCache) {
 +            additionalHeaders['anthropic-beta'] = 'prompt-caching-2024-07-31';
 +        }
         console.log('Claude request:', requestBody);
         const generateResponse = await fetch(apiUrl + '/messages', {