Add Cache Keepalive extension Reviewed-by: auto-review

acdce0422654c044416be6797fdefd3b99a630ec

permissionBRICK <permissionBRICK@gmail.com>

Parents: d52580592541ecaef1873a8a81e59876a4634ed5
3 files changed, +207 -0Ignore whitespace
public/scripts/extensions/keepalive/index.js+171 -0
@@ -0,0 +1,171 @@
1import { extension_settings, renderExtensionTemplateAsync } from '../../extensions.js';
2import {
3 activateSendButtons,
4 deactivateSendButtons,
5 eventSource,
6 event_types,
7 generateQuietPrompt,
8 isGenerating,
9 saveSettingsDebounced,
10 setSendButtonState,
11 streamingProcessor,
12} from '../../../script.js';
13
14export { init };
15
16const MODULE = 'keepalive';
17
18// Cap the keepalive completion to a single token. The full prompt prefix is
19// still processed by the backend (which is what refreshes the server-side
20// cache), but we never pay for a full completion.
21const KEEPALIVE_RESPONSE_LENGTH = 1;
22
23const defaultSettings = {
24 enabled: false,
25 timeoutSeconds: 295,
26 message: 'Keep the cache warm.',
27};
28
29let idleTimer = null;
30let keepaliveActive = false;
31
32/**
33 * Clears the pending idle timer, if any.
34 */
35function clearIdleTimer() {
36 if (idleTimer) {
37 clearTimeout(idleTimer);
38 idleTimer = null;
39 }
40}
41
42/**
43 * Restarts the idle timer based on the current settings.
44 * Does nothing (and leaves the timer cleared) when the feature is disabled.
45 */
46function resetIdleTimer() {
47 clearIdleTimer();
48 const s = extension_settings[MODULE];
49 if (!s.enabled) {
50 return;
51 }
52 idleTimer = setTimeout(fireKeepalive, Math.max(60, Number(s.timeoutSeconds) || 295) * 1000);
53}
54
55/**
56 * Fires a background ("quiet") generation against the active model to keep the
57 * server-side prompt cache warm. The full prompt prefix (current chat context +
58 * the keepalive message) is sent — which is what refreshes the cache — while the
59 * completion is capped to a single token (KEEPALIVE_RESPONSE_LENGTH) so we never
60 * pay for a real response.
61 *
62 * Quiet generations are forced non-streaming by the core (Generate excludes
63 * type === 'quiet' from the StreamingProcessor) and are never saved to chat, so
64 * the output is simply discarded. We deliberately do NOT hook the global
65 * STREAM_TOKEN_RECEIVED / stopGeneration() path: quiet requests never emit it,
66 * and using it would risk aborting a concurrent user generation.
67 *
68 * generateQuietPrompt's responseLength applies a *global* temporary
69 * response-length override for the duration of the request. To make sure a user
70 * generation can never start (and inherit that override) while the keepalive is
71 * in flight, we hold the same generation lock the blocking summary path uses:
72 * setSendButtonState(true) flips `is_send_press`, which every user send entry
73 * point checks and bails on, and deactivateSendButtons() reflects the busy
74 * state in the UI. Both are released in `finally`.
75 */
76async function fireKeepalive() {
77 const s = extension_settings[MODULE];
78 if (!s.enabled) {
79 return;
80 }
81
82 // Never start while any generation is in progress, or while a previous
83 // keepalive is still running. Reschedule and bail.
84 if (keepaliveActive || isGenerating() || streamingProcessor) {
85 resetIdleTimer();
86 return;
87 }
88
89 keepaliveActive = true;
90 setSendButtonState(true);
91 deactivateSendButtons();
92
93 try {
94 await generateQuietPrompt({ quietPrompt: s.message, responseLength: KEEPALIVE_RESPONSE_LENGTH });
95 } catch (error) {
96 // Best-effort: any failure here is harmless and intentionally swallowed.
97 console.debug('[Keepalive] Background generation ended:', error);
98 } finally {
99 setSendButtonState(false);
100 activateSendButtons();
101 keepaliveActive = false;
102 resetIdleTimer();
103 }
104}
105
106/**
107 * Loads settings into the global store and hydrates the UI controls.
108 */
109function loadSettings() {
110 if (extension_settings[MODULE] === undefined) {
111 extension_settings[MODULE] = {};
112 }
113
114 if (Object.keys(extension_settings[MODULE]).length === 0) {
115 Object.assign(extension_settings[MODULE], defaultSettings);
116 }
117
118 for (const key of Object.keys(defaultSettings)) {
119 if (extension_settings[MODULE][key] === undefined) {
120 extension_settings[MODULE][key] = defaultSettings[key];
121 }
122 }
123
124 $('#keepalive_enabled').prop('checked', extension_settings[MODULE].enabled);
125 $('#keepalive_timeout').val(extension_settings[MODULE].timeoutSeconds);
126 $('#keepalive_message').val(extension_settings[MODULE].message);
127}
128
129/**
130 * Wires up the settings UI control handlers.
131 */
132function setupListeners() {
133 $('#keepalive_enabled').on('change', function () {
134 extension_settings[MODULE].enabled = !!$(this).prop('checked');
135 saveSettingsDebounced();
136 resetIdleTimer();
137 });
138
139 $('#keepalive_timeout').on('input', function () {
140 extension_settings[MODULE].timeoutSeconds = Number($(this).val()) || 295;
141 saveSettingsDebounced();
142 resetIdleTimer();
143 });
144
145 $('#keepalive_message').on('input', function () {
146 extension_settings[MODULE].message = String($(this).val());
147 saveSettingsDebounced();
148 });
149}
150
151async function init() {
152 const settingsHtml = await renderExtensionTemplateAsync(MODULE, 'settings');
153 $('#extensions_settings2').append(settingsHtml);
154
155 loadSettings();
156 setupListeners();
157
158 // Reset the idle timer on any chat or generation activity.
159 const activityEvents = [
160 event_types.MESSAGE_SENT,
161 event_types.MESSAGE_RECEIVED,
162 event_types.GENERATION_STARTED,
163 event_types.GENERATION_ENDED,
164 event_types.CHAT_CHANGED,
165 ];
166 for (const event of activityEvents) {
167 eventSource.on(event, resetIdleTimer);
168 }
169
170 resetIdleTimer();
171}
public/scripts/extensions/keepalive/manifest.json+14 -0
@@ -0,0 +1,14 @@
1{
2 "display_name": "Cache Keepalive",
3 "loading_order": 100,
4 "requires": [],
5 "optional": [],
6 "js": "index.js",
7 "css": "",
8 "author": "SillyTavern",
9 "version": "1.0.0",
10 "homePage": "https://github.com/SillyTavern/SillyTavern",
11 "hooks": {
12 "activate": "init"
13 }
14}
public/scripts/extensions/keepalive/settings.html+22 -0
@@ -0,0 +1,22 @@
1<div id="keepalive_settings">
2 <div class="inline-drawer">
3 <div class="inline-drawer-toggle inline-drawer-header">
4 <b data-i18n="ext_keepalive_title">Cache Keepalive</b>
5 <div class="inline-drawer-icon fa-solid fa-circle-chevron-down down"></div>
6 </div>
7 <div class="inline-drawer-content">
8 <label class="checkbox_label" for="keepalive_enabled">
9 <input id="keepalive_enabled" type="checkbox" />
10 <span data-i18n="ext_keepalive_enable">Enable cache keepalive</span>
11 </label>
12
13 <label for="keepalive_timeout" data-i18n="ext_keepalive_timeout">Idle timeout (seconds)</label>
14 <input id="keepalive_timeout" class="text_pole" type="number" min="60" step="1" />
15
16 <label for="keepalive_message" data-i18n="ext_keepalive_message">Keepalive message</label>
17 <input id="keepalive_message" class="text_pole" type="text" />
18
19 <small data-i18n="ext_keepalive_help">After this many seconds with no chat activity, a hidden background prompt (the current chat plus this message) is sent to the active model to keep the server-side prompt cache warm. The completion is capped to a single token and is never saved to the chat. Sending is briefly locked while it runs.</small>
20 </div>
21 </div>
22</div>