Move keepalive scheduling to backend

18bf41c8bdd4f16a5745803d1dc7f4779c15387b

permissionBRICK <40219477+permissionBRICK@users.noreply.github.com>

Parents: abc1cb103bccdb271ac27a2e4ae81b744cd67c57

homelab/README.md+1 -1 Viewed

 | File | Deployed at | Purpose |
 |---|---|---|
 | `whisper-lazy-proxy.py` | `/mnt/user/appdata/whisper-lazy/proxy.py` (container `whisper-lazy`, :10300) | Wyoming lazy proxy: answers HA availability checks from cache, starts `faster-whisper` on demand, stops it after 15 min idle (frees GPU VRAM). |
-| `runpod-lazy-proxy.py` | `/mnt/user/appdata/runpod-lazy/proxy.py` (container `runpod-lazy`, :8189) | ComfyUI-compatible on-demand proxy for a volumeless RunPod GPU pod: only `/lazy/warmup` starts or provisions, while cold ComfyUI requests return 503 for provider failover. `/lazy/status|warmup|shutdown|ping|catalog` provide SillyTavern controls and a NAS-side 15-min idle terminate. Model-set changes requested by Warm up download in place via the worker image's model-manager (port 8189, LRU disk eviction); pods are only recreated when none exists or the image predates the manager. |
+| `runpod-lazy-proxy.py` | `/mnt/user/appdata/runpod-lazy/proxy.py` (container `runpod-lazy`, :8189) | ComfyUI-compatible on-demand proxy for a volumeless RunPod GPU pod: only `/lazy/warmup` starts or provisions, while cold ComfyUI requests return 503 for provider failover. `/lazy/ping` records 20-second frontend heartbeats; the proxy owns the 60-second upstream keepalive cadence and continues it through browser timer pauses up to a 5-minute lease. A NAS-side idle timer terminates the pod 15 minutes after keepalives and other activity stop. Model-set changes requested by Warm up download in place via the worker image's model-manager (port 8189, LRU disk eviction); pods are only recreated when none exists or the image predates the manager. |
 | `runpod-pod.sh` | `/boot/config/runpod-pod.sh` | CLI for the runpod-lazy control API (`status|warmup|shutdown|ping|watch`). |
 Related: the RunPod worker image (ComfyUI + custom nodes + boot-time model

homelab/runpod-lazy-proxy.py+62 -6 Viewed

   GET  /lazy/status    -> {"state": red|orange|green, "model": ..., ...}
   POST /lazy/warmup    -> start pod for the active catalog entry
   POST /lazy/shutdown  -> terminate pod now
-  POST /lazy/ping      -> keepalive for a ready pod (never starts a pod)
+  POST /lazy/ping      -> record a frontend heartbeat (never starts a pod)
   POST /lazy/catalog   -> {"models": [...], "active": "<value>"} store catalog;
                           never starts or provisions a pod
 Start semantics: only POST /lazy/warmup may provision or change models. ComfyUI
 requests return 503 unless a pod is already ready, allowing SillyTavern's image
-target fallback chain to continue without waking this target. The idle timer
+target fallback chain to continue without waking this target. Browser heartbeats
-lives here (NAS side): IDLE_SECONDS after the last activity signal the pod is
+grant a short lease during which this proxy sends keepalive probes on a stable
-terminated, frontend or no frontend.
+backend cadence. The idle timer lives here (NAS side): IDLE_SECONDS after the
+last activity signal the pod is terminated, frontend or no frontend.
 """
 import json
 import os

 RUNPOD_KEY = os.environ['RUNPOD_KEY']
 LISTEN_PORT = int(os.environ.get('LISTEN_PORT', '8189'))
 IDLE_SECONDS = int(os.environ.get('IDLE_SECONDS', '900'))
+KEEPALIVE_SECONDS = max(1, int(os.environ.get('KEEPALIVE_SECONDS', '60')))
+FRONTEND_LEASE_SECONDS = max(1, int(os.environ.get('FRONTEND_LEASE_SECONDS', '300')))
 DATACENTERS = [d for d in os.environ.get('DATACENTERS', '').split(',') if d]
 CLOUD_TYPE = os.environ.get('CLOUD_TYPE', 'SECURE')
 IMAGE = os.environ.get('IMAGE', 'ghcr.io/permissionbrick/comfyui-runpod-worker:latest')

     'gpu': None,
     'since': time.time(),
     'last': time.monotonic(),
+    'frontend_last_seen': None,
+    'keepalive_last': None,
     'ensuring': 0,      # active ensure_pod waiters (status skips probing then)
     'control_epoch': 0, # incremented by shutdown to cancel in-flight warmups
     'lock': threading.Lock(),

         time.sleep(30)
         idle = time.monotonic() - state['last']
         if idle > IDLE_SECONDS:
+            # A heartbeat may arrive just before the next scheduled backend
+            # probe. Its live lease protects the pod until that probe catches
+            # up, without relying on the browser to refresh the idle clock.
+            if frontend_lease_seconds_left() > 0:
+                continue
             pod_id = state['pod_id'] or find_pod()[0]
             if pod_id:
                 log(f'idle {int(idle)}s - terminating pod')

                 state.update({'pod_id': None, 'model': None, 'gpu': None, 'phase': 'red', 'since': time.time()})
+def frontend_lease_seconds_left(now=None):
+    """Returns the remaining frontend heartbeat lease in whole seconds."""
+    now = time.monotonic() if now is None else now
+    last_seen = state['frontend_last_seen']
+    if last_seen is None:
+        return 0
+    return max(0, FRONTEND_LEASE_SECONDS - int(now - last_seen))
+def maintain_frontend_keepalive(now=None):
+    """Probes and refreshes a ready pod while a frontend heartbeat lease is live."""
+    now = time.monotonic() if now is None else now
+    with state['lock']:
+        pod_id = state['pod_id']
+        lease_active = frontend_lease_seconds_left(now) > 0
+        ready = state['phase'] == 'green' and state['ensuring'] == 0
+    if not pod_id or not lease_active or not ready or not upstream_ready(pod_id):
+        return False
+    with state['lock']:
+        # The network probe runs outside the lock. Re-check that its result still
+        # belongs to the current pod and a still-live frontend lease.
+        if state['pod_id'] != pod_id or frontend_lease_seconds_left(now) <= 0:
+            return False
+        state['last'] = now
+        state['keepalive_last'] = now
+    return True
+def keepalive_scheduler():
+    """Runs keepalive probes on a monotonic backend cadence without timer drift."""
+    next_tick = time.monotonic() + KEEPALIVE_SECONDS
+    while True:
+        time.sleep(max(0, next_tick - time.monotonic()))
+        maintain_frontend_keepalive()
+        next_tick += KEEPALIVE_SECONDS
+        now = time.monotonic()
+        if next_tick <= now:
+            # Skip missed ticks after a long process pause instead of issuing a
+            # burst of stale keepalives, while keeping future ticks aligned.
+            missed = int((now - next_tick) // KEEPALIVE_SECONDS) + 1
+            next_tick += missed * KEEPALIVE_SECONDS
 def status_body():
     pod_id = state['pod_id']
     previous_phase = state['phase']

         'since': state['since'],
         'url': pod_url(state['pod_id']) if state['pod_id'] else None,
         'idle_seconds_left': max(0, IDLE_SECONDS - int(time.monotonic() - state['last'])) if state['pod_id'] else 0,
+        'frontend_lease_seconds_left': frontend_lease_seconds_left(),
+        'keepalive_active': bool(state['pod_id'] and state['phase'] == 'green' and frontend_lease_seconds_left() > 0),
     }).encode()

         if path in ('/lazy/ping', '/lazy/warmup', '/lazy/shutdown', '/lazy/catalog') and self.command != 'POST':
             return self._reply(405, b'{"error": "method not allowed"}')
         if path == '/lazy/ping':
-            if state['pod_id'] and state['phase'] == 'green':
+            with state['lock']:
-                state['last'] = time.monotonic()
+                state['frontend_last_seen'] = time.monotonic()
             return self._reply(200, b'{"ok": true}')
         if path == '/lazy/warmup':
             with state['lock']:

 def main():
     threading.Thread(target=idle_reaper, daemon=True).start()
+    threading.Thread(target=keepalive_scheduler, daemon=True).start()
     # Adopt a pre-existing pod after a proxy restart without provisioning or
     # changing it. Read-only status polls will update its readiness phase.
     pod_id, have, gpu = find_pod()

homelab/runpod-pod.sh+1 -1 Viewed

 #!/bin/sh
-# CLI for the runpod-lazy proxy: status | warmup | shutdown | ping
+# CLI for the runpod-lazy proxy: status | warmup | shutdown | ping (renew frontend lease)
 # Usable by anyone (or any AI) without SillyTavern. Proxy owns the pod
 # lifecycle; this only talks to its control API.
 CMD="${1:-status}"

homelab/tests/test_runpod_lazy_proxy.py+36 -0 Viewed

             'gpu': None,
             'since': 0,
             'last': 100,
+            'frontend_last_seen': None,
+            'keepalive_last': None,
             'ensuring': 0,
             'control_epoch': 0,
         })

             self.assertEqual(status, 200)
             self.assertEqual(body['state'], 'red')
+            with mock.patch.object(proxy.time, 'monotonic', return_value=120):
                 status, _ = self.request('POST', '/lazy/ping')
             self.assertEqual(status, 200)
             self.assertEqual(proxy.state['last'], 100)
+            self.assertEqual(proxy.state['frontend_last_seen'], 120)
             catalog = {
                 'models': [{'name': 'Test', 'value': 'test.safetensors', 'files': []}],

         self.assertEqual(status, 200)
         self.assertEqual(proxy.state['last'], 100)
+    def test_backend_maintains_keepalive_during_frontend_lease(self):
+        proxy.state.update({
+            'pod_id': 'ready-pod',
+            'phase': 'green',
+            'last': 100,
+            'frontend_last_seen': 100,
+        })
+        with mock.patch.object(proxy, 'upstream_ready', return_value=True) as upstream_ready:
+            maintained = proxy.maintain_frontend_keepalive(now=399)
+        self.assertTrue(maintained)
+        self.assertEqual(proxy.state['last'], 399)
+        self.assertEqual(proxy.state['keepalive_last'], 399)
+        upstream_ready.assert_called_once_with('ready-pod')
+    def test_backend_stops_keepalive_after_frontend_lease_expires(self):
+        proxy.state.update({
+            'pod_id': 'ready-pod',
+            'phase': 'green',
+            'last': 100,
+            'frontend_last_seen': 100,
+        })
+        with mock.patch.object(proxy, 'upstream_ready') as upstream_ready:
+            maintained = proxy.maintain_frontend_keepalive(now=401)
+        self.assertFalse(maintained)
+        self.assertEqual(proxy.state['last'], 100)
+        self.assertIsNone(proxy.state['keepalive_last'])
+        upstream_ready.assert_not_called()
     def test_get_warmup_is_rejected_without_provisioning(self):
         with mock.patch.object(proxy, 'ensure_pod') as ensure_pod:
             status, body = self.request('GET', '/lazy/warmup')

public/scripts/extensions/keepalive/index.js+258 -118 Viewed

 import { extension_settings, renderExtensionTemplateAsync } from '../../extensions.js';
 import {
-    activateSendButtons,
+    Generate,
-    deactivateSendButtons,
     eventSource,
     event_types,
     extension_prompt_roles,
     extension_prompt_types,
-    generateQuietPrompt,
+    getGenerateUrl,
+    getRequestHeaders,
     isGenerating,
+    main_api,
     saveSettingsDebounced,
     setExtensionPrompt,
-    setSendButtonState,
-    streamingProcessor,
 } from '../../../script.js';
+import { createGenerationParameters, getChatCompletionModel, oai_settings } from '../../openai.js';
+import { horde_settings } from '../../horde.js';
+import { uuidv4 } from '../../utils.js';
 export { init };

 // still processed by the backend (which is what refreshes the server-side
 // cache), but we never pay for a full completion.
 const KEEPALIVE_RESPONSE_LENGTH = 1;
+const HEARTBEAT_INTERVAL_MS = 20000;
+const RETRY_INTERVAL_MS = 5000;
+const SETTINGS_REFRESH_DELAY_MS = 300;
 // Extension-prompt key used to inject the keepalive message as a USER message at depth 0.
 const KEEPALIVE_INJECT_ID = 'keepalive_ping';

 // user message can pre-empt the keepalive.
 const EXPIRED_GRACE_MS = 3000;
-// If the idle timer fires more than this many ms later than it was scheduled to, assume the
-// machine slept / the tab was throttled — far more wall-clock time passed than the interval,
-// so the cache almost certainly lapsed. Skip the ping and go dormant rather than guess.
-const MAX_TIMER_DRIFT_MS = 5000;
 const defaultSettings = {
     enabled: false,
     timeoutSeconds: 295,

     profiles: {},
 };
-let idleTimer = null;
+let heartbeatTimer = null;
-let keepaliveActive = false;
+let jobRegistered = false;
+let refreshPromise = null;
+let refreshRequested = false;
+let refreshTimer = null;
+let preparingJob = false;
+let lifecycleVersion = 0;
 // Whether keepalive is "armed" for the CURRENT chat. It stays dormant until the user sends a new
 // message in the open chat — on a freshly loaded or switched-to chat (even with history already
 // present) we have no way to know whether its prompt is still cached server-side, so pinging would
 // be pointless. Reset on page reload and whenever the chat changes.
 let armed = false;
-// Wall-clock time (Date.now() + delay) the pending idle timer is expected to fire, or null when
-// none is scheduled. Used to detect a timer that fired far too late (sleep/throttle).
-let scheduledFireTime = null;
 // Per-profile "time since last message", keyed by connection profile id (or NONE_KEY).
 // In-memory only: on reload every profile is treated as freshly active.
 const lastActivity = {};
+// Per-document ids keep duplicated tabs independent. Jobs orphaned by a closed or reloaded
+// document are deliberately cleaned up by the backend lease.
+const keepaliveJobId = uuidv4().replaceAll('-', '');
 /**
  * @returns {string} The active connection profile id, or NONE_KEY for a manual connection.
  */

     return perProfile === undefined ? true : !!perProfile;
+}
-/**
+function getTimeoutMs() {
- * Clears the pending idle timer, if any.
+    return Math.max(60, Number(extension_settings[MODULE].timeoutSeconds) || 295) * 1000;
- */
-function clearIdleTimer() {
-    if (idleTimer) {
-        clearTimeout(idleTimer);
-        idleTimer = null;
+}
+async function postKeepaliveBackend(path, body) {
+    return fetch(`/api/keepalive/${path}`, {
+        method: 'POST',
+        headers: getRequestHeaders(),
+        body: JSON.stringify({ id: keepaliveJobId, ...body }),
+    });
+}
-/**
+async function stopBackendJob() {
- * (Re)schedules the keepalive for the ACTIVE profile based on how long it has been since
+    lifecycleVersion++;
- * that profile's last message. Does nothing (and leaves the timer cleared) until keepalive
+    jobRegistered = false;
- * has been armed for the current chat, or when it is disabled for the active connection.
+    try {
- * Called on activity AND whenever the active profile changes, so each profile keeps its own
+        await postKeepaliveBackend('stop', {});
- * independent idle clock.
+    } catch (error) {
- */
+        console.debug('[Keepalive] Could not stop backend job:', error);
-function scheduleIdleTimer() {
+    }
-    clearIdleTimer();
+}
-    if (!armed || !isKeepaliveEnabledForActive()) {
-        scheduledFireTime = null;
+async function sendBackendHeartbeat(activity = false) {
+    if (!jobRegistered || !armed || !isKeepaliveEnabledForActive()) {
         return;
+    }
-    const timeoutMs = Math.max(60, Number(extension_settings[MODULE].timeoutSeconds) || 295) * 1000;
+    try {
-    const last = lastActivity[getActiveProfileKey()] ?? Date.now();
+        const response = await postKeepaliveBackend('heartbeat', { activity });
-    const remaining = timeoutMs - (Date.now() - last);
+        if (response.status === 404) {
-    const delay = remaining > 0 ? remaining : EXPIRED_GRACE_MS;
+            jobRegistered = false;
-    scheduledFireTime = Date.now() + delay;
+            queueJobRefresh();
-    idleTimer = setTimeout(fireKeepalive, delay);
+        }
+    } catch (error) {
+        console.debug('[Keepalive] Backend heartbeat failed:', error);
+    }
+}
-/**
- * Marks the active profile as just-used (resets its idle clock) and reschedules its keepalive.
- */
 function markActivity() {
+    if (preparingJob) {
+        return;
+    }
     lastActivity[getActiveProfileKey()] = Date.now();
-    scheduleIdleTimer();
+    void sendBackendHeartbeat(true);
+}
 /**

  */
 function onChatChanged() {
     armed = false;
-    scheduleIdleTimer();
+    void stopBackendJob();
+}
 /**
- * Fires a background ("quiet") generation against the active model to keep the
+ * Caps a prepared request to a single, non-streaming completion token.
- * server-side prompt cache warm. The full prompt prefix (current chat context +
+ * @param {string} endpoint Backend generation endpoint.
- * the keepalive message) is sent — which is what refreshes the cache — while the
+ * @param {object} payload Prepared generation payload.
- * completion is capped to a single token (KEEPALIVE_RESPONSE_LENGTH) so we never
+ * @returns {object} Cloned and capped payload.
- * pay for a real response.
- *
- * Only the CURRENTLY ACTIVE profile is ever pinged — keepalive never switches the
- * connection in the background. Each profile's idle clock is tracked separately, so
- * switching profiles re-targets the timer at whichever profile is now active.
- *
- * Quiet generations are forced non-streaming by the core (Generate excludes
- * type === 'quiet' from the StreamingProcessor) and are never saved to chat, so the
- * output is simply discarded. generateQuietPrompt's responseLength applies a *global*
- * temporary response-length override for the duration of the request, so we hold the
- * same generation lock the blocking summary path uses — setSendButtonState(true) flips
- * `is_send_press` (which every user send entry point checks and bails on) and
- * deactivateSendButtons() reflects the busy state — and release both in `finally`.
  */
-async function fireKeepalive() {
+function capKeepalivePayload(endpoint, payload) {
+    const capped = structuredClone(payload);
+    capped.stream = false;
+    for (const key of ['max_tokens', 'max_completion_tokens', 'max_length', 'max_new_tokens', 'n_predict']) {
+        if (Object.hasOwn(capped, key)) {
+            capped[key] = KEEPALIVE_RESPONSE_LENGTH;
+        }
+    }
+    if (endpoint === '/api/backends/chat-completions/generate'
+        && !Object.hasOwn(capped, 'max_tokens')
+        && !Object.hasOwn(capped, 'max_completion_tokens')) {
+        capped.max_tokens = KEEPALIVE_RESPONSE_LENGTH;
+    }
+    if (Object.hasOwn(capped, 'n')) {
+        capped.n = 1;
+    }
+    return capped;
+}
+/**
+ * Dry-runs prompt assembly and returns a request the backend can replay later.
+ * @returns {Promise<{endpoint: string, payload: object}>}
+ */
+async function prepareKeepaliveRequest() {
+    const api = main_api;
+    let capturedData = null;
+    const captureData = (data, dryRun) => {
+        if (dryRun && preparingJob) {
+            capturedData = structuredClone(data);
+        }
+    };
+    preparingJob = true;
+    eventSource.on(event_types.GENERATE_AFTER_DATA, captureData);
+    setExtensionPrompt(KEEPALIVE_INJECT_ID, extension_settings[MODULE].message, extension_prompt_types.IN_CHAT, 0, false, extension_prompt_roles.USER);
+    try {
+        await Generate('quiet', { quiet_prompt: '', force_name2: true }, true);
+    } finally {
+        setExtensionPrompt(KEEPALIVE_INJECT_ID, '', extension_prompt_types.IN_CHAT, 0, false, extension_prompt_roles.USER);
+        eventSource.removeListener(event_types.GENERATE_AFTER_DATA, captureData);
+        preparingJob = false;
+    }
+    if (!capturedData || api !== main_api) {
+        throw new Error('Active connection changed while preparing the keepalive request');
+    }
+    let endpoint;
+    let payload;
+    if (api === 'openai') {
+        endpoint = '/api/backends/chat-completions/generate';
+        const model = getChatCompletionModel(oai_settings);
+        ({ generate_data: payload } = await createGenerationParameters(oai_settings, model, 'quiet', capturedData.prompt));
+        await eventSource.emit(event_types.CHAT_COMPLETION_SETTINGS_READY, payload);
+    } else if (api === 'koboldhorde') {
+        endpoint = '/api/horde/generate-text';
+        const params = structuredClone(capturedData);
+        const prompt = params.prompt;
+        delete params.prompt;
+        Object.assign(params, {
+            n: 1,
+            max_length: KEEPALIVE_RESPONSE_LENGTH,
+            frmtadsnsp: false,
+            frmtrmblln: false,
+            frmtrmspch: false,
+            frmttriminc: false,
+        });
+        payload = {
+            prompt,
+            params,
+            trusted_workers: horde_settings.trusted_workers_only,
+            models: horde_settings.models,
+        };
+    } else {
+        endpoint = getGenerateUrl(api);
+        payload = capturedData;
+    }
+    return { endpoint, payload: capKeepalivePayload(endpoint, payload) };
+}
+async function registerBackendJob() {
     if (!armed || !isKeepaliveEnabledForActive()) {
+        await stopBackendJob();
         return;
+    }
+    const version = lifecycleVersion;
-    // If the timer fired much later than scheduled (machine sleep, background-tab throttling,
+    try {
-    // etc.), far more wall-clock time has elapsed than the idle interval, so the cache has
+        if (isGenerating()) {
-    // almost certainly expired. Don't ping on a stale assumption — go dormant until the next
+            throw new Error('Waiting for the active generation to finish');
-    // sent message re-arms keepalive, exactly like a freshly opened chat.
+        }
-    if (scheduledFireTime !== null && (Date.now() - scheduledFireTime) > MAX_TIMER_DRIFT_MS) {
+        const request = await prepareKeepaliveRequest();
-        console.debug('[Keepalive] Idle timer fired', Date.now() - scheduledFireTime, 'ms late; skipping and disarming until next activity.');
+        if (version !== lifecycleVersion || !armed || !isKeepaliveEnabledForActive()) {
-        armed = false;
+            return;
-        clearIdleTimer();
+        }
-        scheduledFireTime = null;
+        const timeoutMs = getTimeoutMs();
+        const last = lastActivity[getActiveProfileKey()] ?? Date.now();
+        const remaining = timeoutMs - (Date.now() - last);
+        const delayMs = remaining > 0 ? remaining : EXPIRED_GRACE_MS;
+        const response = await postKeepaliveBackend('register', {
+            ...request,
+            intervalMs: timeoutMs,
+            delayMs,
+        });
+        if (!response.ok) {
+            throw new Error(`HTTP ${response.status}`);
+        }
+        if (version !== lifecycleVersion || !armed || !isKeepaliveEnabledForActive()) {
+            await stopBackendJob();
             return;
+        }
+        jobRegistered = true;
+    } catch (error) {
+        jobRegistered = false;
+        console.debug('[Keepalive] Could not register backend job:', error);
+        setTimeout(queueJobRefresh, RETRY_INTERVAL_MS);
+    }
+}
-    // Never start while any generation is in progress, or while a previous
+function queueJobRefresh() {
-    // keepalive is still running. Reschedule and bail.
+    refreshRequested = true;
-    if (keepaliveActive || isGenerating() || streamingProcessor) {
+    if (refreshPromise) {
-        scheduleIdleTimer();
         return;
+    }
+    refreshPromise = (async () => {
+        while (refreshRequested) {
+            refreshRequested = false;
+            await registerBackendJob();
+        }
+    })().finally(() => {
+        refreshPromise = null;
+    });
+}
-    keepaliveActive = true;
+function scheduleJobRefresh() {
-    setSendButtonState(true);
+    clearTimeout(refreshTimer);
-    deactivateSendButtons();
+    refreshTimer = setTimeout(queueJobRefresh, SETTINGS_REFRESH_DELAY_MS);
+}
-    try {
+function setupHeartbeatLoop() {
-        // Deliver the keepalive as a USER message at depth 0 (the quiet-prompt path forces a
+    clearInterval(heartbeatTimer);
-        // system role — openai.js hardcodes role:'system' for it), using the same depth+role
+    heartbeatTimer = setInterval(() => void sendBackendHeartbeat(), HEARTBEAT_INTERVAL_MS);
-        // injection Author's Note uses. The quiet prompt itself stays empty so the full chat
+    document.addEventListener('visibilitychange', () => {
-        // context is still assembled (which is what keeps the cached prefix warm).
+        if (document.visibilityState === 'visible') {
-        setExtensionPrompt(KEEPALIVE_INJECT_ID, extension_settings[MODULE].message, extension_prompt_types.IN_CHAT, 0, false, extension_prompt_roles.USER);
+            void sendBackendHeartbeat();
-        await generateQuietPrompt({ quietPrompt: '', responseLength: KEEPALIVE_RESPONSE_LENGTH });
-        // The ping kept this profile warm; treat it as fresh activity so the next
-        // keepalive for it is a full timeout away.
-        lastActivity[getActiveProfileKey()] = Date.now();
-    } catch (error) {
-        // Best-effort: any failure here is harmless and intentionally swallowed.
-        console.debug('[Keepalive] Background generation ended:', error);
-    } finally {
-        setExtensionPrompt(KEEPALIVE_INJECT_ID, '', extension_prompt_types.IN_CHAT, 0, false, extension_prompt_roles.USER);
-        setSendButtonState(false);
-        activateSendButtons();
-        keepaliveActive = false;
-        scheduleIdleTimer();
+        }
+    });
+}
 /**

     $('#keepalive_enabled').on('change', function () {
         extension_settings[MODULE].enabled = !!$(this).prop('checked');
         saveSettingsDebounced();
-        scheduleIdleTimer();
+        if (extension_settings[MODULE].enabled && armed) {
+            queueJobRefresh();
+        } else {
+            void stopBackendJob();
+        }
     });
     $('#keepalive_timeout').on('input', function () {
         extension_settings[MODULE].timeoutSeconds = Number($(this).val()) || 295;
         saveSettingsDebounced();
-        scheduleIdleTimer();
+        if (armed && isKeepaliveEnabledForActive()) {
+            scheduleJobRefresh();
+        }
     });
     $('#keepalive_message').on('input', function () {
         extension_settings[MODULE].message = String($(this).val());
         saveSettingsDebounced();
+        if (armed && isKeepaliveEnabledForActive()) {
+            scheduleJobRefresh();
+        }
     });
     // Per-profile toggles are rendered dynamically; use a delegated handler.

+        }
         extension_settings[MODULE].profiles[profileId] = !!$(this).prop('checked');
         saveSettingsDebounced();
-        // If the toggled profile is the one currently active, reschedule immediately.
+        // If the toggled profile is the one currently active, update its backend job immediately.
         if (profileId === extension_settings.connectionManager?.selectedProfile) {
-            scheduleIdleTimer();
+            if (isKeepaliveEnabledForActive() && armed) {
+                queueJobRefresh();
+            } else {
+                void stopBackendJob();
+            }
+        }
     });
+}
+function onGenerationStarted(_type, _options, dryRun) {
+    if (!dryRun) {
+        markActivity();
+    }
+}
+function onGenerationEnded() {
+    markActivity();
+    if (armed && isKeepaliveEnabledForActive()) {
+        queueJobRefresh();
+    }
+}
+function onChatContentChanged() {
+    markActivity();
+    if (armed && isKeepaliveEnabledForActive()) {
+        queueJobRefresh();
+    }
+}
+async function onConnectionChanged() {
+    // Remove the previous connection's request before assembling one with the new settings.
+    await stopBackendJob();
+    if (armed && isKeepaliveEnabledForActive()) {
+        queueJobRefresh();
+    }
+}
 async function init() {
     const settingsHtml = await renderExtensionTemplateAsync(MODULE, 'settings');
     $('#extensions_settings2').append(settingsHtml);
     loadSettings();
     setupListeners();
+    setupHeartbeatLoop();
     // Only the user sending a new message arms keepalive (for the current chat).
     eventSource.on(event_types.MESSAGE_SENT, onMessageSent);
-    // These reset the active profile's idle clock (so generation time isn't counted as idle) but
+    // Activity resets the backend deadline but never arms keepalive on its own. A completed
-    // never arm keepalive on their own — they also fire for background/quiet generations and while
+    // generation or a chat edit also refreshes the stored request so the server always replays
-    // a chat is loading. They only re-arm the timer if a sent message already armed it.
+    // the latest prompt assembled by the UI.
-    const activityEvents = [
+    eventSource.on(event_types.MESSAGE_RECEIVED, markActivity);
-        event_types.MESSAGE_RECEIVED,
+    eventSource.on(event_types.GENERATION_STARTED, onGenerationStarted);
-        event_types.GENERATION_STARTED,
+    eventSource.on(event_types.GENERATION_ENDED, onGenerationEnded);
-        event_types.GENERATION_ENDED,
+    eventSource.on(event_types.MESSAGE_EDITED, onChatContentChanged);
-    ];
+    eventSource.on(event_types.MESSAGE_DELETED, onChatContentChanged);
-    for (const event of activityEvents) {
+    eventSource.on(event_types.MESSAGE_UPDATED, onChatContentChanged);
-        eventSource.on(event, markActivity);
+    eventSource.on(event_types.MESSAGE_SWIPED, onChatContentChanged);
-    }
     // Opening/switching a chat disarms keepalive — it must be re-armed by a new sent message.
     eventSource.on(event_types.CHAT_CHANGED, onChatChanged);
-    // When the active profile changes, re-target the idle timer at the new profile using
+    // When the active profile changes, replace the backend job without marking it as activity.
-    // ITS own last-activity time (do NOT mark activity — switching is not using it).
+    eventSource.on(event_types.CONNECTION_PROFILE_LOADED, onConnectionChanged);
-    eventSource.on(event_types.CONNECTION_PROFILE_LOADED, scheduleIdleTimer);
+    eventSource.on(event_types.MAIN_API_CHANGED, onConnectionChanged);
+    eventSource.on(event_types.CHATCOMPLETION_SOURCE_CHANGED, onConnectionChanged);
+    eventSource.on(event_types.CHATCOMPLETION_MODEL_CHANGED, onConnectionChanged);
+    eventSource.on(event_types.PRESET_CHANGED, onConnectionChanged);
     // Keep the per-profile toggle list in sync with Connection Manager.
     eventSource.on(event_types.CONNECTION_PROFILE_CREATED, renderProfileToggles);
     eventSource.on(event_types.CONNECTION_PROFILE_UPDATED, renderProfileToggles);
     eventSource.on(event_types.CONNECTION_PROFILE_DELETED, renderProfileToggles);
-    scheduleIdleTimer();
+}

public/scripts/extensions/keepalive/manifest.json+1 -1 Viewed

     "js": "index.js",
     "css": "",
     "author": "SillyTavern",
-    "version": "1.0.0",
+    "version": "1.1.0",
     "homePage": "https://github.com/SillyTavern/SillyTavern",
     "hooks": {
         "activate": "init"

public/scripts/extensions/keepalive/settings.html+1 -1 Viewed

             <label for="keepalive_message" data-i18n="ext_keepalive_message">Keepalive message</label>
             <input id="keepalive_message" class="text_pole" type="text" />
-            <small data-i18n="ext_keepalive_help">After this many seconds without activity on the active connection profile, the current chat plus this message (sent as a hidden user message) is submitted to that profile to keep its server-side prompt cache warm. The completion is capped to a single token and is never saved to the chat. Sending is briefly locked while it runs. Only the profile you're currently using is pinged; each profile has its own idle timer. Keepalive stays off until you send a new message in the current chat (opening, switching to, or reloading a chat isn't proof its prompt is still cached), and it also skips a ping if its timer fired far later than scheduled (e.g. after the machine slept).</small>
+            <small data-i18n="ext_keepalive_help">After this many seconds without activity on the active connection profile, the current chat plus this message (sent as a hidden user message) is submitted to that profile to keep its server-side prompt cache warm. The completion is capped to a single token and is never saved to the chat. The server owns the keepalive timer, while this tab renews its lease every 20 seconds; if the tab sleeps or closes, the server continues on schedule for up to five minutes after the last heartbeat. Only the profile you're currently using is pinged. Keepalive stays off until you send a new message in the current chat (opening, switching to, or reloading a chat isn't proof its prompt is still cached).</small>
             <hr>

public/scripts/extensions/stable-diffusion/index.js+12 -10 Viewed

 /** Poll cadence for the pod status indicator (faster while it is starting). */
 const RUNPOD_POLL_IDLE_MS = 30000;
 const RUNPOD_POLL_BUSY_MS = 5000;
-/** Keepalive cadence for a pod that the status API has confirmed is ready. */
+/** Frontend heartbeat cadence. The proxy tolerates browser timer pauses for five minutes. */
-const RUNPOD_PING_MS = 60000;
+const RUNPOD_HEARTBEAT_MS = 20000;
 let runpodStatusTimer = null;
-let runpodPingTimer = null;
+let runpodHeartbeatTimer = null;
 let runpodLastPhase = 'red';
 function getRunpodLazyUrl() {

     runpodStatusTimer = setTimeout(pollRunpodStatus, RUNPOD_POLL_BUSY_MS);
+}
-function runpodKeepalive() {
+function runpodHeartbeat() {
     const url = getRunpodLazyUrl();
-    if (!url || runpodLastPhase !== 'green') {
+    if (!url) {
         return;
+    }
-    // The proxy also rejects wake-on-ping; this phase guard avoids unnecessary
+    // This only renews the proxy's frontend lease and never starts a pod. The
-    // requests whenever the last read-only status poll says the pod is down.
+    // proxy owns the stable upstream keepalive cadence so browser timer
+    // throttling cannot directly skew it.
     fetch(`${url}/lazy/ping`, { method: 'POST', signal: AbortSignal.timeout(5000) }).catch(() => { });
+}
-/** (Re)starts the polling/keepalive loops based on the configured URL. */
+/** (Re)starts the status polling and frontend-heartbeat loops. */
 function setupRunpodLoops() {
     clearTimeout(runpodStatusTimer);
-    clearInterval(runpodPingTimer);
+    clearInterval(runpodHeartbeatTimer);
     ensureRunpodBarDot();
     const barDot = document.getElementById('sd_runpod_bar_dot');
     if (barDot) {

@@ -1714,7 +1715,8 @@ function setupRunpodLoops() {
1714	return;	1715	return;
1715	}	1716	}
1716	pollRunpodStatus();	1717	pollRunpodStatus();
1717	runpodPingTimer = setInterval(runpodKeepalive, RUNPOD_PING_MS);	1718	runpodHeartbeat();
		1719	runpodHeartbeatTimer = setInterval(runpodHeartbeat, RUNPOD_HEARTBEAT_MS);
1718	}	1720	}
1719		1721
1720	function onRunpodLazyUrlInput() {	1722	function onRunpodLazyUrlInput() {

public/scripts/extensions/stable-diffusion/settings.html+1 -1 Viewed

                     <span data-i18n="Copy ComfyUI URL">Copy ComfyUI URL</span>
                 </div>
             </div>
-            <small data-i18n="sd_runpod_small">The pod starts only when you press Warm up. A SillyTavern tab keeps an already-ready pod alive; it shuts down 15 minutes after the last activity. Image requests fail over instead of starting a stopped pod. Red = off, orange = starting/downloading models, green = ready.</small>
+            <small data-i18n="sd_runpod_small">The pod starts only when you press Warm up. A SillyTavern tab sends a heartbeat every 20 seconds; the proxy keeps an already-ready pod alive on its own stable timer and tolerates browser sleep for up to 5 minutes. The pod shuts down 15 minutes after keepalives and other activity stop. Image requests fail over instead of starting a stopped pod. Red = off, orange = starting/downloading models, green = ready.</small>
             <h5 data-i18n="Model catalog">Model catalog</h5>
             <small data-i18n="sd_runpod_models_small">When the ComfyUI URL points at the proxy, the Model dropdown lists these entries instead of asking the backend. Press Warm up to apply the selected model and download its files; changing the selection or generating an image never starts or provisions a stopped pod. Filename = what &quot;%model%&quot; resolves to; downloads = one &quot;subpath url&quot; per line, including companion files (text encoder, VAE, LoRA).</small>
             <div id="sd_runpod_models_list" class="flex-container flexFlowColumn marginTopBot5"></div>

src/endpoints/keepalive.js+264 -0 Viewed

+import express from 'express';
+import https from 'node:https';
+import fetch from 'node-fetch';
+export const router = express.Router();
+export const FRONTEND_LEASE_MS = 5 * 60 * 1000;
+const MIN_INTERVAL_MS = 60 * 1000;
+const MAX_INTERVAL_MS = 24 * 60 * 60 * 1000;
+const MAX_JOB_ID_LENGTH = 128;
+const ALLOWED_ENDPOINTS = new Set([
+    '/api/backends/chat-completions/generate',
+    '/api/backends/text-completions/generate',
+    '/api/backends/kobold/generate',
+    '/api/novelai/generate',
+    '/api/horde/generate-text',
+]);
+const localHttpsAgent = new https.Agent({ rejectUnauthorized: false });
+function advanceDeadline(deadline, interval, now) {
+    if (deadline > now) {
+        return deadline;
+    }
+    const missedIntervals = Math.floor((now - deadline) / interval) + 1;
+    return deadline + missedIntervals * interval;
+}
+/**
+ * Backend-owned scheduler for prompt-cache keepalive requests.
+ */
+export class KeepaliveScheduler {
+    #jobs = new Map();
+    #timer = null;
+    #now;
+    #dispatch;
+    #setTimer;
+    #clearTimer;
+    constructor({
+        now = Date.now,
+        dispatch = dispatchKeepaliveRequest,
+        setTimer = setTimeout,
+        clearTimer = clearTimeout,
+    } = {}) {
+        this.#now = now;
+        this.#dispatch = dispatch;
+        this.#setTimer = setTimer;
+        this.#clearTimer = clearTimer;
+    }
+    register(key, job) {
+        const now = this.#now();
+        const intervalMs = Math.min(MAX_INTERVAL_MS, Math.max(MIN_INTERVAL_MS, Number(job.intervalMs) || MIN_INTERVAL_MS));
+        const requestedDelay = Number(job.delayMs);
+        const delayMs = Number.isFinite(requestedDelay)
+            ? Math.min(intervalMs, Math.max(0, requestedDelay))
+            : intervalMs;
+        this.#jobs.set(key, {
+            ...job,
+            intervalMs,
+            lastSeen: now,
+            nextRun: now + delayMs,
+            inFlight: false,
+        });
+        this.#schedule();
+        return this.#jobs.get(key);
+    }
+    heartbeat(key, activity = false, request = null) {
+        const job = this.#jobs.get(key);
+        if (!job) {
+            return null;
+        }
+        const now = this.#now();
+        if ((now - job.lastSeen) >= FRONTEND_LEASE_MS) {
+            this.#jobs.delete(key);
+            this.#schedule();
+            return null;
+        }
+        job.lastSeen = now;
+        if (activity) {
+            job.nextRun = now + job.intervalMs;
+        }
+        if (request) {
+            job.request = request;
+        }
+        this.#schedule();
+        return job;
+    }
+    stop(key) {
+        const removed = this.#jobs.delete(key);
+        this.#schedule();
+        return removed;
+    }
+    get(key) {
+        return this.#jobs.get(key) ?? null;
+    }
+    async tick(now = this.#now()) {
+        const pending = [];
+        for (const [key, job] of this.#jobs) {
+            if ((now - job.lastSeen) >= FRONTEND_LEASE_MS) {
+                this.#jobs.delete(key);
+                continue;
+            }
+            if (job.nextRun > now) {
+                continue;
+            }
+            job.nextRun = advanceDeadline(job.nextRun, job.intervalMs, now);
+            if (job.inFlight) {
+                continue;
+            }
+            job.inFlight = true;
+            pending.push(Promise.resolve(this.#dispatch(job)).catch(error => {
+                console.debug(`[Keepalive] Backend request failed for ${job.id}:`, error);
+            }).finally(() => {
+                job.inFlight = false;
+                this.#schedule();
+            }));
+        }
+        this.#schedule();
+        await Promise.all(pending);
+    }
+    #schedule() {
+        if (this.#timer) {
+            this.#clearTimer(this.#timer);
+            this.#timer = null;
+        }
+        if (!this.#jobs.size) {
+            return;
+        }
+        let nextWake = Infinity;
+        for (const job of this.#jobs.values()) {
+            nextWake = Math.min(nextWake, job.nextRun, job.lastSeen + FRONTEND_LEASE_MS);
+        }
+        const delay = Math.max(0, nextWake - this.#now());
+        this.#timer = this.#setTimer(() => {
+            this.#timer = null;
+            void this.tick();
+        }, delay);
+        this.#timer?.unref?.();
+    }
+}
+async function dispatchKeepaliveRequest(job) {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(new Error('Keepalive request timed out')), 120000);
+    timeout.unref?.();
+    try {
+        const response = await fetch(`${job.request.origin}${job.endpoint}`, {
+            method: 'POST',
+            headers: job.request.headers,
+            body: JSON.stringify(job.payload),
+            signal: controller.signal,
+            agent: job.request.secure ? localHttpsAgent : undefined,
+        });
+        await response.arrayBuffer();
+        if (!response.ok) {
+            throw new Error(`HTTP ${response.status}`);
+        }
+    } finally {
+        clearTimeout(timeout);
+    }
+}
+const scheduler = new KeepaliveScheduler();
+function getJobKey(request, id) {
+    return `${request.user.profile.handle}:${id}`;
+}
+function getJobId(request) {
+    const id = String(request.body?.id ?? '');
+    if (!id || id.length > MAX_JOB_ID_LENGTH || !/^[a-zA-Z0-9_-]+$/.test(id)) {
+        throw new Error('Invalid keepalive job id');
+    }
+    return id;
+}
+function getReplayRequest(request) {
+    const localPort = Number(request.socket.localPort);
+    if (!Number.isInteger(localPort) || localPort < 1 || localPort > 65535) {
+        throw new Error('Unable to determine local server port');
+    }
+    let localAddress = request.socket.localAddress || '127.0.0.1';
+    if (localAddress === '::' || localAddress === '0:0:0:0:0:0:0:0') {
+        localAddress = '::1';
+    } else if (localAddress === '0.0.0.0') {
+        localAddress = '127.0.0.1';
+    } else if (localAddress.startsWith('::ffff:')) {
+        localAddress = localAddress.slice('::ffff:'.length);
+    }
+    const loopbackHost = localAddress.includes(':') ? `[${localAddress}]` : localAddress;
+    const headers = {
+        'Content-Type': 'application/json',
+        'Cookie': request.get('cookie') || '',
+        'X-CSRF-Token': request.get('x-csrf-token') || '',
+        'Host': request.get('host') || '',
+    };
+    for (const name of ['authorization', 'origin', 'referer', 'user-agent']) {
+        const value = request.get(name);
+        if (value) {
+            headers[name] = value;
+        }
+    }
+    const secure = !!request.socket.encrypted;
+    return {
+        origin: `${secure ? 'https' : 'http'}://${loopbackHost}:${localPort}`,
+        headers,
+        secure,
+    };
+}
+router.post('/register', (request, response) => {
+    try {
+        const id = getJobId(request);
+        const endpoint = String(request.body?.endpoint ?? '');
+        if (!ALLOWED_ENDPOINTS.has(endpoint)) {
+            return response.status(400).send({ error: 'Unsupported keepalive endpoint' });
+        }
+        if (!request.body?.payload || typeof request.body.payload !== 'object' || Array.isArray(request.body.payload)) {
+            return response.status(400).send({ error: 'Missing keepalive payload' });
+        }
+        const job = scheduler.register(getJobKey(request, id), {
+            id,
+            endpoint,
+            payload: request.body.payload,
+            intervalMs: request.body.intervalMs,
+            delayMs: request.body.delayMs,
+            request: getReplayRequest(request),
+        });
+        return response.send({ ok: true, nextRun: job.nextRun, leaseMs: FRONTEND_LEASE_MS });
+    } catch (error) {
+        return response.status(400).send({ error: String(error.message || error) });
+    }
+});
+router.post('/heartbeat', (request, response) => {
+    try {
+        const id = getJobId(request);
+        const job = scheduler.heartbeat(getJobKey(request, id), !!request.body?.activity, getReplayRequest(request));
+        if (!job) {
+            return response.sendStatus(404);
+        }
+        return response.send({ ok: true, nextRun: job.nextRun, leaseMs: FRONTEND_LEASE_MS });
+    } catch (error) {
+        return response.status(400).send({ error: String(error.message || error) });
+    }
+});
+router.post('/stop', (request, response) => {
+    try {
+        const id = getJobId(request);
+        scheduler.stop(getJobKey(request, id));
+        return response.send({ ok: true });
+    } catch (error) {
+        return response.status(400).send({ error: String(error.message || error) });
+    }
+});

src/server-startup.js+2 -0 Viewed

 import { router as backupsRouter } from './endpoints/backups.js';
 import { router as imageMetadataRouter } from './endpoints/image-metadata.js';
 import { router as volcengineRouter } from './endpoints/volcengine.js';
+import { router as keepaliveRouter } from './endpoints/keepalive.js';
 /**
  * @typedef {object} ServerStartupResult

     app.use('/api/data-maid', dataMaidRouter);
     app.use('/api/backups', backupsRouter);
     app.use('/api/image-metadata', imageMetadataRouter);
+    app.use('/api/keepalive', keepaliveRouter);
+}
 /**

tests/keepalive-scheduler.test.js+80 -0 Viewed

+import { describe, expect, jest, test } from '@jest/globals';
+import { FRONTEND_LEASE_MS, KeepaliveScheduler } from '../src/endpoints/keepalive.js';
+function createScheduler() {
+    let now = 0;
+    const dispatch = jest.fn().mockResolvedValue(undefined);
+    const scheduler = new KeepaliveScheduler({
+        now: () => now,
+        dispatch,
+        setTimer: () => ({ unref() { } }),
+        clearTimer: () => { },
+    });
+    return { scheduler, dispatch, setNow: value => { now = value; } };
+}
+describe('KeepaliveScheduler', () => {
+    test('heartbeats renew the lease without moving the backend deadline', () => {
+        const { scheduler, setNow } = createScheduler();
+        scheduler.register('user:tab', { id: 'tab', intervalMs: 60000, delayMs: 60000 });
+        setNow(20000);
+        scheduler.heartbeat('user:tab');
+        expect(scheduler.get('user:tab').lastSeen).toBe(20000);
+        expect(scheduler.get('user:tab').nextRun).toBe(60000);
+    });
+    test('a heartbeat cannot revive an already expired lease', () => {
+        const { scheduler, setNow } = createScheduler();
+        scheduler.register('user:tab', { id: 'tab', intervalMs: 60000, delayMs: 60000 });
+        setNow(FRONTEND_LEASE_MS);
+        expect(scheduler.heartbeat('user:tab')).toBeNull();
+        expect(scheduler.get('user:tab')).toBeNull();
+    });
+    test('accepts an immediate first deadline', () => {
+        const { scheduler } = createScheduler();
+        scheduler.register('user:tab', { id: 'tab', intervalMs: 60000, delayMs: 0 });
+        expect(scheduler.get('user:tab').nextRun).toBe(0);
+    });
+    test('dispatches on the backend deadline and stays aligned after a late tick', async () => {
+        const { scheduler, dispatch, setNow } = createScheduler();
+        scheduler.register('user:tab', { id: 'tab', intervalMs: 60000, delayMs: 60000 });
+        setNow(60000);
+        await scheduler.tick();
+        expect(dispatch).toHaveBeenCalledTimes(1);
+        expect(scheduler.get('user:tab').nextRun).toBe(120000);
+        setNow(190000);
+        await scheduler.tick();
+        expect(dispatch).toHaveBeenCalledTimes(2);
+        expect(scheduler.get('user:tab').nextRun).toBe(240000);
+    });
+    test('activity resets the next keepalive deadline', () => {
+        const { scheduler, setNow } = createScheduler();
+        scheduler.register('user:tab', { id: 'tab', intervalMs: 60000, delayMs: 60000 });
+        setNow(50000);
+        scheduler.heartbeat('user:tab', true);
+        expect(scheduler.get('user:tab').nextRun).toBe(110000);
+    });
+    test('removes a job after five minutes without a frontend heartbeat', async () => {
+        const { scheduler, dispatch, setNow } = createScheduler();
+        scheduler.register('user:tab', { id: 'tab', intervalMs: 60000, delayMs: 60000 });
+        setNow(FRONTEND_LEASE_MS);
+        await scheduler.tick();
+        expect(scheduler.get('user:tab')).toBeNull();
+        expect(dispatch).not.toHaveBeenCalled();
+    });
+});