Move keepalive scheduling to backend

18bf41c8bdd4f16a5745803d1dc7f4779c15387b

permissionBRICK <40219477+permissionBRICK@users.noreply.github.com>

Parents: abc1cb103bccdb271ac27a2e4ae81b744cd67c57

homelab/README.md+1 -1 Viewed

 | File | Deployed at | Purpose |
 |---|---|---|
 | `whisper-lazy-proxy.py` | `/mnt/user/appdata/whisper-lazy/proxy.py` (container `whisper-lazy`, :10300) | Wyoming lazy proxy: answers HA availability checks from cache, starts `faster-whisper` on demand, stops it after 15 min idle (frees GPU VRAM). |
 | `runpod-lazy-proxy.py` | `/mnt/user/appdata/runpod-lazy/proxy.py` (container `runpod-lazy`, :8189) | ComfyUI-compatible on-demand proxy for a volumeless RunPod GPU pod: only `/lazy/warmup` starts or provisions, while cold ComfyUI requests return 503 for provider failover. `/lazy/status|warmup|shutdown|ping|catalog` providerecords SillyTavern20-second controlsfrontend heartbeats; the proxy owns the 60-second upstream keepalive cadence and continues it through browser timer pauses up to a 5-minute lease. A NAS-side idle timer terminates the pod 15-min idleminutes terminateafter keepalives and other activity stop. Model-set changes requested by Warm up download in place via the worker image's model-manager (port 8189, LRU disk eviction); pods are only recreated when none exists or the image predates the manager. |
 | `runpod-pod.sh` | `/boot/config/runpod-pod.sh` | CLI for the runpod-lazy control API (`status|warmup|shutdown|ping|watch`). |
 Related: the RunPod worker image (ComfyUI + custom nodes + boot-time model

homelab/runpod-lazy-proxy.py+62 -6 Viewed

   GET  /lazy/status    -> {"state": red|orange|green, "model": ..., ...}
   POST /lazy/warmup    -> start pod for the active catalog entry
   POST /lazy/shutdown  -> terminate pod now
   POST /lazy/ping      -> keepalive forrecord a readyfrontend podheartbeat (never starts a pod)
   POST /lazy/catalog   -> {"models": [...], "active": "<value>"} store catalog;
                           never starts or provisions a pod
 Start semantics: only POST /lazy/warmup may provision or change models. ComfyUI
 requests return 503 unless a pod is already ready, allowing SillyTavern's image
 target fallback chain to continue without waking this target. The idleBrowser timerheartbeats
 -lives here (NAS side): IDLE_SECONDS after the last activity signal the pod is
 +grant a short lease during which this proxy sends keepalive probes on a stable
 -terminated, frontend or no frontend.
 +backend cadence. The idle timer lives here (NAS side): IDLE_SECONDS after the
 +last activity signal the pod is terminated, frontend or no frontend.
 """
 import json
 import os

 RUNPOD_KEY = os.environ['RUNPOD_KEY']
 LISTEN_PORT = int(os.environ.get('LISTEN_PORT', '8189'))
 IDLE_SECONDS = int(os.environ.get('IDLE_SECONDS', '900'))
 +KEEPALIVE_SECONDS = max(1, int(os.environ.get('KEEPALIVE_SECONDS', '60')))
 +FRONTEND_LEASE_SECONDS = max(1, int(os.environ.get('FRONTEND_LEASE_SECONDS', '300')))
 DATACENTERS = [d for d in os.environ.get('DATACENTERS', '').split(',') if d]
 CLOUD_TYPE = os.environ.get('CLOUD_TYPE', 'SECURE')
 IMAGE = os.environ.get('IMAGE', 'ghcr.io/permissionbrick/comfyui-runpod-worker:latest')

     'gpu': None,
     'since': time.time(),
     'last': time.monotonic(),
 +    'frontend_last_seen': None,
 +    'keepalive_last': None,
     'ensuring': 0,      # active ensure_pod waiters (status skips probing then)
     'control_epoch': 0, # incremented by shutdown to cancel in-flight warmups
     'lock': threading.Lock(),

         time.sleep(30)
         idle = time.monotonic() - state['last']
         if idle > IDLE_SECONDS:
 +            # A heartbeat may arrive just before the next scheduled backend
 +            # probe. Its live lease protects the pod until that probe catches
 +            # up, without relying on the browser to refresh the idle clock.
 +            if frontend_lease_seconds_left() > 0:
 +                continue
             pod_id = state['pod_id'] or find_pod()[0]
             if pod_id:
                 log(f'idle {int(idle)}s - terminating pod')

                 state.update({'pod_id': None, 'model': None, 'gpu': None, 'phase': 'red', 'since': time.time()})
 +def frontend_lease_seconds_left(now=None):
 +    """Returns the remaining frontend heartbeat lease in whole seconds."""
 +    now = time.monotonic() if now is None else now
 +    last_seen = state['frontend_last_seen']
 +    if last_seen is None:
 +        return 0
 +    return max(0, FRONTEND_LEASE_SECONDS - int(now - last_seen))
++
++
 +def maintain_frontend_keepalive(now=None):
 +    """Probes and refreshes a ready pod while a frontend heartbeat lease is live."""
 +    now = time.monotonic() if now is None else now
 +    with state['lock']:
 +        pod_id = state['pod_id']
 +        lease_active = frontend_lease_seconds_left(now) > 0
 +        ready = state['phase'] == 'green' and state['ensuring'] == 0
 +    if not pod_id or not lease_active or not ready or not upstream_ready(pod_id):
 +        return False
 +    with state['lock']:
 +        # The network probe runs outside the lock. Re-check that its result still
 +        # belongs to the current pod and a still-live frontend lease.
 +        if state['pod_id'] != pod_id or frontend_lease_seconds_left(now) <= 0:
 +            return False
 +        state['last'] = now
 +        state['keepalive_last'] = now
 +    return True
++
++
 +def keepalive_scheduler():
 +    """Runs keepalive probes on a monotonic backend cadence without timer drift."""
 +    next_tick = time.monotonic() + KEEPALIVE_SECONDS
 +    while True:
 +        time.sleep(max(0, next_tick - time.monotonic()))
 +        maintain_frontend_keepalive()
 +        next_tick += KEEPALIVE_SECONDS
 +        now = time.monotonic()
 +        if next_tick <= now:
 +            # Skip missed ticks after a long process pause instead of issuing a
 +            # burst of stale keepalives, while keeping future ticks aligned.
 +            missed = int((now - next_tick) // KEEPALIVE_SECONDS) + 1
 +            next_tick += missed * KEEPALIVE_SECONDS
++
++
 def status_body():
     pod_id = state['pod_id']
     previous_phase = state['phase']

         'since': state['since'],
         'url': pod_url(state['pod_id']) if state['pod_id'] else None,
         'idle_seconds_left': max(0, IDLE_SECONDS - int(time.monotonic() - state['last'])) if state['pod_id'] else 0,
 +        'frontend_lease_seconds_left': frontend_lease_seconds_left(),
 +        'keepalive_active': bool(state['pod_id'] and state['phase'] == 'green' and frontend_lease_seconds_left() > 0),
     }).encode()

         if path in ('/lazy/ping', '/lazy/warmup', '/lazy/shutdown', '/lazy/catalog') and self.command != 'POST':
             return self._reply(405, b'{"error": "method not allowed"}')
         if path == '/lazy/ping':
 -            if state['pod_id'] and state['phase'] == 'green':
 +            with state['lock']:
                 state['lastfrontend_last_seen'] = time.monotonic()
             return self._reply(200, b'{"ok": true}')
         if path == '/lazy/warmup':
             with state['lock']:

 def main():
     threading.Thread(target=idle_reaper, daemon=True).start()
 +    threading.Thread(target=keepalive_scheduler, daemon=True).start()
     # Adopt a pre-existing pod after a proxy restart without provisioning or
     # changing it. Read-only status polls will update its readiness phase.
     pod_id, have, gpu = find_pod()

homelab/runpod-pod.sh+1 -1 Viewed

 #!/bin/sh
 # CLI for the runpod-lazy proxy: status | warmup | shutdown | ping (renew frontend lease)
 # Usable by anyone (or any AI) without SillyTavern. Proxy owns the pod
 # lifecycle; this only talks to its control API.
 CMD="${1:-status}"

homelab/tests/test_runpod_lazy_proxy.py+37 -1 Viewed

             'gpu': None,
             'since': 0,
             'last': 100,
 +            'frontend_last_seen': None,
 +            'keepalive_last': None,
             'ensuring': 0,
             'control_epoch': 0,
         })

             self.assertEqual(status, 200)
             self.assertEqual(body['state'], 'red')
 -            status, _ = self.request('POST', '/lazy/ping')
 +            with mock.patch.object(proxy.time, 'monotonic', return_value=120):
 +                status, _ = self.request('POST', '/lazy/ping')
             self.assertEqual(status, 200)
             self.assertEqual(proxy.state['last'], 100)
 +            self.assertEqual(proxy.state['frontend_last_seen'], 120)
             catalog = {
                 'models': [{'name': 'Test', 'value': 'test.safetensors', 'files': []}],

         self.assertEqual(status, 200)
         self.assertEqual(proxy.state['last'], 100)
 +    def test_backend_maintains_keepalive_during_frontend_lease(self):
 +        proxy.state.update({
 +            'pod_id': 'ready-pod',
 +            'phase': 'green',
 +            'last': 100,
 +            'frontend_last_seen': 100,
 +        })
++
 +        with mock.patch.object(proxy, 'upstream_ready', return_value=True) as upstream_ready:
 +            maintained = proxy.maintain_frontend_keepalive(now=399)
++
 +        self.assertTrue(maintained)
 +        self.assertEqual(proxy.state['last'], 399)
 +        self.assertEqual(proxy.state['keepalive_last'], 399)
 +        upstream_ready.assert_called_once_with('ready-pod')
++
 +    def test_backend_stops_keepalive_after_frontend_lease_expires(self):
 +        proxy.state.update({
 +            'pod_id': 'ready-pod',
 +            'phase': 'green',
 +            'last': 100,
 +            'frontend_last_seen': 100,
 +        })
++
 +        with mock.patch.object(proxy, 'upstream_ready') as upstream_ready:
 +            maintained = proxy.maintain_frontend_keepalive(now=401)
++
 +        self.assertFalse(maintained)
 +        self.assertEqual(proxy.state['last'], 100)
 +        self.assertIsNone(proxy.state['keepalive_last'])
 +        upstream_ready.assert_not_called()
++
     def test_get_warmup_is_rejected_without_provisioning(self):
         with mock.patch.object(proxy, 'ensure_pod') as ensure_pod:
             status, body = self.request('GET', '/lazy/warmup')

public/scripts/extensions/keepalive/index.js+259 -119 Viewed

 import { extension_settings, renderExtensionTemplateAsync } from '../../extensions.js';
 import {
     activateSendButtonsGenerate,
 -    deactivateSendButtons,
     eventSource,
     event_types,
     extension_prompt_roles,
     extension_prompt_types,
     generateQuietPromptgetGenerateUrl,
 +    getRequestHeaders,
     isGenerating,
 +    main_api,
     saveSettingsDebounced,
     setExtensionPrompt,
 -    setSendButtonState,
 -    streamingProcessor,
 } from '../../../script.js';
 +import { createGenerationParameters, getChatCompletionModel, oai_settings } from '../../openai.js';
 +import { horde_settings } from '../../horde.js';
 +import { uuidv4 } from '../../utils.js';
 export { init };

 // still processed by the backend (which is what refreshes the server-side
 // cache), but we never pay for a full completion.
 const KEEPALIVE_RESPONSE_LENGTH = 1;
 +const HEARTBEAT_INTERVAL_MS = 20000;
 +const RETRY_INTERVAL_MS = 5000;
 +const SETTINGS_REFRESH_DELAY_MS = 300;
 // Extension-prompt key used to inject the keepalive message as a USER message at depth 0.
 const KEEPALIVE_INJECT_ID = 'keepalive_ping';

 // user message can pre-empt the keepalive.
 const EXPIRED_GRACE_MS = 3000;
 -// If the idle timer fires more than this many ms later than it was scheduled to, assume the
 -// machine slept / the tab was throttled — far more wall-clock time passed than the interval,
 -// so the cache almost certainly lapsed. Skip the ping and go dormant rather than guess.
 -const MAX_TIMER_DRIFT_MS = 5000;
+-
 const defaultSettings = {
     enabled: false,
     timeoutSeconds: 295,

     profiles: {},
 };
 let idleTimerheartbeatTimer = null;
 let keepaliveActivejobRegistered = false;
 +let refreshPromise = null;
 +let refreshRequested = false;
 +let refreshTimer = null;
 +let preparingJob = false;
 +let lifecycleVersion = 0;
 // Whether keepalive is "armed" for the CURRENT chat. It stays dormant until the user sends a new
 // message in the open chat — on a freshly loaded or switched-to chat (even with history already
 // present) we have no way to know whether its prompt is still cached server-side, so pinging would
 // be pointless. Reset on page reload and whenever the chat changes.
 let armed = false;
 -// Wall-clock time (Date.now() + delay) the pending idle timer is expected to fire, or null when
 -// none is scheduled. Used to detect a timer that fired far too late (sleep/throttle).
 -let scheduledFireTime = null;
 // Per-profile "time since last message", keyed by connection profile id (or NONE_KEY).
 // In-memory only: on reload every profile is treated as freshly active.
 const lastActivity = {};
 +// Per-document ids keep duplicated tabs independent. Jobs orphaned by a closed or reloaded
 +// document are deliberately cleaned up by the backend lease.
 +const keepaliveJobId = uuidv4().replaceAll('-', '');
++
 /**
  * @returns {string} The active connection profile id, or NONE_KEY for a manual connection.
  */

     return perProfile === undefined ? true : !!perProfile;
+}
 -/**
 +function getTimeoutMs() {
 - * Clears the pending idle timer, if any.
 +    return Math.max(60, Number(extension_settings[MODULE].timeoutSeconds) || 295) * 1000;
 - */
 +}
 -function clearIdleTimer() {
++
 -    if (idleTimer) {
 +async function postKeepaliveBackend(path, body) {
 -        clearTimeout(idleTimer);
 +    return fetch(`/api/keepalive/${path}`, {
 -        idleTimer = null;
 +        method: 'POST',
 +        headers: getRequestHeaders(),
 +        body: JSON.stringify({ id: keepaliveJobId, ...body }),
 +    });
 +}
++
 +async function stopBackendJob() {
 +    lifecycleVersion++;
 +    jobRegistered = false;
 +    try {
 +        await postKeepaliveBackend('stop', {});
 +    } catch (error) {
 +        console.debug('[Keepalive] Could not stop backend job:', error);
+    }
+}
 -/**
 +async function sendBackendHeartbeat(activity = false) {
 - * (Re)schedules the keepalive for the ACTIVE profile based on how long it has been since
 +    if (!jobRegistered || !armed || !isKeepaliveEnabledForActive()) {
 - * that profile's last message. Does nothing (and leaves the timer cleared) until keepalive
 - * has been armed for the current chat, or when it is disabled for the active connection.
 - * Called on activity AND whenever the active profile changes, so each profile keeps its own
 - * independent idle clock.
 - */
 -function scheduleIdleTimer() {
 -    clearIdleTimer();
 -    if (!armed || !isKeepaliveEnabledForActive()) {
 -        scheduledFireTime = null;
         return;
+    }
 -    const timeoutMs = Math.max(60, Number(extension_settings[MODULE].timeoutSeconds) || 295) * 1000;
 +    try {
 -    const last = lastActivity[getActiveProfileKey()] ?? Date.now();
 +        const response = await postKeepaliveBackend('heartbeat', { activity });
 -    const remaining = timeoutMs - (Date.now() - last);
 +        if (response.status === 404) {
 -    const delay = remaining > 0 ? remaining : EXPIRED_GRACE_MS;
 +            jobRegistered = false;
 -    scheduledFireTime = Date.now() + delay;
 +            queueJobRefresh();
 -    idleTimer = setTimeout(fireKeepalive, delay);
 +        }
 +    } catch (error) {
 +        console.debug('[Keepalive] Backend heartbeat failed:', error);
 +    }
+}
 -/**
 - * Marks the active profile as just-used (resets its idle clock) and reschedules its keepalive.
 - */
 function markActivity() {
 +    if (preparingJob) {
 +        return;
 +    }
     lastActivity[getActiveProfileKey()] = Date.now();
     scheduleIdleTimervoid sendBackendHeartbeat(true);
+}
 /**

  */
 function onChatChanged() {
     armed = false;
     scheduleIdleTimervoid stopBackendJob();
+}
 /**
 - * Fires a background ("quiet") generation against the active model to keep the
 + * Caps a prepared request to a single, non-streaming completion token.
 - * server-side prompt cache warm. The full prompt prefix (current chat context +
 + * @param {string} endpoint Backend generation endpoint.
 - * the keepalive message) is sent — which is what refreshes the cache — while the
 + * @param {object} payload Prepared generation payload.
 - * completion is capped to a single token (KEEPALIVE_RESPONSE_LENGTH) so we never
 + * @returns {object} Cloned and capped payload.
 - * pay for a real response.
 - *
 - * Only the CURRENTLY ACTIVE profile is ever pinged — keepalive never switches the
 - * connection in the background. Each profile's idle clock is tracked separately, so
 - * switching profiles re-targets the timer at whichever profile is now active.
 - *
 - * Quiet generations are forced non-streaming by the core (Generate excludes
 - * type === 'quiet' from the StreamingProcessor) and are never saved to chat, so the
 - * output is simply discarded. generateQuietPrompt's responseLength applies a *global*
 - * temporary response-length override for the duration of the request, so we hold the
 - * same generation lock the blocking summary path uses — setSendButtonState(true) flips
 - * `is_send_press` (which every user send entry point checks and bails on) and
 - * deactivateSendButtons() reflects the busy state — and release both in `finally`.
  */
 async function fireKeepalivecapKeepalivePayload(endpoint, payload) {
 -    if (!armed || !isKeepaliveEnabledForActive()) {
 +    const capped = structuredClone(payload);
 -        return;
 +    capped.stream = false;
 +    for (const key of ['max_tokens', 'max_completion_tokens', 'max_length', 'max_new_tokens', 'n_predict']) {
 +        if (Object.hasOwn(capped, key)) {
 +            capped[key] = KEEPALIVE_RESPONSE_LENGTH;
 +        }
 +    }
 +    if (endpoint === '/api/backends/chat-completions/generate'
 +        && !Object.hasOwn(capped, 'max_tokens')
 +        && !Object.hasOwn(capped, 'max_completion_tokens')) {
 +        capped.max_tokens = KEEPALIVE_RESPONSE_LENGTH;
+    }
 +    if (Object.hasOwn(capped, 'n')) {
 +        capped.n = 1;
 +    }
 +    return capped;
 +}
 -    // If the timer fired much later than scheduled (machine sleep, background-tab throttling,
 +/**
 -    // etc.), far more wall-clock time has elapsed than the idle interval, so the cache has
 + * Dry-runs prompt assembly and returns a request the backend can replay later.
 -    // almost certainly expired. Don't ping on a stale assumption — go dormant until the next
 + * @returns {Promise<{endpoint: string, payload: object}>}
 -    // sent message re-arms keepalive, exactly like a freshly opened chat.
 + */
 -    if (scheduledFireTime !== null && (Date.now() - scheduledFireTime) > MAX_TIMER_DRIFT_MS) {
 +async function prepareKeepaliveRequest() {
 -        console.debug('[Keepalive] Idle timer fired', Date.now() - scheduledFireTime, 'ms late; skipping and disarming until next activity.');
 +    const api = main_api;
         armed    let capturedData = falsenull;
 -        clearIdleTimer();
 +    const captureData = (data, dryRun) => {
 -        scheduledFireTime = null;
 +        if (dryRun && preparingJob) {
 -        return;
 +            capturedData = structuredClone(data);
 +        }
 +    };
++
 +    preparingJob = true;
 +    eventSource.on(event_types.GENERATE_AFTER_DATA, captureData);
 +    setExtensionPrompt(KEEPALIVE_INJECT_ID, extension_settings[MODULE].message, extension_prompt_types.IN_CHAT, 0, false, extension_prompt_roles.USER);
 +    try {
 +        await Generate('quiet', { quiet_prompt: '', force_name2: true }, true);
 +    } finally {
 +        setExtensionPrompt(KEEPALIVE_INJECT_ID, '', extension_prompt_types.IN_CHAT, 0, false, extension_prompt_roles.USER);
 +        eventSource.removeListener(event_types.GENERATE_AFTER_DATA, captureData);
 +        preparingJob = false;
+    }
 -    // Never start while any generation is in progress, or while a previous
 +    if (!capturedData || api !== main_api) {
 -    // keepalive is still running. Reschedule and bail.
 +        throw new Error('Active connection changed while preparing the keepalive request');
 -    if (keepaliveActive || isGenerating() || streamingProcessor) {
 -        scheduleIdleTimer();
 -        return;
+    }
     keepaliveActive =let trueendpoint;
 -    setSendButtonState(true);
 +    let payload;
 -    deactivateSendButtons();
 +    if (api === 'openai') {
 +        endpoint = '/api/backends/chat-completions/generate';
 +        const model = getChatCompletionModel(oai_settings);
 +        ({ generate_data: payload } = await createGenerationParameters(oai_settings, model, 'quiet', capturedData.prompt));
 +        await eventSource.emit(event_types.CHAT_COMPLETION_SETTINGS_READY, payload);
 +    } else if (api === 'koboldhorde') {
 +        endpoint = '/api/horde/generate-text';
 +        const params = structuredClone(capturedData);
 +        const prompt = params.prompt;
 +        delete params.prompt;
 +        Object.assign(params, {
 +            n: 1,
 +            max_length: KEEPALIVE_RESPONSE_LENGTH,
 +            frmtadsnsp: false,
 +            frmtrmblln: false,
 +            frmtrmspch: false,
 +            frmttriminc: false,
 +        });
 +        payload = {
 +            prompt,
 +            params,
 +            trusted_workers: horde_settings.trusted_workers_only,
 +            models: horde_settings.models,
 +        };
 +    } else {
 +        endpoint = getGenerateUrl(api);
 +        payload = capturedData;
 +    }
 +    return { endpoint, payload: capKeepalivePayload(endpoint, payload) };
 +}
++
 +async function registerBackendJob() {
 +    if (!armed || !isKeepaliveEnabledForActive()) {
 +        await stopBackendJob();
 +        return;
 +    }
 +    const version = lifecycleVersion;
     try {
 -        // Deliver the keepalive as a USER message at depth 0 (the quiet-prompt path forces a
 +        if (isGenerating()) {
 -        // system role — openai.js hardcodes role:'system' for it), using the same depth+role
 +            throw new Error('Waiting for the active generation to finish');
 -        // injection Author's Note uses. The quiet prompt itself stays empty so the full chat
 +        }
 -        // context is still assembled (which is what keeps the cached prefix warm).
 +        const request = await prepareKeepaliveRequest();
 -        setExtensionPrompt(KEEPALIVE_INJECT_ID, extension_settings[MODULE].message, extension_prompt_types.IN_CHAT, 0, false, extension_prompt_roles.USER);
 +        if (version !== lifecycleVersion || !armed || !isKeepaliveEnabledForActive()) {
 -        await generateQuietPrompt({ quietPrompt: '', responseLength: KEEPALIVE_RESPONSE_LENGTH });
 +            return;
 -        // The ping kept this profile warm; treat it as fresh activity so the next
 +        }
 -        // keepalive for it is a full timeout away.
 +        const timeoutMs = getTimeoutMs();
         const last = lastActivity[getActiveProfileKey()] =?? Date.now();
 +        const remaining = timeoutMs - (Date.now() - last);
 +        const delayMs = remaining > 0 ? remaining : EXPIRED_GRACE_MS;
 +        const response = await postKeepaliveBackend('register', {
 +            ...request,
 +            intervalMs: timeoutMs,
 +            delayMs,
 +        });
 +        if (!response.ok) {
 +            throw new Error(`HTTP ${response.status}`);
 +        }
 +        if (version !== lifecycleVersion || !armed || !isKeepaliveEnabledForActive()) {
 +            await stopBackendJob();
 +            return;
 +        }
 +        jobRegistered = true;
     } catch (error) {
 -        // Best-effort: any failure here is harmless and intentionally swallowed.
 +        jobRegistered = false;
         console.debug('[Keepalive] BackgroundCould generationnot endedregister backend job:', error);
 -    } finally {
 +        setTimeout(queueJobRefresh, RETRY_INTERVAL_MS);
 -        setExtensionPrompt(KEEPALIVE_INJECT_ID, '', extension_prompt_types.IN_CHAT, 0, false, extension_prompt_roles.USER);
 -        setSendButtonState(false);
 -        activateSendButtons();
 -        keepaliveActive = false;
 -        scheduleIdleTimer();
+    }
+}
 +function queueJobRefresh() {
 +    refreshRequested = true;
 +    if (refreshPromise) {
 +        return;
 +    }
 +    refreshPromise = (async () => {
 +        while (refreshRequested) {
 +            refreshRequested = false;
 +            await registerBackendJob();
 +        }
 +    })().finally(() => {
 +        refreshPromise = null;
 +    });
 +}
++
 +function scheduleJobRefresh() {
 +    clearTimeout(refreshTimer);
 +    refreshTimer = setTimeout(queueJobRefresh, SETTINGS_REFRESH_DELAY_MS);
 +}
++
 +function setupHeartbeatLoop() {
 +    clearInterval(heartbeatTimer);
 +    heartbeatTimer = setInterval(() => void sendBackendHeartbeat(), HEARTBEAT_INTERVAL_MS);
 +    document.addEventListener('visibilitychange', () => {
 +        if (document.visibilityState === 'visible') {
 +            void sendBackendHeartbeat();
 +        }
 +    });
 +}
++
 /**
  * Renders the per-profile enable checkboxes from the current Connection Manager profiles.
  */

     $('#keepalive_enabled').on('change', function () {
         extension_settings[MODULE].enabled = !!$(this).prop('checked');
         saveSettingsDebounced();
 -        scheduleIdleTimer();
 +        if (extension_settings[MODULE].enabled && armed) {
 +            queueJobRefresh();
 +        } else {
 +            void stopBackendJob();
 +        }
     });
     $('#keepalive_timeout').on('input', function () {
         extension_settings[MODULE].timeoutSeconds = Number($(this).val()) || 295;
         saveSettingsDebounced();
 -        scheduleIdleTimer();
 +        if (armed && isKeepaliveEnabledForActive()) {
 +            scheduleJobRefresh();
 +        }
     });
     $('#keepalive_message').on('input', function () {
         extension_settings[MODULE].message = String($(this).val());
         saveSettingsDebounced();
 +        if (armed && isKeepaliveEnabledForActive()) {
 +            scheduleJobRefresh();
 +        }
     });
     // Per-profile toggles are rendered dynamically; use a delegated handler.

+        }
         extension_settings[MODULE].profiles[profileId] = !!$(this).prop('checked');
         saveSettingsDebounced();
         // If the toggled profile is the one currently active, rescheduleupdate its backend job immediately.
         if (profileId === extension_settings.connectionManager?.selectedProfile) {
 -            scheduleIdleTimer();
 +            if (isKeepaliveEnabledForActive() && armed) {
 +                queueJobRefresh();
 +            } else {
 +                void stopBackendJob();
 +            }
+        }
     });
+}
 +function onGenerationStarted(_type, _options, dryRun) {
 +    if (!dryRun) {
 +        markActivity();
 +    }
 +}
++
 +function onGenerationEnded() {
 +    markActivity();
 +    if (armed && isKeepaliveEnabledForActive()) {
 +        queueJobRefresh();
 +    }
 +}
++
 +function onChatContentChanged() {
 +    markActivity();
 +    if (armed && isKeepaliveEnabledForActive()) {
 +        queueJobRefresh();
 +    }
 +}
++
 +async function onConnectionChanged() {
 +    // Remove the previous connection's request before assembling one with the new settings.
 +    await stopBackendJob();
 +    if (armed && isKeepaliveEnabledForActive()) {
 +        queueJobRefresh();
 +    }
 +}
++
 async function init() {
     const settingsHtml = await renderExtensionTemplateAsync(MODULE, 'settings');
     $('#extensions_settings2').append(settingsHtml);
     loadSettings();
     setupListeners();
 +    setupHeartbeatLoop();
     // Only the user sending a new message arms keepalive (for the current chat).
     eventSource.on(event_types.MESSAGE_SENT, onMessageSent);
     // TheseActivity resetresets the active profile'sbackend idledeadline clockbut (sonever generationarms timekeepalive isn'ton countedits asown. idle)A butcompleted
     // nevergeneration armor keepalivea onchat theiredit ownalso —refreshes theythe alsostored firerequest forso background/quietthe generationsserver andalways whilereplays
 -    // a chat is loading. They only re-arm the timer if a sent message already armed it.
 +    // the latest prompt assembled by the UI.
 -    const activityEvents = [
 +    eventSource.on(event_types.MESSAGE_RECEIVED, markActivity);
 -        event_types.MESSAGE_RECEIVED,
 +    eventSource.on(event_types.GENERATION_STARTED, onGenerationStarted);
 -        event_types.GENERATION_STARTED,
 +    eventSource.on(event_types.GENERATION_ENDED, onGenerationEnded);
 -        event_types.GENERATION_ENDED,
 +    eventSource.on(event_types.MESSAGE_EDITED, onChatContentChanged);
 -    ];
 +    eventSource.on(event_types.MESSAGE_DELETED, onChatContentChanged);
 -    for (const event of activityEvents) {
 +    eventSource.on(event_types.MESSAGE_UPDATED, onChatContentChanged);
             eventSource.on(eventevent_types.MESSAGE_SWIPED, markActivityonChatContentChanged);
 -    }
     // Opening/switching a chat disarms keepalive — it must be re-armed by a new sent message.
     eventSource.on(event_types.CHAT_CHANGED, onChatChanged);
     // When the active profile changes, re-targetreplace the idlebackend timerjob atwithout themarking newit profileas usingactivity.
 -    // ITS own last-activity time (do NOT mark activity — switching is not using it).
 +    eventSource.on(event_types.CONNECTION_PROFILE_LOADED, onConnectionChanged);
     eventSource.on(event_types.CONNECTION_PROFILE_LOADEDMAIN_API_CHANGED, scheduleIdleTimeronConnectionChanged);
 +    eventSource.on(event_types.CHATCOMPLETION_SOURCE_CHANGED, onConnectionChanged);
 +    eventSource.on(event_types.CHATCOMPLETION_MODEL_CHANGED, onConnectionChanged);
 +    eventSource.on(event_types.PRESET_CHANGED, onConnectionChanged);
     // Keep the per-profile toggle list in sync with Connection Manager.
     eventSource.on(event_types.CONNECTION_PROFILE_CREATED, renderProfileToggles);
     eventSource.on(event_types.CONNECTION_PROFILE_UPDATED, renderProfileToggles);
     eventSource.on(event_types.CONNECTION_PROFILE_DELETED, renderProfileToggles);
+-
 -    scheduleIdleTimer();
+}

public/scripts/extensions/keepalive/manifest.json+1 -1 Viewed

     "js": "index.js",
     "css": "",
     "author": "SillyTavern",
     "version": "1.01.0",
     "homePage": "https://github.com/SillyTavern/SillyTavern",
     "hooks": {
         "activate": "init"

public/scripts/extensions/keepalive/settings.html+1 -1 Viewed

             <label for="keepalive_message" data-i18n="ext_keepalive_message">Keepalive message</label>
             <input id="keepalive_message" class="text_pole" type="text" />
             <small data-i18n="ext_keepalive_help">After this many seconds without activity on the active connection profile, the current chat plus this message (sent as a hidden user message) is submitted to that profile to keep its server-side prompt cache warm. The completion is capped to a single token and is never saved to the chat. SendingThe isserver brieflyowns lockedthe keepalive timer, while itthis runstab renews its lease every 20 seconds; if the tab sleeps or closes, the server continues on schedule for up to five minutes after the last heartbeat. Only the profile you're currently using is pinged; each profile has its own idle timer. Keepalive stays off until you send a new message in the current chat (opening, switching to, or reloading a chat isn't proof its prompt is still cached), and it also skips a ping if its timer fired far later than scheduled (e.g. after the machine slept).</small>
             <hr>

public/scripts/extensions/stable-diffusion/index.js+12 -10 Viewed

 /** Poll cadence for the pod status indicator (faster while it is starting). */
 const RUNPOD_POLL_IDLE_MS = 30000;
 const RUNPOD_POLL_BUSY_MS = 5000;
 /** Keepalive cadenceFrontend forheartbeat acadence. podThe thatproxy thetolerates statusbrowser APItimer haspauses confirmedfor isfive readyminutes. */
 const RUNPOD_PING_MSRUNPOD_HEARTBEAT_MS = 6000020000;
 let runpodStatusTimer = null;
 let runpodPingTimerrunpodHeartbeatTimer = null;
 let runpodLastPhase = 'red';
 function getRunpodLazyUrl() {

     runpodStatusTimer = setTimeout(pollRunpodStatus, RUNPOD_POLL_BUSY_MS);
+}
 function runpodKeepaliverunpodHeartbeat() {
     const url = getRunpodLazyUrl();
     if (!url || runpodLastPhase !== 'green') {
         return;
+    }
 -    // The proxy also rejects wake-on-ping; this phase guard avoids unnecessary
 +    // This only renews the proxy's frontend lease and never starts a pod. The
     // requestsproxy wheneverowns the last read-only statusstable pollupstream sayskeepalive thecadence podso isbrowser down.timer
 +    // throttling cannot directly skew it.
     fetch(`${url}/lazy/ping`, { method: 'POST', signal: AbortSignal.timeout(5000) }).catch(() => { });
+}
 /** (Re)starts the polling/keepalive loops basedstatus onpolling theand configuredfrontend-heartbeat URLloops. */
 function setupRunpodLoops() {
     clearTimeout(runpodStatusTimer);
     clearInterval(runpodPingTimerrunpodHeartbeatTimer);
     ensureRunpodBarDot();
     const barDot = document.getElementById('sd_runpod_bar_dot');
     if (barDot) {

@@ -1714,7 +1715,8 @@ function setupRunpodLoops() {
1714	1715	return;
1715	1716	}
1716	1717	pollRunpodStatus();
1717		- runpodPingTimer = setInterval(runpodKeepalive, RUNPOD_PING_MS);
	1718	+ runpodHeartbeat();
	1719	+ runpodHeartbeatTimer = setInterval(runpodHeartbeat, RUNPOD_HEARTBEAT_MS);
1718	1720	}
1719	1721
1720	1722	function onRunpodLazyUrlInput() {

public/scripts/extensions/stable-diffusion/settings.html+1 -1 Viewed

                     <span data-i18n="Copy ComfyUI URL">Copy ComfyUI URL</span>
                 </div>
             </div>
             <small data-i18n="sd_runpod_small">The pod starts only when you press Warm up. A SillyTavern tab sends a heartbeat every 20 seconds; the proxy keeps an already-ready pod alive; iton its own stable timer and tolerates browser sleep for up to 5 minutes. The pod shuts down 15 minutes after thekeepalives lastand other activity stop. Image requests fail over instead of starting a stopped pod. Red = off, orange = starting/downloading models, green = ready.</small>
             <h5 data-i18n="Model catalog">Model catalog</h5>
             <small data-i18n="sd_runpod_models_small">When the ComfyUI URL points at the proxy, the Model dropdown lists these entries instead of asking the backend. Press Warm up to apply the selected model and download its files; changing the selection or generating an image never starts or provisions a stopped pod. Filename = what &quot;%model%&quot; resolves to; downloads = one &quot;subpath url&quot; per line, including companion files (text encoder, VAE, LoRA).</small>
             <div id="sd_runpod_models_list" class="flex-container flexFlowColumn marginTopBot5"></div>

src/endpoints/keepalive.js+264 -0 Viewed

 +import express from 'express';
 +import https from 'node:https';
 +import fetch from 'node-fetch';
++
 +export const router = express.Router();
++
 +export const FRONTEND_LEASE_MS = 5 * 60 * 1000;
 +const MIN_INTERVAL_MS = 60 * 1000;
 +const MAX_INTERVAL_MS = 24 * 60 * 60 * 1000;
 +const MAX_JOB_ID_LENGTH = 128;
 +const ALLOWED_ENDPOINTS = new Set([
 +    '/api/backends/chat-completions/generate',
 +    '/api/backends/text-completions/generate',
 +    '/api/backends/kobold/generate',
 +    '/api/novelai/generate',
 +    '/api/horde/generate-text',
 +]);
 +const localHttpsAgent = new https.Agent({ rejectUnauthorized: false });
++
 +function advanceDeadline(deadline, interval, now) {
 +    if (deadline > now) {
 +        return deadline;
 +    }
 +    const missedIntervals = Math.floor((now - deadline) / interval) + 1;
 +    return deadline + missedIntervals * interval;
 +}
++
 +/**
 + * Backend-owned scheduler for prompt-cache keepalive requests.
 + */
 +export class KeepaliveScheduler {
 +    #jobs = new Map();
 +    #timer = null;
 +    #now;
 +    #dispatch;
 +    #setTimer;
 +    #clearTimer;
++
 +    constructor({
 +        now = Date.now,
 +        dispatch = dispatchKeepaliveRequest,
 +        setTimer = setTimeout,
 +        clearTimer = clearTimeout,
 +    } = {}) {
 +        this.#now = now;
 +        this.#dispatch = dispatch;
 +        this.#setTimer = setTimer;
 +        this.#clearTimer = clearTimer;
 +    }
++
 +    register(key, job) {
 +        const now = this.#now();
 +        const intervalMs = Math.min(MAX_INTERVAL_MS, Math.max(MIN_INTERVAL_MS, Number(job.intervalMs) || MIN_INTERVAL_MS));
 +        const requestedDelay = Number(job.delayMs);
 +        const delayMs = Number.isFinite(requestedDelay)
 +            ? Math.min(intervalMs, Math.max(0, requestedDelay))
 +            : intervalMs;
 +        this.#jobs.set(key, {
 +            ...job,
 +            intervalMs,
 +            lastSeen: now,
 +            nextRun: now + delayMs,
 +            inFlight: false,
 +        });
 +        this.#schedule();
 +        return this.#jobs.get(key);
 +    }
++
 +    heartbeat(key, activity = false, request = null) {
 +        const job = this.#jobs.get(key);
 +        if (!job) {
 +            return null;
 +        }
 +        const now = this.#now();
 +        if ((now - job.lastSeen) >= FRONTEND_LEASE_MS) {
 +            this.#jobs.delete(key);
 +            this.#schedule();
 +            return null;
 +        }
 +        job.lastSeen = now;
 +        if (activity) {
 +            job.nextRun = now + job.intervalMs;
 +        }
 +        if (request) {
 +            job.request = request;
 +        }
 +        this.#schedule();
 +        return job;
 +    }
++
 +    stop(key) {
 +        const removed = this.#jobs.delete(key);
 +        this.#schedule();
 +        return removed;
 +    }
++
 +    get(key) {
 +        return this.#jobs.get(key) ?? null;
 +    }
++
 +    async tick(now = this.#now()) {
 +        const pending = [];
 +        for (const [key, job] of this.#jobs) {
 +            if ((now - job.lastSeen) >= FRONTEND_LEASE_MS) {
 +                this.#jobs.delete(key);
 +                continue;
 +            }
 +            if (job.nextRun > now) {
 +                continue;
 +            }
++
 +            job.nextRun = advanceDeadline(job.nextRun, job.intervalMs, now);
 +            if (job.inFlight) {
 +                continue;
 +            }
 +            job.inFlight = true;
 +            pending.push(Promise.resolve(this.#dispatch(job)).catch(error => {
 +                console.debug(`[Keepalive] Backend request failed for ${job.id}:`, error);
 +            }).finally(() => {
 +                job.inFlight = false;
 +                this.#schedule();
 +            }));
 +        }
 +        this.#schedule();
 +        await Promise.all(pending);
 +    }
++
 +    #schedule() {
 +        if (this.#timer) {
 +            this.#clearTimer(this.#timer);
 +            this.#timer = null;
 +        }
 +        if (!this.#jobs.size) {
 +            return;
 +        }
++
 +        let nextWake = Infinity;
 +        for (const job of this.#jobs.values()) {
 +            nextWake = Math.min(nextWake, job.nextRun, job.lastSeen + FRONTEND_LEASE_MS);
 +        }
 +        const delay = Math.max(0, nextWake - this.#now());
 +        this.#timer = this.#setTimer(() => {
 +            this.#timer = null;
 +            void this.tick();
 +        }, delay);
 +        this.#timer?.unref?.();
 +    }
 +}
++
 +async function dispatchKeepaliveRequest(job) {
 +    const controller = new AbortController();
 +    const timeout = setTimeout(() => controller.abort(new Error('Keepalive request timed out')), 120000);
 +    timeout.unref?.();
 +    try {
 +        const response = await fetch(`${job.request.origin}${job.endpoint}`, {
 +            method: 'POST',
 +            headers: job.request.headers,
 +            body: JSON.stringify(job.payload),
 +            signal: controller.signal,
 +            agent: job.request.secure ? localHttpsAgent : undefined,
 +        });
 +        await response.arrayBuffer();
 +        if (!response.ok) {
 +            throw new Error(`HTTP ${response.status}`);
 +        }
 +    } finally {
 +        clearTimeout(timeout);
 +    }
 +}
++
 +const scheduler = new KeepaliveScheduler();
++
 +function getJobKey(request, id) {
 +    return `${request.user.profile.handle}:${id}`;
 +}
++
 +function getJobId(request) {
 +    const id = String(request.body?.id ?? '');
 +    if (!id || id.length > MAX_JOB_ID_LENGTH || !/^[a-zA-Z0-9_-]+$/.test(id)) {
 +        throw new Error('Invalid keepalive job id');
 +    }
 +    return id;
 +}
++
 +function getReplayRequest(request) {
 +    const localPort = Number(request.socket.localPort);
 +    if (!Number.isInteger(localPort) || localPort < 1 || localPort > 65535) {
 +        throw new Error('Unable to determine local server port');
 +    }
 +    let localAddress = request.socket.localAddress || '127.0.0.1';
 +    if (localAddress === '::' || localAddress === '0:0:0:0:0:0:0:0') {
 +        localAddress = '::1';
 +    } else if (localAddress === '0.0.0.0') {
 +        localAddress = '127.0.0.1';
 +    } else if (localAddress.startsWith('::ffff:')) {
 +        localAddress = localAddress.slice('::ffff:'.length);
 +    }
 +    const loopbackHost = localAddress.includes(':') ? `[${localAddress}]` : localAddress;
 +    const headers = {
 +        'Content-Type': 'application/json',
 +        'Cookie': request.get('cookie') || '',
 +        'X-CSRF-Token': request.get('x-csrf-token') || '',
 +        'Host': request.get('host') || '',
 +    };
 +    for (const name of ['authorization', 'origin', 'referer', 'user-agent']) {
 +        const value = request.get(name);
 +        if (value) {
 +            headers[name] = value;
 +        }
 +    }
 +    const secure = !!request.socket.encrypted;
 +    return {
 +        origin: `${secure ? 'https' : 'http'}://${loopbackHost}:${localPort}`,
 +        headers,
 +        secure,
 +    };
 +}
++
 +router.post('/register', (request, response) => {
 +    try {
 +        const id = getJobId(request);
 +        const endpoint = String(request.body?.endpoint ?? '');
 +        if (!ALLOWED_ENDPOINTS.has(endpoint)) {
 +            return response.status(400).send({ error: 'Unsupported keepalive endpoint' });
 +        }
 +        if (!request.body?.payload || typeof request.body.payload !== 'object' || Array.isArray(request.body.payload)) {
 +            return response.status(400).send({ error: 'Missing keepalive payload' });
 +        }
 +        const job = scheduler.register(getJobKey(request, id), {
 +            id,
 +            endpoint,
 +            payload: request.body.payload,
 +            intervalMs: request.body.intervalMs,
 +            delayMs: request.body.delayMs,
 +            request: getReplayRequest(request),
 +        });
 +        return response.send({ ok: true, nextRun: job.nextRun, leaseMs: FRONTEND_LEASE_MS });
 +    } catch (error) {
 +        return response.status(400).send({ error: String(error.message || error) });
 +    }
 +});
++
 +router.post('/heartbeat', (request, response) => {
 +    try {
 +        const id = getJobId(request);
 +        const job = scheduler.heartbeat(getJobKey(request, id), !!request.body?.activity, getReplayRequest(request));
 +        if (!job) {
 +            return response.sendStatus(404);
 +        }
 +        return response.send({ ok: true, nextRun: job.nextRun, leaseMs: FRONTEND_LEASE_MS });
 +    } catch (error) {
 +        return response.status(400).send({ error: String(error.message || error) });
 +    }
 +});
++
 +router.post('/stop', (request, response) => {
 +    try {
 +        const id = getJobId(request);
 +        scheduler.stop(getJobKey(request, id));
 +        return response.send({ ok: true });
 +    } catch (error) {
 +        return response.status(400).send({ error: String(error.message || error) });
 +    }
 +});

src/server-startup.js+2 -0 Viewed

 import { router as backupsRouter } from './endpoints/backups.js';
 import { router as imageMetadataRouter } from './endpoints/image-metadata.js';
 import { router as volcengineRouter } from './endpoints/volcengine.js';
 +import { router as keepaliveRouter } from './endpoints/keepalive.js';
 /**
  * @typedef {object} ServerStartupResult

     app.use('/api/data-maid', dataMaidRouter);
     app.use('/api/backups', backupsRouter);
     app.use('/api/image-metadata', imageMetadataRouter);
 +    app.use('/api/keepalive', keepaliveRouter);
+}
 /**

tests/keepalive-scheduler.test.js+80 -0 Viewed

 +import { describe, expect, jest, test } from '@jest/globals';
 +import { FRONTEND_LEASE_MS, KeepaliveScheduler } from '../src/endpoints/keepalive.js';
++
 +function createScheduler() {
 +    let now = 0;
 +    const dispatch = jest.fn().mockResolvedValue(undefined);
 +    const scheduler = new KeepaliveScheduler({
 +        now: () => now,
 +        dispatch,
 +        setTimer: () => ({ unref() { } }),
 +        clearTimer: () => { },
 +    });
 +    return { scheduler, dispatch, setNow: value => { now = value; } };
 +}
++
 +describe('KeepaliveScheduler', () => {
 +    test('heartbeats renew the lease without moving the backend deadline', () => {
 +        const { scheduler, setNow } = createScheduler();
 +        scheduler.register('user:tab', { id: 'tab', intervalMs: 60000, delayMs: 60000 });
++
 +        setNow(20000);
 +        scheduler.heartbeat('user:tab');
++
 +        expect(scheduler.get('user:tab').lastSeen).toBe(20000);
 +        expect(scheduler.get('user:tab').nextRun).toBe(60000);
 +    });
++
 +    test('a heartbeat cannot revive an already expired lease', () => {
 +        const { scheduler, setNow } = createScheduler();
 +        scheduler.register('user:tab', { id: 'tab', intervalMs: 60000, delayMs: 60000 });
++
 +        setNow(FRONTEND_LEASE_MS);
++
 +        expect(scheduler.heartbeat('user:tab')).toBeNull();
 +        expect(scheduler.get('user:tab')).toBeNull();
 +    });
++
 +    test('accepts an immediate first deadline', () => {
 +        const { scheduler } = createScheduler();
 +        scheduler.register('user:tab', { id: 'tab', intervalMs: 60000, delayMs: 0 });
++
 +        expect(scheduler.get('user:tab').nextRun).toBe(0);
 +    });
++
 +    test('dispatches on the backend deadline and stays aligned after a late tick', async () => {
 +        const { scheduler, dispatch, setNow } = createScheduler();
 +        scheduler.register('user:tab', { id: 'tab', intervalMs: 60000, delayMs: 60000 });
++
 +        setNow(60000);
 +        await scheduler.tick();
 +        expect(dispatch).toHaveBeenCalledTimes(1);
 +        expect(scheduler.get('user:tab').nextRun).toBe(120000);
++
 +        setNow(190000);
 +        await scheduler.tick();
 +        expect(dispatch).toHaveBeenCalledTimes(2);
 +        expect(scheduler.get('user:tab').nextRun).toBe(240000);
 +    });
++
 +    test('activity resets the next keepalive deadline', () => {
 +        const { scheduler, setNow } = createScheduler();
 +        scheduler.register('user:tab', { id: 'tab', intervalMs: 60000, delayMs: 60000 });
++
 +        setNow(50000);
 +        scheduler.heartbeat('user:tab', true);
++
 +        expect(scheduler.get('user:tab').nextRun).toBe(110000);
 +    });
++
 +    test('removes a job after five minutes without a frontend heartbeat', async () => {
 +        const { scheduler, dispatch, setNow } = createScheduler();
 +        scheduler.register('user:tab', { id: 'tab', intervalMs: 60000, delayMs: 60000 });
++
 +        setNow(FRONTEND_LEASE_MS);
 +        await scheduler.tick();
++
 +        expect(scheduler.get('user:tab')).toBeNull();
 +        expect(dispatch).not.toHaveBeenCalled();
 +    });
 +});