runpod-lazy: no churn on unknown pod model; adopt existing pod on proxy restart

012e9fea1bfc457230dd4650ba1a3a0aed0e8107

permissionBRICK <40219477+permissionBRICK@users.noreply.github.com>

homelab/runpod-lazy-proxy.py+19 -3 Viewed

         pod_id, have, gpu = (state['pod_id'], state['model'], state['gpu'])
         if not pod_id:
             pod_id, have, gpu = find_pod()
-        if pod_id and value and have != value:
+        # Only recreate when the pod's model is KNOWN and different. An unknown
+        # model (e.g. env not reported yet) must not churn pods.
+        if pod_id and value and have and have != value:
             log(f'pod has model={have}, need {value} - recreating')
             terminate(pod_id)
             pod_id, have, gpu = None, None, None

                 catalog = {'models': payload.get('models', []), 'active': payload.get('active')}
                 save_catalog(catalog)
                 log(f"catalog updated: {len(catalog['models'])} models, active={catalog['active']}")
-                # Re-provision proactively when the active model changed under a live pod.
+                # Re-provision proactively when the active model changed under a live pod
-                if state['pod_id'] and catalog['active'] and state['model'] != catalog['active']:
+                # (known-model pods only - never churn a pod of unknown provenance).
+                if state['pod_id'] and catalog['active'] and state['model'] and state['model'] != catalog['active']:
                     log('active model changed - re-provisioning pod')
                     threading.Thread(target=self._safe_ensure, args=(catalog['active'],), daemon=True).start()
                 return self._reply(200, status_body())

 def main():
     threading.Thread(target=idle_reaper, daemon=True).start()
+    # Adopt a pre-existing pod (e.g. after a proxy restart mid-provisioning) so
+    # its readiness wait - which also feeds the idle timer - keeps running.
+    pod_id, have, gpu = find_pod()
+    if pod_id:
+        state.update({'pod_id': pod_id, 'model': have, 'gpu': gpu, 'phase': 'orange', 'last': time.monotonic()})
+        log(f'adopting existing pod {pod_id} (model={have})')
+        def _adopt():
+            try:
+                ensure_pod(have)
+            except Exception as err:
+                log('adoption wait failed:', err)
+        threading.Thread(target=_adopt, daemon=True).start()
     server = ThreadingHTTPServer(('0.0.0.0', LISTEN_PORT), Proxy)
     log(f'runpod-lazy v3 (catalog) on :{LISTEN_PORT} (pod "{POD_NAME}", DCs {DATACENTERS or "any"}, idle {IDLE_SECONDS}s)')
     server.serve_forever()