runpod-lazy: no churn on unknown pod model; adopt existing pod on proxy restart

012e9fea1bfc457230dd4650ba1a3a0aed0e8107

permissionBRICK <40219477+permissionBRICK@users.noreply.github.com>

homelab/runpod-lazy-proxy.py+19 -3 Viewed

         pod_id, have, gpu = (state['pod_id'], state['model'], state['gpu'])
         if not pod_id:
             pod_id, have, gpu = find_pod()
 -        if pod_id and value and have != value:
 +        # Only recreate when the pod's model is KNOWN and different. An unknown
 +        # model (e.g. env not reported yet) must not churn pods.
 +        if pod_id and value and have and have != value:
             log(f'pod has model={have}, need {value} - recreating')
             terminate(pod_id)
             pod_id, have, gpu = None, None, None

                 catalog = {'models': payload.get('models', []), 'active': payload.get('active')}
                 save_catalog(catalog)
                 log(f"catalog updated: {len(catalog['models'])} models, active={catalog['active']}")
                 # Re-provision proactively when the active model changed under a live pod.
 -                if state['pod_id'] and catalog['active'] and state['model'] != catalog['active']:
 +                # (known-model pods only - never churn a pod of unknown provenance).
 +                if state['pod_id'] and catalog['active'] and state['model'] and state['model'] != catalog['active']:
                     log('active model changed - re-provisioning pod')
                     threading.Thread(target=self._safe_ensure, args=(catalog['active'],), daemon=True).start()
                 return self._reply(200, status_body())

 def main():
     threading.Thread(target=idle_reaper, daemon=True).start()
 +    # Adopt a pre-existing pod (e.g. after a proxy restart mid-provisioning) so
 +    # its readiness wait - which also feeds the idle timer - keeps running.
 +    pod_id, have, gpu = find_pod()
 +    if pod_id:
 +        state.update({'pod_id': pod_id, 'model': have, 'gpu': gpu, 'phase': 'orange', 'last': time.monotonic()})
 +        log(f'adopting existing pod {pod_id} (model={have})')
++
 +        def _adopt():
 +            try:
 +                ensure_pod(have)
 +            except Exception as err:
 +                log('adoption wait failed:', err)
 +        threading.Thread(target=_adopt, daemon=True).start()
     server = ThreadingHTTPServer(('0.0.0.0', LISTEN_PORT), Proxy)
     log(f'runpod-lazy v3 (catalog) on :{LISTEN_PORT} (pod "{POD_NAME}", DCs {DATACENTERS or "any"}, idle {IDLE_SECONDS}s)')
     server.serve_forever()