homelab: runpod-lazy v5.2 manager-first boot, US-preference with global fallback
| @@ -150,7 +150,7 @@ def find_pod(): | ||
| 150 | 150 | return None, None, None |
| 151 | 151 | |
| 152 | 152 | |
| 153 | 153 | def create_pod(values, datacenters=None): |
| 154 | 154 | files = [] |
| 155 | 155 | for value in values or []: |
| 156 | 156 | entry = catalog_entry(value) |
| @@ -170,13 +170,14 @@ def create_pod(values): | ||
| 170 | 170 | 'containerDiskInGb': 80, |
| 171 | 171 | 'ports': ['8188/http', '8189/http'], |
| 172 | 172 | 'env': env, |
| 173 | - # The model-manager subshell tolerates old images without the script. | |
| 173 | + # Manager starts FIRST so boot download progress is visible from | |
| 174 | - 'dockerStartCmd': ['bash', '-c', f'python3 /boot-models.py && (python3 /model-manager.py &) && cd /comfyui && exec python main.py {COMFY_ARGS}'], | |
| 174 | + # outside; the subshell tolerates old images without the script. | |
| 175 | + 'dockerStartCmd': ['bash', '-c', f'(python3 /model-manager.py &) ; python3 /boot-models.py && cd /comfyui && exec python main.py {COMFY_ARGS}'], | |
| 175 | 176 | } |
| 176 | 177 | if DATACENTERSdatacenters: |
| 177 | 178 | body['dataCenterIds'] = DATACENTERSdatacenters |
| 178 | 179 | pod = api('POST', '/pods', body) |
| 179 | 180 | log(f"pod created: {pod['id']} models={values_key(values) or 'legacy-all'} {pod.get('machine', {}).get('gpuTypeId')} ${pod.get('costPerHr')}/hr dc={pod.get('machine', {}).get('dataCenterId') or (','.join(datacenters) if datacenters else 'any')}") |
| 180 | 181 | return pod['id'], pod.get('machine', {}).get('gpuTypeId') |
| 181 | 182 | |
| 182 | 183 | |
| @@ -224,12 +225,16 @@ def _create_pod_with_retries(values): | ||
| 224 | 225 | state['phase'] = 'orange' |
| 225 | 226 | state['since'] = time.time() |
| 226 | 227 | last_err = None |
| 227 | - for attempt in range(3): | |
| 228 | + # Prefer the DATACENTERS list (e.g. US - fast HuggingFace peering for model | |
| 229 | + # downloads); if it has no capacity, fall back to any datacenter. | |
| 230 | + plans = [DATACENTERS, DATACENTERS, None] if DATACENTERS else [None, None, None] | |
| 231 | + for attempt, datacenters in enumerate(plans): | |
| 228 | 232 | try: |
| 229 | 233 | return create_pod(values, datacenters) |
| 230 | 234 | except urllib.error.HTTPError as err: |
| 231 | 235 | last_err = err.read().decode()[:200] |
| 232 | - log(f'create_pod attempt {attempt + 1} failed:', last_err) | |
| 236 | + scope = 'preferred DCs' if datacenters else 'global' | |
| 237 | + log(f'create_pod attempt {attempt + 1} ({scope}) failed:', last_err) | |
| 233 | 238 | time.sleep(5) |
| 234 | 239 | state['phase'] = 'red' |
| 235 | 240 | raise RuntimeError(f'could not create pod: {last_err}') |