homelab: runpod-lazy v5.2 manager-first boot, US-preference with global fallback
| @@ -150,7 +150,7 @@ def find_pod(): | |||
| 150 | return None, None, None | 150 | return None, None, None |
| 151 | 151 | ||
| 152 | 152 | ||
| 153 | def create_pod(values): | 153 | def create_pod(values, datacenters=None): |
| 154 | files = [] | 154 | files = [] |
| 155 | for value in values or []: | 155 | for value in values or []: |
| 156 | entry = catalog_entry(value) | 156 | entry = catalog_entry(value) |
| @@ -170,13 +170,14 @@ def create_pod(values): | |||
| 170 | 'containerDiskInGb': 80, | 170 | 'containerDiskInGb': 80, |
| 171 | 'ports': ['8188/http', '8189/http'], | 171 | 'ports': ['8188/http', '8189/http'], |
| 172 | 'env': env, | 172 | 'env': env, |
| 173 | # The model-manager subshell tolerates old images without the script. | 173 | # Manager starts FIRST so boot download progress is visible from |
| 174 | 'dockerStartCmd': ['bash', '-c', f'python3 /boot-models.py && (python3 /model-manager.py &) && cd /comfyui && exec python main.py {COMFY_ARGS}'], | 174 | # outside; the subshell tolerates old images without the script. |
| 175 | 'dockerStartCmd': ['bash', '-c', f'(python3 /model-manager.py &) ; python3 /boot-models.py && cd /comfyui && exec python main.py {COMFY_ARGS}'], | ||
| 175 | } | 176 | } |
| 176 | if DATACENTERS: | 177 | if datacenters: |
| 177 | body['dataCenterIds'] = DATACENTERS | 178 | body['dataCenterIds'] = datacenters |
| 178 | pod = api('POST', '/pods', body) | 179 | pod = api('POST', '/pods', body) |
| 179 | log(f"pod created: {pod['id']} models={values_key(values) or 'legacy-all'} {pod.get('machine', {}).get('gpuTypeId')} ${pod.get('costPerHr')}/hr") | 180 | log(f"pod created: {pod['id']} models={values_key(values) or 'legacy-all'} {pod.get('machine', {}).get('gpuTypeId')} ${pod.get('costPerHr')}/hr dc={pod.get('machine', {}).get('dataCenterId') or (','.join(datacenters) if datacenters else 'any')}") |
| 180 | return pod['id'], pod.get('machine', {}).get('gpuTypeId') | 181 | return pod['id'], pod.get('machine', {}).get('gpuTypeId') |
| 181 | 182 | ||
| 182 | 183 | ||
| @@ -224,12 +225,16 @@ def _create_pod_with_retries(values): | |||
| 224 | state['phase'] = 'orange' | 225 | state['phase'] = 'orange' |
| 225 | state['since'] = time.time() | 226 | state['since'] = time.time() |
| 226 | last_err = None | 227 | last_err = None |
| 227 | for attempt in range(3): | 228 | # Prefer the DATACENTERS list (e.g. US - fast HuggingFace peering for model |
| 229 | # downloads); if it has no capacity, fall back to any datacenter. | ||
| 230 | plans = [DATACENTERS, DATACENTERS, None] if DATACENTERS else [None, None, None] | ||
| 231 | for attempt, datacenters in enumerate(plans): | ||
| 228 | try: | 232 | try: |
| 229 | return create_pod(values) | 233 | return create_pod(values, datacenters) |
| 230 | except urllib.error.HTTPError as err: | 234 | except urllib.error.HTTPError as err: |
| 231 | last_err = err.read().decode()[:200] | 235 | last_err = err.read().decode()[:200] |
| 232 | log(f'create_pod attempt {attempt + 1} failed:', last_err) | 236 | scope = 'preferred DCs' if datacenters else 'global' |
| 237 | log(f'create_pod attempt {attempt + 1} ({scope}) failed:', last_err) | ||
| 233 | time.sleep(5) | 238 | time.sleep(5) |
| 234 | state['phase'] = 'red' | 239 | state['phase'] = 'red' |
| 235 | raise RuntimeError(f'could not create pod: {last_err}') | 240 | raise RuntimeError(f'could not create pod: {last_err}') |