studio: dispatch resilience — kill the intermittent "failed validation" on fix

The /prompt intercept sends every render to the org's GPU worker, but a
transient fleet-check fault (/v1/machines 502 / TLS reset / timeout) made
dispatch_if_worker return False, so the render fell through to LOCAL
validation on the model-less cloud pod → the cryptic "Prompt outputs failed
validation" (unet_name '…' not in []). That's why Fix failed only *sometimes*.

- _fleet_machines: retry 3x with backoff so a transient hiccup isn't read as
  "no GPU" and bounced onto the model-less pod.
- has_local_models() + intercept: on a model-less coordinator pod a failed
  dispatch now returns a clear retryable 503 "GPU render worker is momentarily
  unavailable — please retry", never the doomed local validation. A box WITH
  models still renders locally (dev). v0.17.5

middleware/gpu_dispatch.py

@@ -9,6 +9,7 @@ shared keys. Falls back to the in-pod queue when there is no live worker.
 import base64
 import json
 import os
+import time
 import urllib.request
 
 import folder_paths
@@ -92,11 +93,22 @@ def _node_label(m: dict) -> str:
 
 def _fleet_machines(tok: str) -> list:
     """The caller-org's machines from the cloud fleet. The token scopes the fleet to
-    the caller's org, so this is inherently tenant-isolated."""
-    req = urllib.request.Request(f"{CLOUD}/v1/machines", headers={"Authorization": tok})
-    d = json.loads(urllib.request.urlopen(req, timeout=10).read())
-    ms = d.get("machines", [])
-    return ms if isinstance(ms, list) else []
+    the caller's org, so this is inherently tenant-isolated. Retried a few times so a
+    transient edge fault (502, TLS reset, timeout) does NOT read as "no GPU" and
+    bounce the render onto the model-less pod — the cause of intermittent
+    "Prompt outputs failed validation"."""
+    last = None
+    for attempt in range(3):
+        try:
+            req = urllib.request.Request(f"{CLOUD}/v1/machines", headers={"Authorization": tok})
+            d = json.loads(urllib.request.urlopen(req, timeout=10).read())
+            ms = d.get("machines", [])
+            return ms if isinstance(ms, list) else []
+        except Exception as e:  # any transient fault (HTTP/TLS/timeout/non-JSON) is retryable
+            last = e
+            if attempt < 2:
+                time.sleep(0.4 * (attempt + 1))
+    raise last
 
 
 def _online_gpu_nodes(tok: str) -> list:
@@ -108,6 +120,21 @@ def _has_online_gpu(tok: str) -> bool:
     return bool(_online_gpu_nodes(tok))
 
 
+def has_local_models() -> bool:
+    """True if THIS instance actually has diffusion models to render with. A cloud
+    coordinator pod is model-less — every render is dispatched to the org's GPU
+    worker; a box WITH a GPU has models and can render locally. Lets the /prompt
+    intercept turn a failed dispatch on a model-less pod into a clear "worker
+    unavailable, retry" instead of a cryptic "not in []" validation error."""
+    try:
+        for kind in ("diffusion_models", "checkpoints"):
+            if folder_paths.get_filename_list(kind):
+                return True
+    except Exception:
+        pass
+    return False
+
+
 def dispatch_if_worker(request, org_id: str, prompt_id: str, prompt: dict) -> bool:
     """If the caller's org has an online GPU node in the fleet, enqueue a
     studio.render job to gpu-jobs and return True. Return False to run in-pod.

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "hanzo-studio"
-version = "0.17.4"
+version = "0.17.5"
 readme = "README.md"
 license = { file = "LICENSE" }
 requires-python = ">=3.10"

server.py

@@ -1100,11 +1100,21 @@ class PromptServer():
                 # every checkpoint-referencing graph ("not in []").
                 if not args.worker_mode:
                     try:
-                        from middleware.gpu_dispatch import dispatch_if_worker
+                        from middleware.gpu_dispatch import dispatch_if_worker, has_local_models
                     except ImportError:  # package-style checkout
-                        from .middleware.gpu_dispatch import dispatch_if_worker
+                        from .middleware.gpu_dispatch import dispatch_if_worker, has_local_models
                     if dispatch_if_worker(request, org_id, prompt_id, prompt):
                         return web.json_response({"prompt_id": prompt_id, "number": number, "node_errors": {}})
+                    # Dispatch didn't happen. On a model-less coordinator pod, local
+                    # validation is GUARANTEED to fail with a cryptic "not in []" —
+                    # the render belongs on the org's GPU worker, momentarily
+                    # unreachable. Return a clear, retryable error instead of the
+                    # doomed validation. A box WITH models falls through and renders.
+                    if not has_local_models():
+                        return web.json_response({"error": {
+                            "type": "gpu_worker_unavailable",
+                            "message": "GPU render worker is momentarily unavailable — please retry.",
+                            "details": "", "extra_info": {}}, "node_errors": {}}, status=503)
 
                 valid = await execution.validate_prompt(prompt_id, prompt, partial_execution_targets)
                 extra_data = {}