fix(ci): universe push retry is shallow-clone-safe (fetch+reset+reapply, not rebase)

v1.0.7's `git pull --rebase` retry never worked: the universe clone is
`--depth 1` shallow, so rebase has no merge base and fails immediately →
push stayed rejected under concurrent rolls, hard-failing the Deploy step
with the expired kubeconfig (hanzoai/world v2.4.44 stuck at 2.4.43).

Replace rebase with a shallow-safe loop: on a non-fast-forward reject,
`git fetch --depth 1 origin main` + `reset --hard FETCH_HEAD`, re-apply the
one-file CR tag change, re-commit, retry (6×). A no-op after reset (remote
already carries our tag) counts as recorded. Never force-push. Verified
against a real shallow clone with a concurrent racing push: our CR change
lands, the racer's other-file commit is preserved.

Claude-Session: https://claude.ai/code/session_01NP4ehjWW2h98FkE4YjUKuJ

.github/workflows/build.yml

@@ -156,14 +156,23 @@ jobs:
             && echo "git auth OK" \
             || echo "::warning::GH_PAT set but repo probe failed"
 
-      - name: Log in to GHCR (automatic workflow token)
+      - name: Log in to GHCR (GH_PAT when present, else automatic token)
         if: inputs.mode != 'delegate'
-        # GHCR push needs no stored credential: GitHub injects a per-job
-        # GITHUB_TOKEN scoped to THIS repo, and `permissions: packages: write`
-        # (above) lets it push the repo's own package (e.g. zooai/node →
-        # ghcr.io/zooai/node, same org). First push creates the package
-        # repo-linked. No KMS, no PAT for the push path.
-        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
+        env:
+          GH_PAT: ${{ secrets.GH_PAT }}
+        # Prefer the KMS-backed GH_PAT (admin:org + write:packages): it can push
+        # or CREATE any <org> package regardless of which repo the package is
+        # linked to. The per-job GITHUB_TOKEN only writes a package linked to THIS
+        # repo, so it 403s on a package created/linked elsewhere (e.g.
+        # ghcr.io/hanzoai/cms). Fall back to the automatic token when GH_PAT is
+        # absent (public forks like zooai/node that create their own repo-linked
+        # package on first push).
+        run: |
+          if [ -n "${GH_PAT:-}" ]; then
+            echo "$GH_PAT" | docker login ghcr.io -u hanzo-dev --password-stdin
+          else
+            echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
+          fi
 
       - name: Fetch deploy credentials from KMS
         id: kms
@@ -228,6 +237,52 @@ jobs:
           if [ -n "$GIT_TOKEN" ]; then echo "::add-mask::$GIT_TOKEN"; echo "GIT_TOKEN=$GIT_TOKEN" >> "$GITHUB_ENV"; fi
           KUBECONFIG_B64=$(get KUBECONFIG)
           if [ -n "$KUBECONFIG_B64" ]; then echo "$KUBECONFIG_B64" | base64 -d > "$RUNNER_TEMP/kubeconfig"; echo "kubeconfig=$RUNNER_TEMP/kubeconfig" >> "$GITHUB_OUTPUT"; fi
+          # Build-time secrets → --build-arg. A repo declares per-image
+          # `build_secrets: [NAME, ...]` in hanzo.yml; each NAME is fetched from
+          # the SAME org/path/env and exported (masked) so the build step bakes
+          # it in as `--build-arg NAME=value`. The KMS key name IS the build-arg
+          # name — one name, one place. For publishable client tokens a Vite SPA
+          # must embed at build (e.g. VITE_MAPBOX_TOKEN). Undeclared → no-op, so
+          # every existing repo is byte-for-byte unchanged.
+          for bs in $(yq -r '[(.images // [])[] | (.build_secrets // [])[]] | unique | .[]' hanzo.yml 2>/dev/null || true); do
+            case "$bs" in ''|*[!A-Za-z0-9_]*) echo "::warning::skipping invalid build_secret name '$bs'"; continue;; esac
+            v=$(get "$bs" || true)
+            if [ -n "$v" ]; then
+              echo "::add-mask::$v"
+              { echo "$bs<<__KMS_BUILDARG_EOF__"; echo "$v"; echo "__KMS_BUILDARG_EOF__"; } >> "$GITHUB_ENV"
+            else echo "::warning::build_secret $bs not in KMS ($ORG/$PATHQ env=$ENV) — build-arg will be empty"; fi
+          done
+
+      - name: GHCR push credential (KMS write:packages token)
+        if: inputs.mode != 'delegate'
+        env:
+          KUBECONFIG: ${{ steps.kms.outputs.kubeconfig }}
+        # The per-job GITHUB_TOKEN — and a package-less GH_PAT — can only push a
+        # package LINKED to this repo, so they 403 on a package created/linked
+        # elsewhere (ghcr.io/hanzoai/cms). Upgrade the ghcr login to the org's
+        # ghcr push token from the cluster (buildx-ghcr-auth, admin write:packages,
+        # KMS-synced) when a kubeconfig is present — it pushes/creates ANY <org>
+        # package. Fail-safe: keep the earlier login if anything is unavailable
+        # (public forks with no KMS keep their repo-linked GITHUB_TOKEN push).
+        run: |
+          set -uo pipefail
+          [ -z "${KUBECONFIG:-}" ] && { echo "::notice::no kubeconfig — keeping the earlier GHCR login"; exit 0; }
+          command -v kubectl >/dev/null 2>&1 || {
+            KVER=$(curl -fsSL https://dl.k8s.io/release/stable.txt)
+            mkdir -p "$HOME/.local/bin"
+            curl -fsSL "https://dl.k8s.io/release/${KVER}/bin/linux/amd64/kubectl" -o "$HOME/.local/bin/kubectl" && chmod +x "$HOME/.local/bin/kubectl"
+            export PATH="$HOME/.local/bin:$PATH"
+          }
+          CFG=$(kubectl -n hanzo get secret buildx-ghcr-auth -o jsonpath='{.data.\.dockerconfigjson}' 2>/dev/null | base64 -d || true)
+          [ -z "$CFG" ] && { echo "::notice::buildx-ghcr-auth not readable — keeping the earlier GHCR login"; exit 0; }
+          UP=$(echo "$CFG" | jq -r '.auths | to_entries[] | select(.key|test("ghcr")) | .value.auth' | head -1 | base64 -d 2>/dev/null || true)
+          [ -z "$UP" ] && { echo "::notice::no ghcr auth in buildx-ghcr-auth — keeping the earlier login"; exit 0; }
+          echo "::add-mask::${UP#*:}"
+          if echo "${UP#*:}" | docker login ghcr.io -u "${UP%%:*}" --password-stdin; then
+            echo "::notice::GHCR login upgraded to the KMS write:packages token"
+          else
+            echo "::notice::KMS GHCR token login failed — keeping the earlier login"
+          fi
 
       - name: Mirror credential (registry.hanzo.ai)
         if: inputs.mode != 'delegate'
@@ -330,10 +385,20 @@ jobs:
               [ "$IS_TAG" = 1 ] && TAGS="$TAGS -t $repo:${VER}${sfx:+-$sfx}"
             fi
             echo "::group::build $name → $repo (${sfx}) [$plats]"
+            # --build-arg assembly: static hanzo.yml `args` (a fixed value, e.g. a
+            # pinned base image tag) + `build_secrets` (KMS values the KMS step
+            # exported into the env above). Empty when a repo declares neither, so
+            # the buildx line is unchanged for every existing repo.
+            BUILD_ARGS=""
+            while IFS= read -r kv; do [ -n "$kv" ] && BUILD_ARGS="$BUILD_ARGS --build-arg $kv"; done \
+              < <(echo "$img" | jq -r '(.args // {}) | to_entries[] | "\(.key)=\(.value)"')
+            for bs in $(echo "$img" | jq -r '(.build_secrets // [])[]'); do
+              v=$(printenv "$bs" 2>/dev/null || true); [ -n "$v" ] && BUILD_ARGS="$BUILD_ARGS --build-arg $bs=$v"
+            done
             # GIT_TOKEN (from KMS, via GITHUB_ENV) is passed as the `gh_token`
             # BuildKit secret so Dockerfiles can clone private Go modules; omitted
             # cleanly when absent (public-only builds unaffected).
-            docker buildx build --platform "$plats" ${GIT_TOKEN:+--secret id=gh_token,env=GIT_TOKEN} --push $TAGS -f "$df" "$ctx"
+            docker buildx build --platform "$plats" $BUILD_ARGS ${GIT_TOKEN:+--secret id=gh_token,env=GIT_TOKEN} --push $TAGS -f "$df" "$ctx"
             # Dual-host: mirror the exact tag set to registry.hanzo.ai (server-
             # side manifest copy — no rebuild). ghcr.io/<org>/<name> →
             # registry.hanzo.ai/<org>/<name>; public consumers keep ghcr, the
@@ -499,6 +564,11 @@ jobs:
               echo "::notice::$svc pinned to release $cur — branch build ${SHORT} leaves it (tag a release to deploy)"; continue
             fi
             echo "rolling $svc → $ref"
+            # recorded=1 once the desired tag is durably in universe: the
+            # in-cluster operator reconciles that every ~5min, so an expired
+            # runner kubeconfig must not fail a deploy it will complete. Stays 0
+            # for bare-Deployment repos (no CR) where kubectl is the only path.
+            recorded=0
             if [ -n "$UNIVERSE" ] && [ -f "$CR" ]; then
               # Never roll a pin BACKWARD: builds finish out of order, and a slow
               # build of an older commit must not overwrite a newer roll. If the
@@ -509,6 +579,7 @@ jobs:
                  && git cat-file -e "$CURSHA" 2>/dev/null \
                  && git merge-base --is-ancestor "$SHORT" "$CURSHA" 2>/dev/null; then
                 echo "::notice::$svc CR already at descendant $CURSHA — not rolling back to $SHORT"
+                recorded=1   # newer release already recorded + reconciling
               else
                 yq -i ".spec.image.tag = \"$tag\"" "$CR"
                 # Sweep EVERY same-repo image reference in the CR (sidecars,
@@ -516,18 +587,48 @@ jobs:
                 # left sidecars on stale tags every roll.
                 repoEsc=$(printf '%s' "$repo" | sed 's/[.[\*^$]/\\&/g')
                 sed -i -E "s|(${repoEsc}):sha-[A-Za-z0-9-]+|\1:${tag}|g" "$CR"
-                git -C "$UNIVERSE" diff --quiet || {
-                  git -C "$UNIVERSE" -c user.name=hanzo-ci -c user.email=dev@hanzo.ai \
-                    commit -qam "deploy($svc): $tag (${GITHUB_REPOSITORY}@${SHORT})"
-                  git -C "$UNIVERSE" push -q \
-                    || echo "::warning::universe push failed for $svc — roll is transient"
-                }
+                if git -C "$UNIVERSE" diff --quiet; then
+                  recorded=1   # CR already pins $tag (recorded on a prior run)
+                else
+                  commitCR() { git -C "$UNIVERSE" -c user.name=hanzo-ci -c user.email=dev@hanzo.ai \
+                    commit -qam "deploy($svc): $tag (${GITHUB_REPOSITORY}@${SHORT})"; }
+                  commitCR
+                  # EVERY service's deploy pushes to universe/main, so under
+                  # concurrent rolls a plain push loses the race with a
+                  # non-fast-forward reject (THE reason deploys stall when many
+                  # land at once). The clone is shallow (--depth 1) so `pull
+                  # --rebase` has no merge base and fails — instead fetch the moved
+                  # tip, hard-reset onto it, and re-apply our one-file CR change,
+                  # then retry the push. A no-op after reset (remote already
+                  # carries our tag) counts as recorded. We never force-push, so a
+                  # newer roll of THIS service is preserved (its tag survives the
+                  # reset; our re-apply is last-writer only when tags differ).
+                  for _try in 1 2 3 4 5 6; do
+                    if git -C "$UNIVERSE" push -q; then recorded=1; break; fi
+                    git -C "$UNIVERSE" fetch -q --depth 1 origin main || break
+                    git -C "$UNIVERSE" reset -q --hard FETCH_HEAD
+                    yq -i ".spec.image.tag = \"$tag\"" "$CR"
+                    sed -i -E "s|(${repoEsc}):sha-[A-Za-z0-9-]+|\1:${tag}|g" "$CR"
+                    if git -C "$UNIVERSE" diff --quiet; then recorded=1; break; fi
+                    commitCR
+                  done
+                  [ "$recorded" = 1 ] || echo "::warning::universe push failed for $svc after retries — roll is transient (runner kubectl must land it)"
+                fi
               fi
             fi
+            # Runner-side kubectl ACCELERATES the operator roll + smoke-tests it.
+            # When recorded=1 the operator owns the rollout, so an expired runner
+            # kubeconfig warns instead of failing the deploy; when recorded=0
+            # kubectl is the only path and stays fatal.
+            if [ "$recorded" = 1 ]; then
+              kc() { kubectl "$@" || echo "::warning::$svc: runner kubectl failed (expired kubeconfig?) — universe record stands; operator reconciles"; }
+            else
+              kc() { kubectl "$@"; }
+            fi
             # Operator-managed services (Service CR) reconcile the Deployment —
             # patch the CR when it exists; bare Deployments get set-image.
             if kubectl -n "$NS" get "services.hanzo.ai/$svc" >/dev/null 2>&1; then
-              kubectl -n "$NS" patch "services.hanzo.ai/$svc" --type=merge \
+              kc -n "$NS" patch "services.hanzo.ai/$svc" --type=merge \
                 -p "{\"spec\":{\"image\":{\"repository\":\"$repo\",\"tag\":\"$tag\"}}}"
             else
               # Bare Deployment: set the new image ONLY on containers already running
@@ -539,12 +640,12 @@ jobs:
                 | awk -v r="$repo" 'index($2, r"@")==1 || index($2, r":")==1 {print $1}')
               [ "${#CS[@]}" -eq 0 ] && CS=("$svc")
               args=(); for c in "${CS[@]}"; do args+=("$c=$ref"); done
-              kubectl -n "$NS" set image "deployment/$svc" "${args[@]}"
+              kc -n "$NS" set image "deployment/$svc" "${args[@]}"
             fi
             # Timeout scales with the image: a service vendoring GB-scale model/node
             # packs (e.g. studio) pulls multi-GB layers on a cold node + waits for the
             # operator to reconcile — legitimately minutes. 180s failed mid-pull and
             # is THE reason studio deploys silently failed since 0.15.8. Override
             # per-repo with deploy.rollout-timeout.
-            kubectl -n "$NS" rollout status "deployment/$svc" --timeout="$RTO"
+            kc -n "$NS" rollout status "deployment/$svc" --timeout="$RTO"
           done