Compare commits
25
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 hanzo-dev
|
7077cec764 | ||
|
hanzo-dev
|
1aae2b65a3 | ||
|
Hanzo AI
|
93eafe4f55 | ||
|
Hanzo AI
|
4fcc2e02ce | ||
|
Hanzo AI
|
2866795e2d | ||
|
zeekay
|
3083524803 | ||
|
zeekay
|
313180e201 | ||
|
zeekay
|
76cf840edb | ||
|
zeekay
|
28f1664092 | ||
|
zeekay
|
f3acdc8cf7 | ||
|
zeekay
|
02370735a0 | ||
|
zeekay
|
39553ee364 | ||
|
zeekay
|
5ee1b857de | ||
|
zeekay
|
e628b788c4 | ||
|
hanzo-dev
|
56e83218ca | ||
|
hanzo-dev
|
b5df8baca7 | ||
|
hanzo-dev
|
3668bb9ebd | ||
|
hanzo-dev
|
86428e2f8a | ||
|
hanzo-dev
|
9f4fd9a959 | ||
|
|
f7426add1d | ||
|
|
0770a28962 | ||
|
zeekayandClaude Opus 4.8
|
5106bdeada | ||
|
zeekayandClaude Opus 4.8
|
0339fc7dee | ||
|
hanzo-dev
|
0542e97a06 | ||
|
|
265f807635 |
+282
-21
@@ -48,15 +48,27 @@ jobs:
|
||||
# onto a bare runner and it just works.
|
||||
run: |
|
||||
set -e
|
||||
need=0
|
||||
python3 -c 'import yaml' 2>/dev/null || need=1
|
||||
command -v jq >/dev/null 2>&1 || need=1
|
||||
if [ "$need" = 1 ]; then
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq python3-yaml jq
|
||||
# Sudo-free FIRST (bare arc nodes often lack passwordless sudo / apt
|
||||
# network — `sudo apt-get` then dies with no captured logs). Install jq
|
||||
# as a static binary and PyYAML via pip --user into ~/.local/bin; fall
|
||||
# back to apt only if those are unavailable. Works on bare AND baked nodes.
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
mkdir -p "$HOME/.local/bin"
|
||||
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
curl -fsSL https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64 \
|
||||
-o "$HOME/.local/bin/jq" && chmod +x "$HOME/.local/bin/jq" \
|
||||
|| { sudo apt-get update -qq && sudo apt-get install -y -qq jq; }
|
||||
fi
|
||||
# yq (mikefarah) — static Go binary, curl-installed like jq. Replaces the
|
||||
# PyYAML/python3 YAML parse that could not be provisioned on locked-down
|
||||
# arc nodes (sudo blocked by no_new_privs, pip/PyPI unavailable).
|
||||
if ! command -v yq >/dev/null 2>&1; then
|
||||
curl -fsSL https://github.com/mikefarah/yq/releases/download/v4.44.3/yq_linux_amd64 \
|
||||
-o "$HOME/.local/bin/yq" && chmod +x "$HOME/.local/bin/yq"
|
||||
fi
|
||||
python3 -c 'import yaml; print("PyYAML", yaml.__version__)'
|
||||
jq --version
|
||||
yq --version
|
||||
|
||||
- name: Delegate build to platform (mode=delegate)
|
||||
# The GHA-escape fast path: instead of running buildx on this runner, POST
|
||||
@@ -85,7 +97,7 @@ jobs:
|
||||
# One enqueue per (image, platform), mirroring the buildx tag shape the
|
||||
# deploy path expects (`sha-<short>-<arch>[-<suffix>]`). Default arch is
|
||||
# amd64 (single-arch), so an existing repo's tag shape is unchanged.
|
||||
python3 -c "import yaml,json;print(json.dumps(yaml.safe_load(open('hanzo.yml'))['images']))" | jq -c '.[]' | while read -r img; do
|
||||
yq -o=json -I=0 '.images' hanzo.yml | jq -c '.[]' | while read -r img; do
|
||||
name=$(echo "$img"|jq -r .name); repo=$(echo "$img"|jq -r .repo)
|
||||
ctx=$(echo "$img"|jq -r .context); df=$(echo "$img"|jq -r '.dockerfile // (.context+"/Dockerfile")')
|
||||
sfx=$(echo "$img"|jq -r '."tag-suffix" // ""')
|
||||
@@ -182,10 +194,26 @@ jobs:
|
||||
# repo-scoped automatic token cannot read. Handed to buildx as the
|
||||
# `gh_token` BuildKit secret.
|
||||
# KUBECONFIG — cluster access for the deploy step.
|
||||
ORG="$(python3 -c "import yaml;print((yaml.safe_load(open('hanzo.yml')).get('kms') or {}).get('org','') )" 2>/dev/null || true)"
|
||||
ORG="$(yq -r '.kms.org // ""' hanzo.yml 2>/dev/null || true)"
|
||||
: "${ORG:=${KMS_ORG:-}}"
|
||||
ENV="$(python3 -c "import yaml;print((yaml.safe_load(open('hanzo.yml')).get('kms') or {}).get('environment','prod'))" 2>/dev/null || echo prod)"
|
||||
PATHQ="$(python3 -c "import yaml;print((yaml.safe_load(open('hanzo.yml')).get('kms') or {}).get('path','/deploy').strip('/'))" 2>/dev/null || echo deploy)"
|
||||
# Systemic default: when neither hanzo.yml `kms.org` nor the KMS_ORG var
|
||||
# is set, derive the KMS org from the GitHub owner. The org → KMS-org
|
||||
# relation is a small, stable first-party fact; keeping it here (ONE
|
||||
# place) is why every repo gets the canonical KMS deploy-cred path with
|
||||
# zero per-repo config — before this, ORG was always empty and this whole
|
||||
# step short-circuited, so no build ever fetched its private-dep git
|
||||
# credential from KMS (it silently fell back to GH_PAT and failed on
|
||||
# private hanzoai/cloud). hanzo.yml `kms.org` and KMS_ORG still override.
|
||||
if [ -z "$ORG" ]; then
|
||||
case "${{ github.repository_owner }}" in
|
||||
hanzoai) ORG=hanzo ;;
|
||||
luxfi) ORG=lux ;;
|
||||
zooai) ORG=zoo ;;
|
||||
*) ORG="${{ github.repository_owner }}" ;;
|
||||
esac
|
||||
fi
|
||||
ENV="$(yq -r '.kms.environment // "prod"' hanzo.yml 2>/dev/null || echo prod)"
|
||||
PATHQ="$(yq -r '.kms.path // "/deploy"' hanzo.yml | sed 's#^/##; s#/$##' 2>/dev/null || echo deploy)"
|
||||
if [ -z "${KMS_CLIENT_ID:-}" ] || [ -z "$ORG" ]; then
|
||||
echo "::notice::KMS not configured (no KMS_CLIENT_ID or org) — skipping; GHCR push uses the workflow token"; exit 0
|
||||
fi
|
||||
@@ -201,12 +229,64 @@ jobs:
|
||||
KUBECONFIG_B64=$(get KUBECONFIG)
|
||||
if [ -n "$KUBECONFIG_B64" ]; then echo "$KUBECONFIG_B64" | base64 -d > "$RUNNER_TEMP/kubeconfig"; echo "kubeconfig=$RUNNER_TEMP/kubeconfig" >> "$GITHUB_OUTPUT"; fi
|
||||
|
||||
- name: Mirror credential (registry.hanzo.ai)
|
||||
if: inputs.mode != 'delegate'
|
||||
env:
|
||||
KUBECONFIG: ${{ steps.kms.outputs.kubeconfig }}
|
||||
REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
|
||||
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
# Direct credential first (repo/org secret — works on private repos,
|
||||
# where the Free plan hides org secrets... including these; private
|
||||
# repos set them at REPO level). KMS-kubeconfig read is the fallback.
|
||||
if [ -n "${REGISTRY_USER:-}" ] && [ -n "${REGISTRY_PASSWORD:-}" ]; then
|
||||
# Best-effort: a registry.hanzo.ai login FAILURE (not just a missing
|
||||
# cred) must never fail the run — the image still pushes to GHCR, the
|
||||
# primary. Without this guard, bash -e aborts the step and SKIPS the
|
||||
# build entirely (a registry hiccup takes the whole lane red).
|
||||
if echo "$REGISTRY_PASSWORD" | docker login registry.hanzo.ai -u "$REGISTRY_USER" --password-stdin; then
|
||||
echo "MIRROR_OK=1" >> "$GITHUB_ENV"
|
||||
else
|
||||
echo "::notice::registry.hanzo.ai login failed — mirror skipped (GHCR-only push)"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
[ -z "${KUBECONFIG:-}" ] && { echo "::notice::no registry secret and no kubeconfig — mirror skipped"; exit 0; }
|
||||
# Bare arc runners ship no kubectl — same static provision the deploy
|
||||
# step uses.
|
||||
command -v kubectl >/dev/null 2>&1 || {
|
||||
KVER=$(curl -fsSL https://dl.k8s.io/release/stable.txt)
|
||||
mkdir -p "$HOME/.local/bin"
|
||||
curl -fsSL "https://dl.k8s.io/release/${KVER}/bin/linux/amd64/kubectl" -o "$HOME/.local/bin/kubectl" && chmod +x "$HOME/.local/bin/kubectl"
|
||||
echo "$HOME/.local/bin" >> "$GITHUB_PATH"; export PATH="$HOME/.local/bin:$PATH"
|
||||
}
|
||||
CFG=$(kubectl -n hanzo get secret registry-credentials -o jsonpath='{.data.\.dockerconfigjson}' 2>/dev/null | base64 -d || true)
|
||||
if [ -z "$CFG" ]; then
|
||||
echo "::notice::registry-credentials not readable from this kubeconfig — mirror skipped (GHCR-only push)"; exit 0
|
||||
fi
|
||||
USERPASS=$(echo "$CFG" | jq -r '.auths["registry.hanzo.ai"].auth // empty' | base64 -d)
|
||||
[ -z "$USERPASS" ] && { echo "::notice::no registry.hanzo.ai auth in dockerconfig — mirror skipped"; exit 0; }
|
||||
echo "::add-mask::${USERPASS#*:}"
|
||||
# Best-effort: login failure → skip mirror, never fail the run (see above).
|
||||
if echo "${USERPASS#*:}" | docker login registry.hanzo.ai -u "${USERPASS%%:*}" --password-stdin; then
|
||||
echo "MIRROR_OK=1" >> "$GITHUB_ENV"
|
||||
else
|
||||
echo "::notice::registry.hanzo.ai login failed — mirror skipped (GHCR-only push)"
|
||||
fi
|
||||
|
||||
- name: Build & push images (per hanzo.yml)
|
||||
if: inputs.mode != 'delegate'
|
||||
env:
|
||||
GH_PAT: ${{ secrets.GH_PAT }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Test-only callers (hanzo.yml without `images:` — e.g. a repo whose
|
||||
# image lane lives in its own release.yml, or a pure library) skip the
|
||||
# build step entirely instead of exploding on a null .images.
|
||||
if [ "$(yq -r '.images // [] | length' hanzo.yml 2>/dev/null || echo 0)" = "0" ]; then
|
||||
echo "::notice::no images: in hanzo.yml — test-only caller, skipping build"; exit 0
|
||||
fi
|
||||
# Build-time private cross-org Go module read (the buildx `gh_token`
|
||||
# secret): prefer the KMS-fetched GIT_TOKEN, else fall back to the org
|
||||
# GH_PAT — the SAME BuildKit gh_token cloud's release.yml uses (proven
|
||||
@@ -219,7 +299,7 @@ jobs:
|
||||
SHORT=$(echo "${{ github.sha }}" | cut -c1-7)
|
||||
IS_TAG=$([ "${{ github.ref_type }}" = "tag" ] && echo 1 || echo 0)
|
||||
VER="${{ github.ref_name }}"; VER="${VER#v}"
|
||||
python3 -c "import yaml,json;print(json.dumps(yaml.safe_load(open('hanzo.yml'))['images']))" | jq -c '.[]' | while read -r img; do
|
||||
yq -o=json -I=0 '.images' hanzo.yml | jq -c '.[]' | while read -r img; do
|
||||
name=$(echo "$img"|jq -r .name); ctx=$(echo "$img"|jq -r .context)
|
||||
df=$(echo "$img"|jq -r '.dockerfile // (.context+"/Dockerfile")'); repo=$(echo "$img"|jq -r .repo)
|
||||
# tag-suffix is OPTIONAL (most repos ship a single variant). When set
|
||||
@@ -239,7 +319,10 @@ jobs:
|
||||
if [ "$plats" = "linux/amd64" ]; then
|
||||
# single-arch: keep the exact legacy tag shape (-amd64) deploys expect.
|
||||
TAGS="-t $repo:sha-${SHORT}-amd64${sfx:+-$sfx} -t $repo:${sfx:+$sfx-}latest"
|
||||
[ "$IS_TAG" = 1 ] && TAGS="$TAGS -t $repo:${VER}-amd64${sfx:+-$sfx}"
|
||||
# Release (tag) build: publish the CANONICAL bare semver tag (what the
|
||||
# universe CR pins to — matches world 2.4.10 / cloud v1.801.62) AND the
|
||||
# legacy -amd64 semver alias (back-compat for CRs still on that shape).
|
||||
[ "$IS_TAG" = 1 ] && TAGS="$TAGS -t $repo:${VER}${sfx:+-$sfx} -t $repo:${VER}-amd64${sfx:+-$sfx}"
|
||||
else
|
||||
# multi-arch: one arch-neutral manifest-list tag (no -amd64 suffix).
|
||||
docker run --privileged --rm tonistiigi/binfmt --install arm64 >/dev/null 2>&1 || true
|
||||
@@ -251,14 +334,103 @@ jobs:
|
||||
# BuildKit secret so Dockerfiles can clone private Go modules; omitted
|
||||
# cleanly when absent (public-only builds unaffected).
|
||||
docker buildx build --platform "$plats" ${GIT_TOKEN:+--secret id=gh_token,env=GIT_TOKEN} --push $TAGS -f "$df" "$ctx"
|
||||
# Dual-host: mirror the exact tag set to registry.hanzo.ai (server-
|
||||
# side manifest copy — no rebuild). ghcr.io/<org>/<name> →
|
||||
# registry.hanzo.ai/<org>/<name>; public consumers keep ghcr, the
|
||||
# fleet DEPLOYS from our registry.
|
||||
if [ "${MIRROR_OK:-}" = "1" ]; then
|
||||
# crane, not buildx imagetools: the IAM token realm doesn't answer
|
||||
# buildx's multi-scope token request (spec gap, tracked).
|
||||
command -v crane >/dev/null 2>&1 || {
|
||||
mkdir -p "$HOME/.local/bin"
|
||||
curl -fsSL https://github.com/google/go-containerregistry/releases/download/v0.20.2/go-containerregistry_Linux_x86_64.tar.gz \
|
||||
| tar -xz -C "$HOME/.local/bin" crane
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
}
|
||||
mrepo="registry.hanzo.ai/${repo#*/}"
|
||||
echo "$TAGS" | tr ' ' '\n' | grep -v '^-t$' | grep -v '^$' | while read -r ref; do
|
||||
crane copy "$ref" "${mrepo}:${ref##*:}" \
|
||||
|| echo "::warning::mirror of $ref failed (ghcr push unaffected)"
|
||||
done
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
done
|
||||
|
||||
- name: Provision Go toolchain (go test gates on bare runners)
|
||||
# hanzo.yml `test:` gates (e.g. `go vet ./...`, `go test ...`) run
|
||||
# DIRECTLY on the runner, NOT inside a build container — but the stock
|
||||
# arc runner image (ghcr.io/actions/actions-runner) ships no Go, so a Go
|
||||
# gate dies with `go: command not found` (exit 127). Provision the repo's
|
||||
# OWN Go version from go.mod so the toolchain matches the module exactly.
|
||||
# Guarded to Go repos (go.mod present) so pure-JS/TS callers are
|
||||
# unaffected; harmless if a future runner image bakes Go in.
|
||||
if: inputs.mode != 'delegate' && hashFiles('go.mod') != ''
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: false
|
||||
|
||||
- name: Provision C toolchain (cgo test gates)
|
||||
# CGO_ENABLED=1 gates (e.g. go-sqlite3, which bundles the sqlite
|
||||
# amalgamation) need a C compiler; the minimal arc runner ships none
|
||||
# (`cgo: gcc not found`). Same guarded provision the parse-toolchain step
|
||||
# above uses — a no-op when gcc is already present. If apt-get update
|
||||
# fails (arc snapshot mirror rot: "no longer has a Release file"),
|
||||
# repoint archive.ubuntu.com at the DO mirror — both sources.list and
|
||||
# noble's deb822 ubuntu.sources — and retry once.
|
||||
if: inputs.mode != 'delegate' && hashFiles('go.mod') != ''
|
||||
run: |
|
||||
command -v gcc >/dev/null 2>&1 && exit 0
|
||||
sudo apt-get update -qq || {
|
||||
sudo find /etc/apt -name '*.list' -o -name '*.sources' | \
|
||||
xargs -r sudo sed -i 's|https\?://archive.ubuntu.com/ubuntu|http://mirrors.digitalocean.com/ubuntu|g'
|
||||
sudo apt-get update -qq
|
||||
}
|
||||
sudo apt-get install -y -qq gcc
|
||||
|
||||
- name: Provision Node toolchain (JS test gates)
|
||||
# JS/TS `test:` gates (e.g. `pnpm install --frozen-lockfile && pnpm lint`)
|
||||
# also run DIRECTLY on the runner, and the stock arc runner image ships
|
||||
# no Node — so the gate dies at `corepack: command not found` (exit 127)
|
||||
# before it ever reads package.json. Same guarded provision as the Go
|
||||
# step above: only for JS callers (package.json present), harmless if a
|
||||
# future runner image bakes Node in. `corepack enable` shims the repo's
|
||||
# own packageManager (pnpm/yarn) at its pinned version.
|
||||
if: inputs.mode != 'delegate' && hashFiles('package.json') != ''
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
- name: Enable corepack (pnpm/yarn shims for JS test gates)
|
||||
if: inputs.mode != 'delegate' && hashFiles('package.json') != ''
|
||||
run: corepack enable
|
||||
|
||||
- name: Authenticate runner git for private Go modules (test gates)
|
||||
# The Test step runs `go vet`/`go test` ON the runner (not in buildx), so
|
||||
# `go` fetches private hanzoai/* modules (GOPRIVATE → direct) through the
|
||||
# runner's git, which needs a credential. The IMAGE build authenticates
|
||||
# via the KMS `gh_token` inside buildx (GIT_TOKEN); the earlier GH_PAT
|
||||
# git-auth step is a no-op for repos that rely on that KMS token (GH_PAT
|
||||
# unset) — so `go vet` dies with `could not read Username for github.com`.
|
||||
# Reuse the SAME token here for the runner's git (GIT_TOKEN, set by the
|
||||
# KMS step above; GH_PAT fallback). No-op when neither is present.
|
||||
if: inputs.mode != 'delegate' && hashFiles('go.mod') != ''
|
||||
env:
|
||||
GH_PAT: ${{ secrets.GH_PAT }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
TOKEN="${GIT_TOKEN:-${GH_PAT:-}}"
|
||||
if [ -z "$TOKEN" ]; then echo "no git token — public modules only"; exit 0; fi
|
||||
CFG="$RUNNER_TEMP/gitconfig-go-test"; : > "$CFG"
|
||||
GIT_CONFIG_GLOBAL="$CFG" git config --global \
|
||||
url."https://x-access-token:${TOKEN}@github.com/".insteadOf "https://github.com/"
|
||||
{ echo "GIT_CONFIG_GLOBAL=$CFG"; echo "GIT_CONFIG_NOSYSTEM=1"; } >> "$GITHUB_ENV"
|
||||
echo "runner git authenticated for private Go modules"
|
||||
|
||||
- name: Test (per hanzo.yml)
|
||||
if: inputs.mode != 'delegate'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 -c "import yaml,json;print(json.dumps(yaml.safe_load(open('hanzo.yml')).get('test') or []))" | jq -c '.[]' | while read -r t; do
|
||||
yq -o=json -I=0 '.test // []' hanzo.yml | jq -c '.[]' | while read -r t; do
|
||||
name=$(echo "$t"|jq -r .name); cmd=$(echo "$t"|jq -r .run)
|
||||
echo "::group::test $name"; bash -c "$cmd"; echo "::endgroup::"
|
||||
done
|
||||
@@ -270,20 +442,109 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
REF="${{ github.ref_name }}"
|
||||
ON=$(python3 -c "import yaml,json;print(json.dumps((yaml.safe_load(open('hanzo.yml')).get('deploy') or {}).get('on') or []))")
|
||||
ON=$(yq -o=json -I=0 '.deploy.on // []' hanzo.yml)
|
||||
IS_TAG=$([ "${{ github.ref_type }}" = "tag" ] && echo 1 || echo 0)
|
||||
VER="${REF#v}" # canonical semver for tag releases (ref_name without the v)
|
||||
if [ "$IS_TAG" != 1 ] && ! echo "$ON" | jq -e --arg b "$REF" 'index($b)' >/dev/null; then
|
||||
echo "branch $REF not in deploy.on — skipping deploy"; exit 0
|
||||
fi
|
||||
# Bare arc runners ship no kubectl — provision the static binary
|
||||
# (same sudo-free pattern as jq/yq above).
|
||||
command -v kubectl >/dev/null 2>&1 || {
|
||||
mkdir -p "$HOME/.local/bin"; export PATH="$HOME/.local/bin:$PATH"
|
||||
KVER=$(curl -fsSL https://dl.k8s.io/release/stable.txt)
|
||||
curl -fsSL "https://dl.k8s.io/release/${KVER}/bin/linux/amd64/kubectl" \
|
||||
-o "$HOME/.local/bin/kubectl" && chmod +x "$HOME/.local/bin/kubectl"
|
||||
}
|
||||
SHORT=$(echo "${{ github.sha }}" | cut -c1-7)
|
||||
NS=$(python3 -c "import yaml;print(yaml.safe_load(open('hanzo.yml'))['deploy']['namespace'])")
|
||||
python3 -c "import yaml,json;print(json.dumps(yaml.safe_load(open('hanzo.yml'))['images']))" > /tmp/imgs.json
|
||||
python3 -c "import yaml,json;print(json.dumps(yaml.safe_load(open('hanzo.yml'))['deploy']['services']))" | jq -c '.[]' | while read -r s; do
|
||||
NS=$(yq -r '.deploy.namespace' hanzo.yml)
|
||||
RTO=$(yq -r '.deploy."rollout-timeout" // "600s"' hanzo.yml)
|
||||
yq -o=json -I=0 '.images' hanzo.yml > /tmp/imgs.json
|
||||
# Desired state lives in the universe repo: the in-cluster
|
||||
# gitops-reconcile job re-applies its CRs every ~5 minutes, so any
|
||||
# direct patch that is not ALSO recorded there is reverted on the
|
||||
# next cycle. Record first (durable), then patch (accelerates the
|
||||
# roll). Universe write rides the same KMS git token as module reads.
|
||||
UNIVERSE=""
|
||||
if [ -n "${GIT_TOKEN:-}" ]; then
|
||||
git clone -q --depth 1 \
|
||||
"https://x-access-token:${GIT_TOKEN}@github.com/hanzoai/universe.git" \
|
||||
/tmp/universe 2>/dev/null && UNIVERSE=/tmp/universe \
|
||||
|| echo "::warning::universe clone failed — rolls below are transient until recorded there"
|
||||
fi
|
||||
yq -o=json -I=0 '.deploy.services' hanzo.yml | jq -c '.[]' | while read -r s; do
|
||||
svc=$(echo "$s"|jq -r .name); imgname=$(echo "$s"|jq -r .image)
|
||||
repo=$(jq -r --arg n "$imgname" '.[]|select(.name==$n)|.repo' /tmp/imgs.json)
|
||||
sfx=$(jq -r --arg n "$imgname" '.[]|select(.name==$n)|."tag-suffix" // ""' /tmp/imgs.json)
|
||||
ref="$repo:sha-${SHORT}-amd64${sfx:+-$sfx}"
|
||||
plats=$(jq -r --arg n "$imgname" '.[]|select(.name==$n)|(.platforms // ["linux/amd64"])|join(",")' /tmp/imgs.json)
|
||||
# A TAGGED release pins the CANONICAL bare semver (VER = ref_name w/o the
|
||||
# v) — the very image the build step published above; a branch build stays
|
||||
# on its transient per-commit sha tag (arch-matched: bare for multi-arch).
|
||||
if [ "$IS_TAG" = 1 ]; then
|
||||
tag="${VER}${sfx:+-$sfx}"
|
||||
elif [ "$plats" = "linux/amd64" ]; then
|
||||
tag="sha-${SHORT}-amd64${sfx:+-$sfx}"
|
||||
else
|
||||
tag="sha-${SHORT}${sfx:+-$sfx}"
|
||||
fi
|
||||
ref="$repo:$tag"
|
||||
CR="$UNIVERSE/infra/k8s/operator/crs/$svc.yaml"
|
||||
# Backward-clobber guard (semver): a transient BRANCH build must NEVER
|
||||
# overwrite a service pinned to a semver RELEASE (rolls prod back to a dev
|
||||
# image). Complements the sha-ancestry guard below, which only sees sha-
|
||||
# tags. Releases are tagged; a branch push leaves the semver pin untouched.
|
||||
cur=""
|
||||
if [ -n "$UNIVERSE" ] && [ -f "$CR" ]; then cur="$(yq -r '.spec.image.tag // ""' "$CR")"; fi
|
||||
if [ "$IS_TAG" != 1 ] && printf '%s' "$cur" | grep -qE '^v?[0-9]+\.[0-9]+\.[0-9]+'; then
|
||||
echo "::notice::$svc pinned to release $cur — branch build ${SHORT} leaves it (tag a release to deploy)"; continue
|
||||
fi
|
||||
echo "rolling $svc → $ref"
|
||||
kubectl -n "$NS" set image "deployment/$svc" "*=$ref"
|
||||
kubectl -n "$NS" rollout status "deployment/$svc" --timeout=180s
|
||||
if [ -n "$UNIVERSE" ] && [ -f "$CR" ]; then
|
||||
# Never roll a pin BACKWARD: builds finish out of order, and a slow
|
||||
# build of an older commit must not overwrite a newer roll. If the
|
||||
# CR's current sha is a descendant of ours, ours is stale — skip.
|
||||
CURSHA=$(yq -r '.spec.image.tag // ""' "$CR" | sed -nE 's/^sha-([a-f0-9]{7}).*/\1/p')
|
||||
git fetch -q --unshallow origin 2>/dev/null || true
|
||||
if [ -n "$CURSHA" ] && [ "$CURSHA" != "$SHORT" ] \
|
||||
&& git cat-file -e "$CURSHA" 2>/dev/null \
|
||||
&& git merge-base --is-ancestor "$SHORT" "$CURSHA" 2>/dev/null; then
|
||||
echo "::notice::$svc CR already at descendant $CURSHA — not rolling back to $SHORT"
|
||||
else
|
||||
yq -i ".spec.image.tag = \"$tag\"" "$CR"
|
||||
# Sweep EVERY same-repo image reference in the CR (sidecars,
|
||||
# initContainers pinned to this image) — the tag field alone
|
||||
# left sidecars on stale tags every roll.
|
||||
repoEsc=$(printf '%s' "$repo" | sed 's/[.[\*^$]/\\&/g')
|
||||
sed -i -E "s|(${repoEsc}):sha-[A-Za-z0-9-]+|\1:${tag}|g" "$CR"
|
||||
git -C "$UNIVERSE" diff --quiet || {
|
||||
git -C "$UNIVERSE" -c user.name=hanzo-ci -c user.email=dev@hanzo.ai \
|
||||
commit -qam "deploy($svc): $tag (${GITHUB_REPOSITORY}@${SHORT})"
|
||||
git -C "$UNIVERSE" push -q \
|
||||
|| echo "::warning::universe push failed for $svc — roll is transient"
|
||||
}
|
||||
fi
|
||||
fi
|
||||
# Operator-managed services (Service CR) reconcile the Deployment —
|
||||
# patch the CR when it exists; bare Deployments get set-image.
|
||||
if kubectl -n "$NS" get "services.hanzo.ai/$svc" >/dev/null 2>&1; then
|
||||
kubectl -n "$NS" patch "services.hanzo.ai/$svc" --type=merge \
|
||||
-p "{\"spec\":{\"image\":{\"repository\":\"$repo\",\"tag\":\"$tag\"}}}"
|
||||
else
|
||||
# Bare Deployment: set the new image ONLY on containers already running
|
||||
# THIS repo's image — never the '*' wildcard, which clobbers foreign
|
||||
# sidecars (e.g. an rclone S3-mirror) with the app image and wedges the
|
||||
# rollout. Fall back to the service-named container on first deploy.
|
||||
mapfile -t CS < <(kubectl -n "$NS" get "deployment/$svc" \
|
||||
-o jsonpath='{range .spec.template.spec.containers[*]}{.name} {.image}{"\n"}{end}' \
|
||||
| awk -v r="$repo" 'index($2, r"@")==1 || index($2, r":")==1 {print $1}')
|
||||
[ "${#CS[@]}" -eq 0 ] && CS=("$svc")
|
||||
args=(); for c in "${CS[@]}"; do args+=("$c=$ref"); done
|
||||
kubectl -n "$NS" set image "deployment/$svc" "${args[@]}"
|
||||
fi
|
||||
# Timeout scales with the image: a service vendoring GB-scale model/node
|
||||
# packs (e.g. studio) pulls multi-GB layers on a cold node + waits for the
|
||||
# operator to reconcile — legitimately minutes. 180s failed mid-pull and
|
||||
# is THE reason studio deploys silently failed since 0.15.8. Override
|
||||
# per-repo with deploy.rollout-timeout.
|
||||
kubectl -n "$NS" rollout status "deployment/$svc" --timeout="$RTO"
|
||||
done
|
||||
|
||||
Reference in New Issue
Block a user