fix(kms): fail-closed app-layer auth on validator-key routes (sign/keygen/rotate)

The /v1/kms/keys/* routes (keygen, sign, rotate, metadata reads) were gated
ONLY by requireMPC — no caller-identity check. The routes were declared
"unprotected at the HTTP layer — auth enforced by Gateway/NetworkPolicy". Any
caller reaching :8080 directly (NetworkPolicy misconfig, pod compromise,
port-forward, SSRF) could sign arbitrary digests with any custodied key, or
keygen/rotate at will. Trusting a Gateway-injected role header is not enough:
a direct caller forges the header; only an IAM signature cannot be forged.

Add requireKeyAuth (orgJWTAuth): verifies the IAM-signed JWT HERE against
JWKS (sig + issuer + expiry) and requires the kms-admin role (superadmin is
break-glass) — the same write authority the zapserver /v1/sdk sign op and the
mpc KMS handlers already enforce. Fail closed at every step: nil auth => 503,
missing/malformed/bad token => 401, authenticated-but-no-role => 403. This is
defense in depth BEHIND the NetworkPolicy, never a replacement for it. The
/v1/kms/status health probe stays open (no key material).

Tests (SDKROOT="$(xcrun --show-sdk-path)" go test ./cmd/kms):
  requireKeyAuth: missing bearer 401, malformed 401, nil-auth 503 (fail
  closed), valid-token-without-role 403, kms-admin 200, superadmin 200.
  failopen suite updated to authenticate (kms-admin) so it still pins the
  requireMPC 503/recovery path behind the new gate. Full cmd/kms: ok.

Co-authored-by: Hanzo Dev <dev@hanzo.ai>

cmd/kms/auth.go

@@ -297,6 +297,60 @@ func (a *orgJWTAuth) requireOrgJWT(next http.HandlerFunc) http.HandlerFunc {
 	}
 }
 
+// requireKeyAuth wraps a validator-key-management handler (keygen, sign,
+// rotate, and the key-metadata reads) with app-layer IAM JWT verification.
+//
+// Unlike requireOrgJWT there is no {org} path param to bind to: validator
+// key custody is a cluster-admin operation, not an org-scoped secret. It is
+// gated on the kms-admin role (superadmin is the break-glass override),
+// granted in IAM — the same authority the zapserver /v1/sdk sign op and the
+// mpc KMS handlers enforce for their write path.
+//
+// Fail closed at every step:
+//   - nil receiver (IAM not wired)     => 503, never open
+//   - missing / malformed bearer       => 401
+//   - bad signature / issuer / expiry  => 401
+//   - authenticated but missing role   => 403
+//
+// The JWT signature is verified HERE against IAM's JWKS. We deliberately do
+// NOT trust a gateway-injected role header (e.g. X-IAM-Roles): a caller that
+// reaches :8080 directly — NetworkPolicy gap, port-forward, pod compromise,
+// SSRF — could forge such a header but cannot forge an IAM signature. This
+// is the defense-in-depth layer that stands even if the NetworkPolicy fails,
+// not a replacement for it.
+func (a *orgJWTAuth) requireKeyAuth(next http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if a == nil {
+			writeJSON(w, http.StatusServiceUnavailable, map[string]any{
+				"statusCode": 503, "message": "auth not configured",
+			})
+			return
+		}
+		raw := bearerToken(r)
+		if raw == "" {
+			writeJSON(w, http.StatusUnauthorized, map[string]any{
+				"statusCode": 401, "message": "missing bearer token",
+			})
+			return
+		}
+		claims, err := a.validate(r.Context(), raw)
+		if err != nil {
+			log.Printf("kms: key-auth reject: %v (path=%s)", err, r.URL.Path)
+			writeJSON(w, http.StatusUnauthorized, map[string]any{
+				"statusCode": 401, "message": "invalid token",
+			})
+			return
+		}
+		if !hasRole(claims.Roles, roleKMSAdmin) && !hasRole(claims.Roles, roleSuperadmin) {
+			writeJSON(w, http.StatusForbidden, map[string]any{
+				"statusCode": 403, "message": "kms key operations require the kms-admin role",
+			})
+			return
+		}
+		next(w, r)
+	}
+}
+
 func bearerToken(r *http.Request) string {
 	h := r.Header.Get("Authorization")
 	if h == "" {

cmd/kms/auth_test.go

@@ -524,6 +524,99 @@ func TestRequireOrgJWT_multiIssuerWhiteLabel(t *testing.T) {
 	}
 }
 
+// newTestKeyAuth returns an *orgJWTAuth backed by a stub IAM plus a bearer
+// token carrying the given roles — the credential the validator-key routes
+// require. Caller defers cleanup() to shut the stub IAM down.
+func newTestKeyAuth(t *testing.T, roles ...string) (auth *orgJWTAuth, bearer string, cleanup func()) {
+	t.Helper()
+	signer, jwks := newTestSigner(t)
+	iam := httptest.NewServer(jwksHandler(jwks))
+	auth = newOrgJWTAuth(iam.URL, "")
+	bearer = signOrgClaims(t, signer, orgClaims{
+		Claims: jwt.Claims{
+			Issuer:  iam.URL,
+			Subject: "ops",
+			Expiry:  jwt.NewNumericDate(time.Now().Add(time.Hour)),
+		},
+		Owner: "operator-org",
+		Roles: roles,
+	})
+	return auth, bearer, iam.Close
+}
+
+// serveKeyAuth drives one request through requireKeyAuth-wrapped handler
+// and returns the recorder. bearer=="" sends no Authorization header.
+func serveKeyAuth(t *testing.T, auth *orgJWTAuth, bearer string) *httptest.ResponseRecorder {
+	t.Helper()
+	mux := http.NewServeMux()
+	mux.HandleFunc("POST /v1/kms/keys/{id}/sign", auth.requireKeyAuth(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	req := httptest.NewRequest(http.MethodPost, "/v1/kms/keys/v-1/sign", strings.NewReader(`{}`))
+	if bearer != "" {
+		req.Header.Set("Authorization", "Bearer "+bearer)
+	}
+	rec := httptest.NewRecorder()
+	mux.ServeHTTP(rec, req)
+	return rec
+}
+
+// The validator-key routes must fail CLOSED when auth cannot be
+// established. This is the fix for the /sign hole: a caller reaching
+// :8080 directly (NetworkPolicy gap, port-forward, SSRF) is rejected
+// before any signing occurs.
+
+func TestRequireKeyAuth_missingBearer(t *testing.T) {
+	rec := serveKeyAuth(t, newOrgJWTAuth("https://iam.example.com", ""), "")
+	if rec.Code != http.StatusUnauthorized {
+		t.Errorf("no token: got %d want 401; body=%s", rec.Code, rec.Body.String())
+	}
+}
+
+func TestRequireKeyAuth_malformedTokenRejected(t *testing.T) {
+	rec := serveKeyAuth(t, newOrgJWTAuth("https://iam.example.com", ""), "not-a-jwt")
+	if rec.Code != http.StatusUnauthorized {
+		t.Errorf("garbage token: got %d want 401; body=%s", rec.Code, rec.Body.String())
+	}
+}
+
+// nil auth (IAM_ENDPOINT unwired) must 503 — NEVER fall open to signing.
+func TestRequireKeyAuth_nilAuthFailsClosed(t *testing.T) {
+	rec := serveKeyAuth(t, (*orgJWTAuth)(nil), "")
+	if rec.Code != http.StatusServiceUnavailable {
+		t.Errorf("nil auth: got %d want 503 (fail closed); body=%s", rec.Code, rec.Body.String())
+	}
+}
+
+// A validly-signed token that lacks kms-admin/superadmin must 403 — a
+// bearer of ANY IAM token cannot sign validator keys.
+func TestRequireKeyAuth_validTokenWithoutRoleForbidden(t *testing.T) {
+	auth, bearer, cleanup := newTestKeyAuth(t) // no roles
+	defer cleanup()
+	rec := serveKeyAuth(t, auth, bearer)
+	if rec.Code != http.StatusForbidden {
+		t.Errorf("no role: got %d want 403; body=%s", rec.Code, rec.Body.String())
+	}
+}
+
+func TestRequireKeyAuth_kmsAdminAllowed(t *testing.T) {
+	auth, bearer, cleanup := newTestKeyAuth(t, roleKMSAdmin)
+	defer cleanup()
+	rec := serveKeyAuth(t, auth, bearer)
+	if rec.Code != http.StatusOK {
+		t.Errorf("kms-admin: got %d want 200; body=%s", rec.Code, rec.Body.String())
+	}
+}
+
+func TestRequireKeyAuth_superadminAllowed(t *testing.T) {
+	auth, bearer, cleanup := newTestKeyAuth(t, roleSuperadmin)
+	defer cleanup()
+	rec := serveKeyAuth(t, auth, bearer)
+	if rec.Code != http.StatusOK {
+		t.Errorf("superadmin: got %d want 200; body=%s", rec.Code, rec.Body.String())
+	}
+}
+
 func TestBearerToken_extracts(t *testing.T) {
 	cases := []struct {
 		header, want string

cmd/kms/failopen_test.go

@@ -79,6 +79,25 @@ func (f *fakeBackend) Decrypt(context.Context, string, []byte) (*mpc.DecryptResu
 	return nil, errors.New("decrypt unused in failopen tests")
 }
 
+// authedPost issues a POST carrying a kms-admin bearer — the credential
+// the validator-key routes now require. These tests pin the requireMPC
+// behaviour, which sits BEHIND requireKeyAuth, so they authenticate first;
+// an unauthenticated request would 401 before reaching requireMPC.
+func authedPost(t *testing.T, url, bearer, body string) *http.Response {
+	t.Helper()
+	req, err := http.NewRequest(http.MethodPost, url, strings.NewReader(body))
+	if err != nil {
+		t.Fatalf("build req: %v", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", "Bearer "+bearer)
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("do post %s: %v", url, err)
+	}
+	return resp
+}
+
 // requireMPC must short-circuit signing routes with 503 and a structured
 // error body when MPC is unreachable. Without this, a degraded KMS would
 // quietly invoke MPC and surface 500s with cryptic ZAP errors instead of
@@ -87,18 +106,17 @@ func TestRegisterKMSRoutes_SignReturns503WhenMPCDown(t *testing.T) {
 	backend := &fakeBackend{}
 	backend.setErr(errors.New("connection reset by peer"))
 
+	auth, bearer, cleanup := newTestKeyAuth(t, roleKMSAdmin)
+	defer cleanup()
 	mgr := keys.NewManager(backend, nil, "vault-1")
 	mux := http.NewServeMux()
 	mpcAvailable := false
-	registerKMSRoutes(mux, mgr, backend, &mpcAvailable)
+	registerKMSRoutes(mux, auth, mgr, backend, &mpcAvailable)
 
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
 
-	resp, err := http.Post(srv.URL+"/v1/kms/keys/v-1/sign", "application/json", strings.NewReader(`{"key_type":"bls","message":"aGVsbG8="}`))
-	if err != nil {
-		t.Fatalf("post sign: %v", err)
-	}
+	resp := authedPost(t, srv.URL+"/v1/kms/keys/v-1/sign", bearer, `{"key_type":"bls","message":"aGVsbG8="}`)
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusServiceUnavailable {
@@ -126,29 +144,25 @@ func TestRegisterKMSRoutes_KeygenAndRotateAlsoGated(t *testing.T) {
 	backend := &fakeBackend{}
 	backend.setErr(errors.New("dial tcp: i/o timeout"))
 
+	auth, bearer, cleanup := newTestKeyAuth(t, roleKMSAdmin)
+	defer cleanup()
 	mgr := keys.NewManager(backend, nil, "vault-1")
 	mux := http.NewServeMux()
 	mpcAvailable := false
-	registerKMSRoutes(mux, mgr, backend, &mpcAvailable)
+	registerKMSRoutes(mux, auth, mgr, backend, &mpcAvailable)
 
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
 
-	keygenResp, err := http.Post(srv.URL+"/v1/kms/keys/generate", "application/json",
-		strings.NewReader(`{"validator_id":"v-1","threshold":2,"parties":3}`))
-	if err != nil {
-		t.Fatalf("post generate: %v", err)
-	}
+	keygenResp := authedPost(t, srv.URL+"/v1/kms/keys/generate", bearer,
+		`{"validator_id":"v-1","threshold":2,"parties":3}`)
 	defer keygenResp.Body.Close()
 	if keygenResp.StatusCode != http.StatusServiceUnavailable {
 		t.Fatalf("generate status: got %d want 503", keygenResp.StatusCode)
 	}
 
-	rotateResp, err := http.Post(srv.URL+"/v1/kms/keys/v-1/rotate", "application/json",
-		strings.NewReader(`{"new_threshold":3}`))
-	if err != nil {
-		t.Fatalf("post rotate: %v", err)
-	}
+	rotateResp := authedPost(t, srv.URL+"/v1/kms/keys/v-1/rotate", bearer,
+		`{"new_threshold":3}`)
 	defer rotateResp.Body.Close()
 	if rotateResp.StatusCode != http.StatusServiceUnavailable {
 		t.Fatalf("rotate status: got %d want 503", rotateResp.StatusCode)
@@ -165,10 +179,12 @@ func TestRegisterKMSRoutes_RecoversAfterMPCComesUp(t *testing.T) {
 	backend := &fakeBackend{}
 	backend.setErr(errors.New("temporary outage"))
 
+	auth, bearer, cleanup := newTestKeyAuth(t, roleKMSAdmin)
+	defer cleanup()
 	mgr := keys.NewManager(backend, nil, "vault-1")
 	mux := http.NewServeMux()
 	mpcAvailable := false
-	registerKMSRoutes(mux, mgr, backend, &mpcAvailable)
+	registerKMSRoutes(mux, auth, mgr, backend, &mpcAvailable)
 
 	// Wrap with a recovery middleware so the inner mgr.SignWithBLS panic
 	// (nil store, expected for this minimal test rig) doesn't kill the
@@ -185,11 +201,7 @@ func TestRegisterKMSRoutes_RecoversAfterMPCComesUp(t *testing.T) {
 	defer srv.Close()
 
 	// First call: still down, expect 503 + a re-probe.
-	resp1, err := http.Post(srv.URL+"/v1/kms/keys/v-1/sign", "application/json",
-		strings.NewReader(`{"key_type":"bls","message":"aGVsbG8="}`))
-	if err != nil {
-		t.Fatalf("first post: %v", err)
-	}
+	resp1 := authedPost(t, srv.URL+"/v1/kms/keys/v-1/sign", bearer, `{"key_type":"bls","message":"aGVsbG8="}`)
 	resp1.Body.Close()
 	if resp1.StatusCode != http.StatusServiceUnavailable {
 		t.Fatalf("first call: got %d want 503", resp1.StatusCode)
@@ -207,11 +219,7 @@ func TestRegisterKMSRoutes_RecoversAfterMPCComesUp(t *testing.T) {
 	// Sign error). Either way the request crossed requireMPC, which is
 	// what we're pinning.
 	backend.setErr(nil)
-	resp2, err := http.Post(srv.URL+"/v1/kms/keys/v-1/sign", "application/json",
-		strings.NewReader(`{"key_type":"bls","message":"aGVsbG8="}`))
-	if err != nil {
-		t.Fatalf("second post: %v", err)
-	}
+	resp2 := authedPost(t, srv.URL+"/v1/kms/keys/v-1/sign", bearer, `{"key_type":"bls","message":"aGVsbG8="}`)
 	resp2.Body.Close()
 	if resp2.StatusCode == http.StatusServiceUnavailable {
 		t.Fatalf("second call: got 503 after MPC recovered; want any non-503")
@@ -224,11 +232,7 @@ func TestRegisterKMSRoutes_RecoversAfterMPCComesUp(t *testing.T) {
 	}
 
 	// Third call: flag is now true, requireMPC must skip the re-probe.
-	resp3, err := http.Post(srv.URL+"/v1/kms/keys/v-1/sign", "application/json",
-		strings.NewReader(`{"key_type":"bls","message":"aGVsbG8="}`))
-	if err != nil {
-		t.Fatalf("third post: %v", err)
-	}
+	resp3 := authedPost(t, srv.URL+"/v1/kms/keys/v-1/sign", bearer, `{"key_type":"bls","message":"aGVsbG8="}`)
 	resp3.Body.Close()
 	if got := backend.statusCalls.Load(); got != 2 {
 		t.Fatalf("status probe count must NOT increase after flag is up: got %d want 2", got)
@@ -342,18 +346,16 @@ func TestRegisterKMSRoutes_NilFlagFallsBackToProbe(t *testing.T) {
 	backend := &fakeBackend{}
 	backend.setErr(errors.New("down"))
 
+	auth, bearer, cleanup := newTestKeyAuth(t, roleKMSAdmin)
+	defer cleanup()
 	mgr := keys.NewManager(backend, nil, "vault-1")
 	mux := http.NewServeMux()
-	registerKMSRoutes(mux, mgr, backend, nil)
+	registerKMSRoutes(mux, auth, mgr, backend, nil)
 
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
 
-	resp, err := http.Post(srv.URL+"/v1/kms/keys/v-1/sign", "application/json",
-		strings.NewReader(`{"key_type":"bls","message":"aGVsbG8="}`))
-	if err != nil {
-		t.Fatalf("post: %v", err)
-	}
+	resp := authedPost(t, srv.URL+"/v1/kms/keys/v-1/sign", bearer, `{"key_type":"bls","message":"aGVsbG8="}`)
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusServiceUnavailable {
 		t.Fatalf("nil flag: got %d want 503", resp.StatusCode)

cmd/kms/main.go

@@ -366,7 +366,7 @@ func main() {
 			mgr := keys.NewManager(zapClient, keyStore, vaultID)
 			// Enable /v1/sdk sign/verify over the same MPC-backed manager.
 			signBackend = sdksign.New(mgr)
-			registerKMSRoutes(mux, mgr, zapClient, &mpcAvailable)
+			registerKMSRoutes(mux, auth, mgr, zapClient, &mpcAvailable)
 		}
 	}
 	if vaultID == "" {
@@ -574,11 +574,16 @@ func healthHandler(vaultID string, mpcAvailable *bool) http.HandlerFunc {
 	}
 }
 
-func registerKMSRoutes(mux *http.ServeMux, mgr *keys.Manager, mpcBackend keys.MPCBackend, mpcAvailable *bool) {
-	// KMS key routes are unprotected at the HTTP layer — auth is enforced
-	// by the Gateway (JWT validation + X-IAM-Roles header injection).
-	// In-cluster callers (ATS, BD, TA) go through Gateway; direct access
-	// is blocked by K8s NetworkPolicy (only Gateway can reach port 8080).
+func registerKMSRoutes(mux *http.ServeMux, auth *orgJWTAuth, mgr *keys.Manager, mpcBackend keys.MPCBackend, mpcAvailable *bool) {
+	// KMS validator-key routes (keygen / sign / rotate / metadata reads)
+	// are gated by app-layer IAM JWT auth (auth.requireKeyAuth: kms-admin
+	// role, fail closed). This is defense in depth BEHIND the Gateway and
+	// NetworkPolicy — never a substitute for them. A caller that reaches
+	// :8080 directly (NetworkPolicy gap, port-forward, pod compromise,
+	// SSRF) still cannot keygen/sign/rotate without a valid IAM signature,
+	// because the signature is verified here and no injected header is
+	// trusted. The /v1/kms/status health probe stays open (no key
+	// material; consumed by circuit-breakers that hold no token).
 	//
 	// requireMPC short-circuits with 503 + a re-probe attempt when the
 	// boot-time MPC handshake failed. The re-probe lets KMS recover
@@ -605,7 +610,7 @@ func registerKMSRoutes(mux *http.ServeMux, mgr *keys.Manager, mpcBackend keys.MP
 		return true
 	}
 
-	mux.HandleFunc("POST /v1/kms/keys/generate", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("POST /v1/kms/keys/generate", auth.requireKeyAuth(func(w http.ResponseWriter, r *http.Request) {
 		if !requireMPC(w, r) {
 			return
 		}
@@ -644,17 +649,17 @@ func registerKMSRoutes(mux *http.ServeMux, mgr *keys.Manager, mpcBackend keys.MP
 		log.Printf("kms: audit: keygen OK validator_id=%s bls_wallet=%s corona_wallet=%s threshold=%d parties=%d",
 			ks.ValidatorID, ks.BLSWalletID, ks.CoronaWalletID, ks.Threshold, ks.Parties)
 		writeJSON(w, http.StatusCreated, ks)
-	})
+	}))
 
-	mux.HandleFunc("GET /v1/kms/keys", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("GET /v1/kms/keys", auth.requireKeyAuth(func(w http.ResponseWriter, r *http.Request) {
 		list := mgr.List()
 		if list == nil {
 			list = []*keys.ValidatorKeySet{}
 		}
 		writeJSON(w, http.StatusOK, list)
-	})
+	}))
 
-	mux.HandleFunc("GET /v1/kms/keys/{id}", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("GET /v1/kms/keys/{id}", auth.requireKeyAuth(func(w http.ResponseWriter, r *http.Request) {
 		id := r.PathValue("id")
 		ks, err := mgr.Get(id)
 		if err != nil {
@@ -662,9 +667,9 @@ func registerKMSRoutes(mux *http.ServeMux, mgr *keys.Manager, mpcBackend keys.MP
 			return
 		}
 		writeJSON(w, http.StatusOK, ks)
-	})
+	}))
 
-	mux.HandleFunc("POST /v1/kms/keys/{id}/sign", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("POST /v1/kms/keys/{id}/sign", auth.requireKeyAuth(func(w http.ResponseWriter, r *http.Request) {
 		if !requireMPC(w, r) {
 			return
 		}
@@ -701,9 +706,9 @@ func registerKMSRoutes(mux *http.ServeMux, mgr *keys.Manager, mpcBackend keys.MP
 		}
 		log.Printf("kms: audit: sign OK validator_id=%s key_type=%s", id, req.KeyType)
 		writeJSON(w, http.StatusOK, resp)
-	})
+	}))
 
-	mux.HandleFunc("POST /v1/kms/keys/{id}/rotate", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("POST /v1/kms/keys/{id}/rotate", auth.requireKeyAuth(func(w http.ResponseWriter, r *http.Request) {
 		if !requireMPC(w, r) {
 			return
 		}
@@ -731,7 +736,7 @@ func registerKMSRoutes(mux *http.ServeMux, mgr *keys.Manager, mpcBackend keys.MP
 		log.Printf("kms: audit: rotate OK validator_id=%s new_threshold=%d new_parties=%d",
 			id, ks.Threshold, ks.Parties)
 		writeJSON(w, http.StatusOK, ks)
-	})
+	}))
 
 	mux.HandleFunc("GET /v1/kms/status", func(w http.ResponseWriter, r *http.Request) {
 		status, err := mpcBackend.Status(r.Context())