fix(s3api): bound gateway startup filer reads so an unresponsive filer can't hang :8333

Red flagged NewCircuitBreaker's unbounded context.Background() filer read as a latent
CrashLoopBackOff: on a K8s cold/co-restart the s3 gateway dials the filer Service (TCP
accepted by kube endpoints) before the filer answers RPCs, the unbounded read blocks
~4min, :8333 never binds, liveness kills the pod. Confirmed via goroutine dump; the fix
is a bounded context (zap-proto muxConn.CallContext honors ctx.Done()).

The startup path has THREE reads of this anti-pattern, all of which block :8333 — fixing
only the circuit breaker leaves the gateway hanging at the first one:
- command/s3.go GetFilerConfiguration: REQUIRED read, already in a for{}+Sleep(1s) retry
  loop, but the unbounded ctx defeats the retry (first call hangs forever). Bound each
  attempt (10s) so the retry loop actually retries until the filer answers.
- s3api/s3api_circuit_breaker.go: OPTIONAL read. Bound (10s); the existing fail-open path
  (circuit breaker disabled) runs on deadline.
- s3api/bucket_metadata.go BucketRegistry.init: OPTIONAL warm-up whose error path was
  glog.Fatal (itself a CrashLoop trigger). Bound (10s) + warn-and-proceed instead of
  Fatal; the registry warms lazily via GetBucketMetadata -> LoadBucketMetadataFromFiler.

Empirically verified on a local single-node cluster: the bounded ctx is honored
(muxConn.CallContext returns "context deadline exceeded" at 10s), :8333 binds (~10s vs an
indefinite hang before), and a SigV4 create-bucket / PUT / GET (byte-identical) / DELETE
round-trip succeeds. GOWORK=off go build ./s3/... = 0; s3api bucket registry/metadata
tests pass. Keeps the fail-open intent; only bounds the wait.

s3/command/s3.go

@@ -296,7 +296,13 @@ func (s3opt *S3Options) startS3Server() bool {
 
 	for {
 		err := pb.WithOneOfGrpcFilerClients(false, filerAddresses, grpcDialOption, func(client filer_pb.HanzoFilerClient) error {
-			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+			// Bound each attempt so this required-config read cannot block forever
+			// when a filer accepts TCP (kube endpoints on a cold/co-restart) but is
+			// not yet answering RPCs. On deadline the enclosing for-loop sleeps and
+			// retries — the intended behavior, which an unbounded ctx defeats.
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer cancel()
+			resp, err := client.GetFilerConfiguration(ctx, &filer_pb.GetFilerConfigurationRequest{})
 			if err != nil {
 				return fmt.Errorf("get filer configuration: %v", err)
 			}

s3/s3api/bucket_metadata.go

@@ -6,6 +6,7 @@ import (
 	"math"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/hanzoai/s3/s3/glog"
@@ -60,17 +61,23 @@ func NewBucketRegistry(s3a *S3ApiServer) *BucketRegistry {
 		notFound:      make(map[string]struct{}),
 		s3a:           s3a,
 	}
-	err := br.init()
-	if err != nil {
-		glog.Fatal("init bucket registry failed", err)
-		return nil
+	// Fail-open: a filer that is unresponsive at gateway startup must not crash
+	// the S3 gateway (glog.Fatal -> CrashLoopBackOff). init() only warms the
+	// metadata cache; the registry populates lazily via GetBucketMetadata ->
+	// LoadBucketMetadataFromFiler, so a cold start stays correct.
+	if err := br.init(); err != nil {
+		glog.Warningf("bucket registry warm-up skipped (filer unavailable at startup, will warm lazily): %v", err)
 	}
 	return br
 }
 
 func (r *BucketRegistry) init() error {
 	var bucketCount int
-	err := filer_pb.List(context.Background(), r.s3a, r.s3a.option.BucketsPath, "", func(entry *filer_pb.Entry, isLast bool) error {
+	// Bound the warm-up list so an unresponsive filer at startup cannot block
+	// gateway startup indefinitely; on deadline we fail open (see NewBucketRegistry).
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	err := filer_pb.List(ctx, r.s3a, r.s3a.option.BucketsPath, "", func(entry *filer_pb.Entry, isLast bool) error {
 		if entry != nil && strings.HasPrefix(entry.Name, ".") {
 			return nil
 		}

s3/s3api/s3api_circuit_breaker.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"sync"
 	"sync/atomic"
+	"time"
 
 	"github.com/gorilla/mux"
 	"github.com/hanzoai/s3/s3/filer"
@@ -43,7 +44,13 @@ func NewCircuitBreaker(option *S3ApiServerOption) *CircuitBreaker {
 
 	// Use WithOneOfGrpcFilerClients to support multiple filers with failover
 	err := pb.WithOneOfGrpcFilerClients(false, option.Filers, option.GrpcDialOption, func(client filer_pb.HanzoFilerClient) error {
-		content, err := filer.ReadInsideFiler(context.Background(), client, s3_constants.CircuitBreakerConfigDir, s3_constants.CircuitBreakerConfigFile)
+		// Bound the startup read: a filer whose port is dialable (TCP accepted by
+		// kube endpoints on a cold/co-restart) but not yet answering RPCs must not
+		// block gateway startup indefinitely. On deadline the fail-open path below
+		// (circuit breaker disabled) runs and :8333 binds.
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		content, err := filer.ReadInsideFiler(ctx, client, s3_constants.CircuitBreakerConfigDir, s3_constants.CircuitBreakerConfigFile)
 		if errors.Is(err, filer_pb.ErrNotFound) {
 			return nil
 		}