chains: size-chunk GetContext responses so a behind validator resyncs under heavy load (1/3)

Benchmark-proven live mainnet bug (250-trader DEX load): a validator that falls behind cannot resync the C-Chain. GetContext (the catch-up "context response", wire = Ancestors) bounded the response ONLY by block COUNT (maxContextBlocks=256). Under heavy DEX load 256 blocks summed to 3.4-5.7 MB, exceeding the 2 MB peer message cap (the zstd compressor refuses uncompressed input above constants.DefaultMaxMessageSize to prevent a decompression bomb), so msgCreator.Ancestors FAILED to build and the behind validator received NOTHING — permanently stuck while the tip advanced (luxd-3 stuck at 256 while tip went 293→300, looping empty-block builds). Fix (piece 1/3 of the layered design — defense in depth): GetContext now ALSO bounds the response by serialized SIZE. It stops adding blocks before the accumulated payload would exceed byteBudget (cap − 128 KiB envelope margin), but ALWAYS includes at least one block so a behind node makes progress every round; the requester re-requests the remaining gap (context fetch is already a multi-round oldest-first fill). A single block that alone exceeds the budget is still served (best-effort) so the walk never deadlocks — the forthcoming trust-tiered validator cap (pieces 2/3) gives such a block the send headroom; a stranger's tight cap correctly rejects it. Tests (chains/context_chunk_test.go): TestGetContext_ChunksBySize_FitsUnderCap (100×150 KiB chain → 12 blocks / 1.84 MB, under budget, vs ~15 MB packed by count — fail-on-old); TestGetContext_SingleOversizeBlock_StillServed (one oversize block still served, no deadlock). Pieces 2/3 to follow: trust-tiered message cap (validator peers get headroom, strangers keep 2 MB) + confirm the inbound throttler/benchlist penalizes oversized-message senders. Co-authored-by: Hanzo Dev <dev@hanzo.ai>
2026-07-27 03:39:39 +00:00 · 2026-07-09 23:30:40 -07:00
parent 83c1a83765
commit a93e5cdd9d
2 changed files with 171 additions and 1 deletions
@@ -0,0 +1,139 @@
+// Copyright (C) 2019-2026, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+// context_chunk_test.go — the heavy-block self-heal fix: GetContext must bound its Ancestors
+// response by SERIALIZED SIZE (not just block COUNT) so it stays under the peer message cap.
+//
+// Benchmark-proven live bug: under 250-trader DEX load, heavy blocks made a 256-block context
+// response sum to 3.4-5.7 MB > the 2 MB compressor cap; msgCreator.Ancestors FAILED to build,
+// so a behind validator received NOTHING and could never resync (permanently stuck while the tip
+// advanced). This pins the fix: the response is chunked to fit the budget, always serving at
+// least one block so the behind node makes progress every round.
+
+package chains
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	consensuschain "github.com/luxfi/consensus/engine/chain"
+	consensusblock "github.com/luxfi/consensus/engine/chain/block"
+	"github.com/luxfi/constants"
+	"github.com/luxfi/ids"
+	"github.com/luxfi/log"
+	"github.com/luxfi/node/message"
+)
+
+// heavyBlock is a consensuschain.Block of a controlled byte size; only the methods GetContext
+// touches (ID/Parent/Bytes) are overridden — the rest of the interface is embedded and unused.
+type heavyBlock struct {
+	consensusblock.Block
+	id, parent ids.ID
+	bytes      []byte
+}
+
+func (b *heavyBlock) ID() ids.ID     { return b.id }
+func (b *heavyBlock) Parent() ids.ID { return b.parent }
+func (b *heavyBlock) Bytes() []byte  { return b.bytes }
+
+// sizeStubVM serves heavyBlocks by id (only GetBlock is called by GetContext).
+type sizeStubVM struct {
+	consensuschain.BlockBuilder
+	blocks map[ids.ID]consensusblock.Block
+}
+
+func (v *sizeStubVM) GetBlock(_ context.Context, id ids.ID) (consensusblock.Block, error) {
+	b, ok := v.blocks[id]
+	if !ok {
+		return nil, errors.New("not found")
+	}
+	return b, nil
+}
+
+// sizeRecMsg records the containers GetContext hands to Ancestors, so the test can measure the
+// assembled response size (the input the real zstd compressor would reject above the cap).
+type sizeRecMsg struct {
+	message.OutboundMsgBuilder
+	containers [][]byte
+}
+
+func (m *sizeRecMsg) Ancestors(_ ids.ID, _ uint32, containers [][]byte) (message.OutboundMessage, error) {
+	m.containers = containers
+	return nil, nil
+}
+
+func TestGetContext_ChunksBySize_FitsUnderCap(t *testing.T) {
+	// A 100-block chain of ~150 KiB blocks. Packed by COUNT alone (up to maxContextBlocks=256,
+	// i.e. all 100), the response is ~15 MB — 7x over the 2 MB cap, so the old handler's message
+	// failed to build. Bounded by SIZE, it serves only as many as fit.
+	const nBlocks = 100
+	const blkSize = 150 * 1024
+
+	vm := &sizeStubVM{blocks: map[ids.ID]consensusblock.Block{}}
+	parent := ids.Empty
+	var tip ids.ID
+	for i := 0; i < nBlocks; i++ {
+		id := ids.GenerateTestID()
+		vm.blocks[id] = &heavyBlock{id: id, parent: parent, bytes: make([]byte, blkSize)}
+		parent = id
+		tip = id
+	}
+
+	msg := &sizeRecMsg{}
+	bh := &blockHandler{
+		logger:           log.NewNoOpLogger(),
+		vm:               vm,
+		msgCreator:       msg,
+		net:              &redStubNet{},
+		chainID:          ids.GenerateTestID(),
+		networkID:        ids.GenerateTestID(),
+		maxContextBlocks: 256,
+	}
+
+	if err := bh.GetContext(context.Background(), ids.GenerateTestNodeID(), 1, time.Now().Add(time.Second), tip); err != nil {
+		t.Fatalf("GetContext: %v", err)
+	}
+
+	total := 0
+	for _, c := range msg.containers {
+		total += len(c)
+	}
+	budget := constants.DefaultMaxMessageSize - 128*1024 // the handler's byteBudget
+
+	if len(msg.containers) == 0 {
+		t.Fatal("must serve at least one block (a behind node must always make progress)")
+	}
+	if total > budget {
+		t.Fatalf("context response %d bytes exceeds budget %d — the real Ancestors compressor would REJECT it "+
+			"(cap %d), stranding a behind validator (the live bug)", total, budget, constants.DefaultMaxMessageSize)
+	}
+	if len(msg.containers) >= nBlocks {
+		t.Fatalf("expected SIZE truncation (fewer than %d blocks), got %d — response not chunked", nBlocks, len(msg.containers))
+	}
+	t.Logf("size-chunked: %d blocks, %d payload bytes (budget %d, cap %d) — fits, so the compressor accepts it",
+		len(msg.containers), total, budget, constants.DefaultMaxMessageSize)
+}
+
+// A single heavy block is ALWAYS served even if it alone exceeds the budget — the walk must never
+// deadlock (the trust-tiered validator cap gives such a block the send headroom; a stranger's
+// tight cap correctly rejects it downstream).
+func TestGetContext_SingleOversizeBlock_StillServed(t *testing.T) {
+	oversize := constants.DefaultMaxMessageSize + 1<<20 // > the cap on its own
+	id := ids.GenerateTestID()
+	vm := &sizeStubVM{blocks: map[ids.ID]consensusblock.Block{
+		id: &heavyBlock{id: id, parent: ids.Empty, bytes: make([]byte, oversize)},
+	}}
+	msg := &sizeRecMsg{}
+	bh := &blockHandler{
+		logger: log.NewNoOpLogger(), vm: vm, msgCreator: msg, net: &redStubNet{},
+		chainID: ids.GenerateTestID(), networkID: ids.GenerateTestID(), maxContextBlocks: 256,
+	}
+	if err := bh.GetContext(context.Background(), ids.GenerateTestNodeID(), 1, time.Now().Add(time.Second), id); err != nil {
+		t.Fatalf("GetContext: %v", err)
+	}
+	if len(msg.containers) != 1 {
+		t.Fatalf("a single (even oversize) block must be served so the walk never deadlocks, got %d blocks", len(msg.containers))
+	}
+}
@@ -3260,10 +3260,31 @@ func (b *blockHandler) GetContext(ctx context.Context, nodeID ids.NodeID, reques
 		log.Stringer("containerID", containerID),
 		log.Uint32("requestID", requestID))

-	// Collect context blocks (walk parent chain)
+	// Collect context blocks (walk parent chain).
+	//
+	// SIZE-CHUNKING (the heavy-DEX-block self-heal fix). The response is the Ancestors
+	// wire message, whose UNCOMPRESSED size the peer compressor refuses above the message
+	// cap (constants.DefaultMaxMessageSize; the zstd compressor bounds its input to prevent a
+	// decompression bomb). The old loop bounded ONLY by COUNT (maxContextBlocks=256), so under
+	// heavy DEX load 256 blocks summed to 3.4-5.7 MB > the 2 MB cap, msgCreator.Ancestors FAILED
+	// to build, and the behind validator got NOTHING — it could never resync and fell
+	// permanently behind (the benchmark-proven stall). We now ALSO bound by serialized size:
+	// stop before the accumulated payload would exceed the budget, but ALWAYS include at least
+	// one block so a behind node makes progress every round; the requester re-requests for the
+	// remaining gap (GetAncestors/context is already a multi-round, oldest-first fill). A single
+	// block that alone exceeds the budget is still served (best-effort) so the walk never
+	// deadlocks — the trust-tiered validator cap (peer layer) gives such a block the headroom to
+	// actually send; for a stranger it will be refused by the tight cap, which is correct.
 	var containers [][]byte
 	currentID := containerID

+	// Leave margin under the cap for the p2p envelope (chainID, requestID, per-container length
+	// prefixes, compression framing) so the assembled message stays comfortably below the limit.
+	const contextResponseMargin = 128 * 1024 // 128 KiB
+	byteBudget := constants.DefaultMaxMessageSize - contextResponseMargin
+	accumulated := 0
+	truncatedForSize := false
+
 	for i := 0; i < b.maxContextBlocks; i++ {
 		// First check pending blocks (for recently proposed but not yet accepted blocks)
 		// This is critical: when we propose a block and send PullQuery, other validators
@@ -3294,6 +3315,14 @@ func (b *blockHandler) GetContext(ctx context.Context, nodeID ids.NodeID, reques
 			certBytes, _ = b.engine.CertForBlock(blk.ID())
 		}
 		entry := encodeCatchupEntry(blk.Bytes(), certBytes)
+
+		// SIZE GATE: stop before exceeding the budget — but never drop the FIRST block, so a
+		// behind node always receives at least one block per request and cannot deadlock.
+		if len(containers) > 0 && accumulated+len(entry) > byteBudget {
+			truncatedForSize = true
+			break
+		}
+		accumulated += len(entry)
 		containers = append([][]byte{entry}, containers...)

 		// Get parent ID for next iteration
@@ -3329,6 +3358,8 @@ func (b *blockHandler) GetContext(ctx context.Context, nodeID ids.NodeID, reques
 		log.Stringer("to", nodeID),
 		log.Stringer("containerID", containerID),
 		log.Int("numBlocks", len(containers)),
+		log.Int("payloadBytes", accumulated),
+		log.Bool("truncatedForSize", truncatedForSize),
 		log.Int("sentTo", sentTo.Len()))

 	return nil