node v1.36.0 — Nova/Quasar export-frontier bridge; consensus v1.36.0

Wires the consensus EXPORT (Quasar, two-thirds-stake) frontier into the C-Chain VM so
the EVM finalized/safe tags and the warp cross-chain gate resolve to the Quasar tip.
Consensus v1.35.38 -> v1.36.0.
This commit is contained in:
zeekay
2026-07-10 09:44:14 -07:00
12 changed files with 933 additions and 214 deletions
+1 -1
View File
@@ -257,7 +257,7 @@ RUN . ./build_env.sh && \
# failed ValidateState, and BRICKED the node. Proven on-node: real swap → kill -9 →
# clean reboot, state intact. v1.99.40 = v1.99.39 + deps to latest. consensus v1.25.21 =
# stake-weighted alpha-of-K quorum finality + per-height single-finalize + epoch-bound certs.
ARG EVM_VERSION=v1.104.3
ARG EVM_VERSION=v1.104.7
ARG EVM_VM_ID=mgj786NP7uDwBCcq6YwThhaN8FLyybkCa4zBWTQbNgmK6k9A6
# the pinned evm go.mod may pin a dead luxfi/upgrade pseudo-version
# (v1.0.1-0.20260603055252-f51810805436 — commit pruned from origin). Heal it to
+139
View File
@@ -0,0 +1,139 @@
// Copyright (C) 2019-2026, Lux Industries, Inc. All rights reserved.
// See the file LICENSE for licensing terms.
// context_chunk_test.go — the heavy-block self-heal fix: GetContext must bound its Ancestors
// response by SERIALIZED SIZE (not just block COUNT) so it stays under the peer message cap.
//
// Benchmark-proven live bug: under 250-trader DEX load, heavy blocks made a 256-block context
// response sum to 3.4-5.7 MB > the 2 MB compressor cap; msgCreator.Ancestors FAILED to build,
// so a behind validator received NOTHING and could never resync (permanently stuck while the tip
// advanced). This pins the fix: the response is chunked to fit the budget, always serving at
// least one block so the behind node makes progress every round.
package chains
import (
"context"
"errors"
"testing"
"time"
consensuschain "github.com/luxfi/consensus/engine/chain"
consensusblock "github.com/luxfi/consensus/engine/chain/block"
"github.com/luxfi/constants"
"github.com/luxfi/ids"
"github.com/luxfi/log"
"github.com/luxfi/node/message"
)
// heavyBlock is a consensuschain.Block of a controlled byte size; only the methods GetContext
// touches (ID/Parent/Bytes) are overridden — the rest of the interface is embedded and unused.
type heavyBlock struct {
consensusblock.Block
id, parent ids.ID
bytes []byte
}
func (b *heavyBlock) ID() ids.ID { return b.id }
func (b *heavyBlock) Parent() ids.ID { return b.parent }
func (b *heavyBlock) Bytes() []byte { return b.bytes }
// sizeStubVM serves heavyBlocks by id (only GetBlock is called by GetContext).
type sizeStubVM struct {
consensuschain.BlockBuilder
blocks map[ids.ID]consensusblock.Block
}
func (v *sizeStubVM) GetBlock(_ context.Context, id ids.ID) (consensusblock.Block, error) {
b, ok := v.blocks[id]
if !ok {
return nil, errors.New("not found")
}
return b, nil
}
// sizeRecMsg records the containers GetContext hands to Ancestors, so the test can measure the
// assembled response size (the input the real zstd compressor would reject above the cap).
type sizeRecMsg struct {
message.OutboundMsgBuilder
containers [][]byte
}
func (m *sizeRecMsg) Ancestors(_ ids.ID, _ uint32, containers [][]byte) (message.OutboundMessage, error) {
m.containers = containers
return nil, nil
}
func TestGetContext_ChunksBySize_FitsUnderCap(t *testing.T) {
// A 100-block chain of ~150 KiB blocks. Packed by COUNT alone (up to maxContextBlocks=256,
// i.e. all 100), the response is ~15 MB — 7x over the 2 MB cap, so the old handler's message
// failed to build. Bounded by SIZE, it serves only as many as fit.
const nBlocks = 100
const blkSize = 150 * 1024
vm := &sizeStubVM{blocks: map[ids.ID]consensusblock.Block{}}
parent := ids.Empty
var tip ids.ID
for i := 0; i < nBlocks; i++ {
id := ids.GenerateTestID()
vm.blocks[id] = &heavyBlock{id: id, parent: parent, bytes: make([]byte, blkSize)}
parent = id
tip = id
}
msg := &sizeRecMsg{}
bh := &blockHandler{
logger: log.NewNoOpLogger(),
vm: vm,
msgCreator: msg,
net: &redStubNet{},
chainID: ids.GenerateTestID(),
networkID: ids.GenerateTestID(),
maxContextBlocks: 256,
}
if err := bh.GetContext(context.Background(), ids.GenerateTestNodeID(), 1, time.Now().Add(time.Second), tip); err != nil {
t.Fatalf("GetContext: %v", err)
}
total := 0
for _, c := range msg.containers {
total += len(c)
}
budget := constants.DefaultMaxMessageSize - 128*1024 // the handler's byteBudget
if len(msg.containers) == 0 {
t.Fatal("must serve at least one block (a behind node must always make progress)")
}
if total > budget {
t.Fatalf("context response %d bytes exceeds budget %d — the real Ancestors compressor would REJECT it "+
"(cap %d), stranding a behind validator (the live bug)", total, budget, constants.DefaultMaxMessageSize)
}
if len(msg.containers) >= nBlocks {
t.Fatalf("expected SIZE truncation (fewer than %d blocks), got %d — response not chunked", nBlocks, len(msg.containers))
}
t.Logf("size-chunked: %d blocks, %d payload bytes (budget %d, cap %d) — fits, so the compressor accepts it",
len(msg.containers), total, budget, constants.DefaultMaxMessageSize)
}
// A single heavy block is ALWAYS served even if it alone exceeds the budget — the walk must never
// deadlock (the trust-tiered validator cap gives such a block the send headroom; a stranger's
// tight cap correctly rejects it downstream).
func TestGetContext_SingleOversizeBlock_StillServed(t *testing.T) {
oversize := constants.DefaultMaxMessageSize + 1<<20 // > the cap on its own
id := ids.GenerateTestID()
vm := &sizeStubVM{blocks: map[ids.ID]consensusblock.Block{
id: &heavyBlock{id: id, parent: ids.Empty, bytes: make([]byte, oversize)},
}}
msg := &sizeRecMsg{}
bh := &blockHandler{
logger: log.NewNoOpLogger(), vm: vm, msgCreator: msg, net: &redStubNet{},
chainID: ids.GenerateTestID(), networkID: ids.GenerateTestID(), maxContextBlocks: 256,
}
if err := bh.GetContext(context.Background(), ids.GenerateTestNodeID(), 1, time.Now().Add(time.Second), id); err != nil {
t.Fatalf("GetContext: %v", err)
}
if len(msg.containers) != 1 {
t.Fatalf("a single (even oversize) block must be served so the walk never deadlocks, got %d blocks", len(msg.containers))
}
}
+117 -126
View File
@@ -15,7 +15,6 @@ import (
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
gatomic "sync/atomic"
@@ -1163,7 +1162,7 @@ func (m *manager) buildChain(chainParams ChainParameters, sb nets.Net) (*chainIn
vmConfigBytes := m.injectAutominingConfig(chainParams.VMID, chainConfig.Config)
vmConfigBytes = m.injectSecurityProfileConfig(chainParams.VMID, vmConfigBytes)
// CONSENSUS-SAFETY (single-proposer-per-height): re-wrap multi-validator
// linear chains in proposervm so block production follows the Snowman++
// linear chains in proposervm so block production follows the proposervm's
// proposer schedule — exactly ONE validator builds height H, the rest wait
// and vote. Without it every validator's engine calls BuildBlock
// UNCONDITIONALLY at every height off a slightly-different mempool, so two
@@ -1200,19 +1199,10 @@ func (m *manager) buildChain(chainParams ChainParameters, sb nets.Net) (*chainIn
return nil, fmt.Errorf("refusing to start multi-node chain %s with non-BFT consensus params: %w", chainParams.ID, err)
}
}
// Round-scoped view-change (restores liveness under competing siblings + a zero-margin
// quorum — the 415→416 freeze). OPT-IN per deployment via LUX_CONSENSUS_VIEW_CHANGE=true
// so devnet/testnet can enable it without a mainnet default (mainnet is owner-gated). Only
// meaningful on a multi-validator (K>1) chain; K==1 has no competing proposers. The engine
// itself fail-secure HALTS the view-change if the committee fails the 2α−n>f bound, so
// enabling it can never weaken safety — at worst it halts (never forks).
if consensusParams.K > 1 && strings.EqualFold(os.Getenv("LUX_CONSENSUS_VIEW_CHANGE"), "true") {
consensusParams.ViewChange = true
m.Log.Info("round-scoped view-change ENABLED for chain",
log.Stringer("chainID", chainParams.ID),
log.Int("K", consensusParams.K),
log.Int("alpha", consensusParams.AlphaConfidence))
}
// v1.36 "Nova": the round-scoped VIEW-CHANGE (prevote/POL/lock) was DELETED from the
// consensus engine (174af3c31). Nova metastable sampling is the sole decider and the ⅔
// Quasar attestation trails it — there is no view-change to opt into, so the former
// LUX_CONSENSUS_VIEW_CHANGE env gate is gone. Keep the braid dead.
_, innerIsDAGNative := vmTyped.(interface {
Linearize(context.Context, ids.ID, chan<- vm.Message) error
})
@@ -1519,7 +1509,7 @@ func (m *manager) buildChain(chainParams ChainParameters, sb nets.Net) (*chainIn
// the proposervm, whose SignedBlock carries the real P-chain height
// (selectChildPChainHeight = max(GetCurrentHeight, parentH)) and exposes
// PChainHeight() — the SAME value the engine's pChainHeightOf reads. That
// is precisely the Snowman++ mechanism newPChainHeightVM was a stand-in
// is precisely the proposervm mechanism newPChainHeightVM was a stand-in
// for, so stacking both would double-stamp the height. We keep
// newPChainHeightVM only for the unwrapped K>1 chains (P-Chain, X-Chain).
if blockBuilder != nil && !wrapInProposerVM {
@@ -1530,66 +1520,34 @@ func (m *manager) buildChain(chainParams ChainParameters, sb nets.Net) (*chainIn
log.Stringer("networkID", networkID))
}
}
// DEFENSIVE (view-change hygiene): a view-change chain that cannot BROADCAST its
// prevotes/precommits (its gossiper is not a QuorumGossiper) or cannot SIGN them (no
// VoteSigner) would emit votes into the void — every node tallies only its OWN vote, no
// peer ever receives one, α is never reached, and finality silently stalls. Refuse to
// start such a chain LOUDLY rather than freeze in production.
if consensusParams.ViewChange {
if _, ok := netCfg.Gossiper.(consensuschain.QuorumGossiper); !ok {
return nil, fmt.Errorf("refusing to start view-change chain %s: gossiper %T does not implement "+
"QuorumGossiper — prevotes/precommits could not be broadcast and finality would silently stall",
chainParams.ID, netCfg.Gossiper)
// EXPORT-FRONTIER BRIDGE (two-tier consensus, v1.36). VM.Accept now advances the
// local NOVA (bare-majority) accept tip, which is reorgable and MUST NOT be exported.
// Push each EXPORT (Quasar, ⅔-by-stake) frontier advance into the VM so the EVM
// `finalized`/`safe` block tags and the warp cross-chain export gate resolve to the
// Quasar tip, NEVER the Nova tip (the "semantic collapse" the split exists to prevent).
// Interface-gated: only a VM that exposes the export sink participates (the C-Chain
// EVM); other VMs are Nova-only with no export surface. Push into the RAW inner VM
// (vmTyped) — the eth backend / warp backend live there, not on the proposervm wrapper.
if qvm, ok := vmTyped.(interface{ SetLastQuasarFinalized(uint64) }); ok {
netCfg.QuasarObserver = func(_ ids.ID, height uint64) {
qvm.SetLastQuasarFinalized(height)
}
m.Log.Info("wired EXPORT-frontier (quasar) bridge into the VM (finalized/safe + warp gate track ⅔-stake finality)",
log.Stringer("chainID", chainParams.ID))
}
if netCfg.VoteSigner == nil {
return nil, fmt.Errorf("refusing to start view-change chain %s: no VoteSigner wired — this node "+
"could not sign its prevotes/precommits and finality would silently stall", chainParams.ID)
}
}
consensusEngine := consensuschain.NewRuntime(netCfg)
consensusEngine := consensuschain.NewRuntime(netCfg)
// STALE-LOCK MIGRATION (one-shot, operator-gated). AFTER NewRuntime seeds the durable
// view-change locks and BEFORE Start drives any view, optionally CANONICALIZE (or prune)
// pre-fix OUTER-wrapper view-change lock metadata ABOVE the decided floor — the block-1082880
// durable split-lock: locks stored by the proposervm outer id that the canonical=inner roll
// never re-canonicalized, so honest nodes stay locked on stale aliases, split below α, and
// never form a POL. All finalized + vote-guard state AT OR BELOW the floor is preserved
// verbatim and the floor is never lowered — a narrow metadata migration, not a state reset.
// LUX_CONSENSUS_MIGRATE_STALE_LOCKS: "inspect" prints the plan and mutates NOTHING (the
// per-pod audit gate); "apply:<floor>" performs the idempotent, safety-gated repair, where
// <floor> is the decided floor the operator OBSERVED via inspect (1082879 for the mainnet
// one-shot). The explicit target makes the apply SELF-DISARMING: once the chain recovers
// and the floor advances past the target, a lingering env no-ops instead of degrading into
// an every-boot prune of genuine crash locks (the HIGH-1 regression). A bare "apply" is
// REFUSED. Only for a K>1 view-change chain (the only chains that carry these locks).
if consensusParams.K > 1 && consensusParams.ViewChange {
env := strings.ToLower(os.Getenv("LUX_CONSENSUS_MIGRATE_STALE_LOCKS"))
switch {
case env == "inspect":
logStaleLockReport(m.Log, chainParams.ID, "inspect (no write)", consensusEngine.InspectLocks(context.Background()))
case env == "apply" || strings.HasPrefix(env, "apply:"):
target, tErr := strconv.ParseUint(strings.TrimPrefix(env, "apply:"), 10, 64)
if env == "apply" || tErr != nil {
m.Log.Error("stale-lock migration REFUSED: apply requires an explicit floor target — "+
"run inspect, read decidedFloor, then set LUX_CONSENSUS_MIGRATE_STALE_LOCKS=apply:<decidedFloor>",
log.Stringer("chainID", chainParams.ID), log.String("env", env))
break
}
rep, mErr := consensusEngine.MigrateStaleLocks(context.Background(), target)
logStaleLockReport(m.Log, chainParams.ID, "apply", rep)
if mErr != nil {
return nil, fmt.Errorf("stale-lock migration for chain %s failed (fail-closed, nothing changed): %w", chainParams.ID, mErr)
}
if rep.Stop {
m.Log.Error("stale-lock migration STOPPED — manual recovery required (no write performed)",
log.Stringer("chainID", chainParams.ID), log.String("reason", rep.StopReason))
}
if rep.Skipped {
m.Log.Warn("stale-lock migration self-disarmed (no write) — unset LUX_CONSENSUS_MIGRATE_STALE_LOCKS",
log.Stringer("chainID", chainParams.ID), log.String("reason", rep.SkipReason))
// Re-seed the consensus EXPORT frontier from the VM's DURABLE Quasar height so
// GetQuasarTip / QuasarHeight do not regress on restart (the in-memory frontier resets
// to (Empty,0) until a fresh ⅔-stake cert re-forms; the VM persisted the export height).
// Advance-only; the observer above refines it as new certs land this session.
if qvm, ok := vmTyped.(interface{ LastQuasarHeight() uint64 }); ok {
if h := qvm.LastQuasarHeight(); h > 0 {
consensusEngine.SyncQuasarFrontier(ids.Empty, h)
m.Log.Info("re-seeded consensus export (quasar) frontier from the VM's durable height on boot",
log.Stringer("chainID", chainParams.ID), log.Uint64("quasarHeight", h))
}
}
}
// Start the consensus engine with a LIFETIME context (not a timeout):
// engine.Start parents all four long-running loops (poll, vote, pipeline,
@@ -3186,14 +3144,45 @@ func (b *blockHandler) requestContext(ctx context.Context, nodeID ids.NodeID, bl
return
}
nodeSet := set.NewSet[ids.NodeID](1)
nodeSet.Add(nodeID)
// PEER SELECTION (defect #3). The consensus layer signals "I hold a VERIFIED cert
// for a block I don't track — fetch it" by passing ids.EmptyNodeID (topology.go:
// requestCatchup(cert.Position.BlockID, ids.EmptyNodeID)); picking a real peer is
// the node layer's job. The prior code blindly Add(EmptyNodeID) + Send, so
// GetAncestors went to ZERO peers (the "sentTo=0" spam on the frozen fleet) and the
// certified-but-untracked block was NEVER fetched — the node saw the cert, could not
// finalize, and never recovered. When nodeID is Empty, sample real connected peers
// that track this chain's network (the SAME selection pollFrontierOnce uses); a valid
// cert already gated this request, so asking any network peer is sound (the served
// gap is cert-verified on accept).
nodeSet := set.NewSet[ids.NodeID](frontierPollSample)
if nodeID == ids.EmptyNodeID {
for _, p := range b.net.PeerInfo(nil) {
if p.TrackedChains.Contains(b.networkID) {
nodeSet.Add(p.ID)
if nodeSet.Len() >= frontierPollSample {
break
}
}
}
} else {
nodeSet.Add(nodeID)
}
if nodeSet.Len() == 0 {
// No reachable peer to serve the block. Release the pending slot so a later tick
// (frontier poll → AcceptedFrontier, or a re-gossiped cert) can retry — otherwise
// the block stays pinned unrequestable until the TTL reap.
b.contextRequestMu.Lock()
delete(b.pendingContext, blockID)
b.contextRequestMu.Unlock()
return
}
sentTo := b.net.Send(msg, nodeSet, b.networkID, 0)
b.logger.Info("requested context for missing prerequisites",
log.Stringer("from", nodeID),
log.Stringer("blockID", blockID),
log.Uint32("requestID", requestID),
log.Int("asked", nodeSet.Len()),
log.Int("sentTo", sentTo.Len()))
}
@@ -3283,10 +3272,31 @@ func (b *blockHandler) GetContext(ctx context.Context, nodeID ids.NodeID, reques
log.Stringer("containerID", containerID),
log.Uint32("requestID", requestID))
// Collect context blocks (walk parent chain)
// Collect context blocks (walk parent chain).
//
// SIZE-CHUNKING (the heavy-DEX-block self-heal fix). The response is the Ancestors
// wire message, whose UNCOMPRESSED size the peer compressor refuses above the message
// cap (constants.DefaultMaxMessageSize; the zstd compressor bounds its input to prevent a
// decompression bomb). The old loop bounded ONLY by COUNT (maxContextBlocks=256), so under
// heavy DEX load 256 blocks summed to 3.4-5.7 MB > the 2 MB cap, msgCreator.Ancestors FAILED
// to build, and the behind validator got NOTHING — it could never resync and fell
// permanently behind (the benchmark-proven stall). We now ALSO bound by serialized size:
// stop before the accumulated payload would exceed the budget, but ALWAYS include at least
// one block so a behind node makes progress every round; the requester re-requests for the
// remaining gap (GetAncestors/context is already a multi-round, oldest-first fill). A single
// block that alone exceeds the budget is still served (best-effort) so the walk never
// deadlocks — the trust-tiered validator cap (peer layer) gives such a block the headroom to
// actually send; for a stranger it will be refused by the tight cap, which is correct.
var containers [][]byte
currentID := containerID
// Leave margin under the cap for the p2p envelope (chainID, requestID, per-container length
// prefixes, compression framing) so the assembled message stays comfortably below the limit.
const contextResponseMargin = 128 * 1024 // 128 KiB
byteBudget := constants.DefaultMaxMessageSize - contextResponseMargin
accumulated := 0
truncatedForSize := false
for i := 0; i < b.maxContextBlocks; i++ {
// First check pending blocks (for recently proposed but not yet accepted blocks)
// This is critical: when we propose a block and send PullQuery, other validators
@@ -3317,6 +3327,14 @@ func (b *blockHandler) GetContext(ctx context.Context, nodeID ids.NodeID, reques
certBytes, _ = b.engine.CertForBlock(blk.ID())
}
entry := encodeCatchupEntry(blk.Bytes(), certBytes)
// SIZE GATE: stop before exceeding the budget — but never drop the FIRST block, so a
// behind node always receives at least one block per request and cannot deadlock.
if len(containers) > 0 && accumulated+len(entry) > byteBudget {
truncatedForSize = true
break
}
accumulated += len(entry)
containers = append([][]byte{entry}, containers...)
// Get parent ID for next iteration
@@ -3352,6 +3370,8 @@ func (b *blockHandler) GetContext(ctx context.Context, nodeID ids.NodeID, reques
log.Stringer("to", nodeID),
log.Stringer("containerID", containerID),
log.Int("numBlocks", len(containers)),
log.Int("payloadBytes", accumulated),
log.Bool("truncatedForSize", truncatedForSize),
log.Int("sentTo", sentTo.Len()))
return nil
@@ -3489,15 +3509,28 @@ func (b *blockHandler) AcceptedFrontier(ctx context.Context, nodeID ids.NodeID,
if b.deliverBootstrapFrontier(nodeID, containerID) {
return nil
}
if _, err := b.vm.GetBlock(ctx, containerID); err == nil {
return nil // we already have the peer's tip — not behind
}
if b.engine != nil {
if blk, err := b.vm.GetBlock(ctx, containerID); err == nil {
// HAVE-BLOCK, LACK-FINALIZATION (defect #5). Holding the peer's tip block does NOT
// mean we are caught up. A verified-but-unfinalized block (we voted for it, but the
// α-of-K cert never reached us) leaves us behind on the CERT, not the block bytes.
// The prior check returned "not behind" on GetBlock success, so such a node NEVER
// fetched the missing cert and sat stuck at its unfinalized height forever (the exact
// "verified 288 but no cert" condition). Only "have the block AND it is finalized
// here" is truly not-behind; otherwise fall through to fetch the cert-carrying gap so
// AcceptCatchupBlock can finalize it on its verified cert (no re-vote).
if b.engine == nil {
return nil
}
if fin, ok := b.engine.FinalizedBlockAtHeight(blk.Height()); ok && fin == containerID {
return nil // have the block AND finalized it — truly not behind
}
// have the block but not finalized here → behind on the cert → fetch below
} else if b.engine != nil {
if _, found := b.engine.GetPendingBlock(containerID); found {
return nil // already tracked
return nil // already tracked (pending) — the live path is handling it
}
}
b.requestContext(ctx, nodeID, containerID) // behind → fetch the gap
b.requestContext(ctx, nodeID, containerID) // behind (missing block OR its cert) → fetch the gap
return nil
}
@@ -3883,10 +3916,6 @@ func (b *blockHandler) Gossip(ctx context.Context, nodeID ids.NodeID, msg []byte
b.engine.HandleIncomingVote(blockID, payload)
case quorumKindCert:
b.engine.HandleIncomingCert(payload)
case quorumKindPrevote:
// Round-scoped view-change prevote: the engine decodes+verifies
// (height,round,canonical,sig) from the payload and tallies it toward a POL.
b.engine.HandleIncomingPrevote(payload)
}
return nil
}
@@ -4132,30 +4161,6 @@ func (n *noopWarpSender) SendGossip(ctx context.Context, config warp.SendConfig,
return nil
}
// logStaleLockReport renders a stale-lock migration plan as an auditable per-height trace table in
// the node log (one line per lock above the decided floor: the persisted outer id, the inner
// canonical it resolves to, the lock round, whether a cert binds the height, and the planned
// disposition in the Reason). Used by both the read-only inspect mode and the apply mode so the
// operator sees the identical plan before and after a write.
func logStaleLockReport(logger log.Logger, chainID ids.ID, mode string, rep consensuschain.LockMigrationReport) {
logger.Warn("stale-lock migration report",
log.Stringer("chainID", chainID), log.String("mode", mode),
log.Uint64("decidedFloor", rep.DecidedFloor), log.Uint64("finalizedThrough", rep.FinalizedThrough),
log.Int("locksAboveFloor", len(rep.Entries)), log.Bool("changed", rep.Changed),
log.Bool("stop", rep.Stop), log.String("stopReason", rep.StopReason))
for _, e := range rep.Entries {
logger.Warn("stale-lock entry",
log.Stringer("chainID", chainID),
log.Uint64("height", e.Height),
log.Stringer("lockOuter", e.LockOuter),
log.Stringer("lockInner", e.LockCanon),
log.Uint32("lockRound", e.LockRound),
log.Bool("hasRound", e.HasRound),
log.Bool("certAt", e.CertAt),
log.String("plan", e.Reason))
}
}
// networkGossiper implements consensuschain.Gossiper for Lux consensus integration.
// It adapts the node's network layer to the minimal Gossiper interface used by
// the integrated consensus engine.
@@ -4225,23 +4230,9 @@ func (g *networkGossiper) BroadcastVote(chainID ids.ID, networkID ids.ID, blockI
return g.net.Gossip(msg, nil, g.networkID, -1, 0, 0).Len()
}
// BroadcastPrevote sends this node's signed ROUND-SCOPED view-change prevote (the
// non-binding preference signal) for `canonical` at (height, round) to ALL validators,
// framed in a quorum envelope (kind 3) and decoded by blockHandler.Gossip into
// engine.HandleIncomingPrevote. Prevotes never finalize anything — they drive the POL +
// the lock/unlock rule that lets a competing-sibling split RE-CONVERGE (liveness under a
// down proposer + zero-margin quorum). Only emitted when the chain runs params.ViewChange.
func (g *networkGossiper) BroadcastPrevote(chainID ids.ID, networkID ids.ID, height uint64, round uint32, canonical ids.ID, voteBytes []byte) int {
if g.net == nil || g.msgCreator == nil {
return 0
}
envelope := encodeQuorumGossip(quorumKindPrevote, canonical, voteBytes)
msg, err := g.msgCreator.Gossip(chainID, envelope)
if err != nil {
return 0
}
return g.net.Gossip(msg, nil, g.networkID, -1, 0, 0).Len()
}
// v1.36 "Nova": BroadcastPrevote was DELETED — the round-scoped view-change (prevote/POL/lock)
// it fed no longer exists in the consensus engine (174af3c31). Nova sampling decides; the ⅔
// Quasar attestation (a plain accept-vote, gossiped via BroadcastVote) trails it. Keep the braid dead.
// GossipCert broadcasts an assembled α-of-K finality cert to ALL validators so
// followers finalize blockID on a verifiable proof (HandleIncomingCert), not a
+16 -5
View File
@@ -89,7 +89,7 @@ func selectConsensusParams(sybilProtection bool, networkID uint32) consensusconf
// shouldWrapInProposerVM decides whether a linear chain.ChainVM is wrapped in
// proposervm to enforce single-proposer-per-height block production (the
// Snowman++ window). It is the SINGLE policy gate (the manager calls it once);
// proposer window). It is the SINGLE policy gate (the manager calls it once);
// keeping it a pure function makes the policy unit-testable without standing up
// a whole chain. All three conditions must hold:
//
@@ -267,6 +267,16 @@ func (s *validatorStakeSource) TotalStake(height uint64) uint64 {
return total
}
// ValidatorCount implements consensuschain.StakeSource. The number of DISTINCT
// validators in the set IN FORCE AT height — the round-scoped view-change's BFT
// committee size (it sizes its POL/precommit quorum to bftAlpha over this count,
// NOT the oversized sample K). Read from the SAME height-indexed set as
// Weight/TotalStake so every node computes the identical committee and the
// count-quorum matches the ⅔-by-stake set exactly.
func (s *validatorStakeSource) ValidatorCount(height uint64) int {
return len(validatorSetAtHeight(s.state, s.networkID, height))
}
var _ consensuschain.StakeSource = (*validatorStakeSource)(nil)
// --- validator-set-root source (MEDIUM: epoch binding) -----------------------
@@ -372,9 +382,10 @@ func hashValidatorSet(set map[ids.NodeID]*validators.GetValidatorOutput) ids.ID
var quorumGossipMagic = [4]byte{'L', 'X', 'Q', 0x01}
const (
quorumKindVote byte = 1
quorumKindCert byte = 2
quorumKindPrevote byte = 3
quorumKindVote byte = 1
quorumKindCert byte = 2
// kind 3 (prevote) was DELETED with the v1.36 view-change rip-out (174af3c31); Nova sampling
// decides and the ⅔ Quasar attestation rides quorumKindVote. Do not reuse 3 — keep the braid dead.
)
// ErrNotQuorumGossip signals a payload is not a quorum envelope (so the caller
@@ -400,7 +411,7 @@ func decodeQuorumGossip(data []byte) (kind byte, blockID ids.ID, payload []byte,
kind = data[4]
copy(blockID[:], data[5:5+32])
payload = data[5+32:]
if kind != quorumKindVote && kind != quorumKindCert && kind != quorumKindPrevote {
if kind != quorumKindVote && kind != quorumKindCert {
return 0, ids.Empty, nil, ErrNotQuorumGossip
}
return kind, blockID, payload, nil
@@ -0,0 +1,163 @@
# Postmortem: C-Chain accepted-head state GC eviction (mainnet freeze)
**Status:** Resolved. Mainnet recovered and accepted at 5/5 validators, C-Chain
height 1085412, hash `0xd957eae6cb0bbef37174…`, all validators in agreement,
explorer at tip, treasury and Genesis NFT state verified.
**Severity:** Critical (mainnet C-Chain unable to build blocks; no state loss).
## Accepted permanent invariant
> The accepted C-Chain head's state root must never be GC/pruning eligible —
> across idle windows, duplicate empty-block state roots, cold snapshot/cache
> layers, small state history, and restarts.
>
> accepted head ⇒ accepted head state root is pinned ⇒ GC/pruning cannot evict
> the execution base for H+1.
The specific accepted-head GC eviction failure is **structurally prevented** by
the head-state pin, and production evidence confirms the fleet no longer
exhibits the prior failure signature. (A formal long-idle ritual was not
completed to termination during the incident window; the sign-off rests on the
structural invariant plus production evidence: a head idle for 3h04m was built
on cleanly, multiple 515 minute zero-traffic windows passed with tip state
readable, and zero eviction/materialize canaries appeared fleet-wide after the
fix.)
## Failure mode (exact)
The C-Chain EVM (coreth-lineage, `luxfi/evm`) in pruning mode manages trie
memory with `cappedMemoryTrieWriter` (`core/state_manager.go`):
- Accepted state roots are held in a `tipBuffer` (`BoundedBuffer`) of depth
`state-history` (default **32**); as roots age out of the buffer they are
`Dereference`d.
- Dirty trie nodes are only committed to disk at `commit-interval` boundaries
(default **4096** blocks), with optimistic `Cap` flushes near the boundary.
- The insert-time `triedb.Reference(root, {})` in `writeBlockAndSetHead` is
**refcount-balanced**: it is consumed as the block ages through the
tipBuffer (or via `RejectTrie`). It therefore does not protect an idle head.
On an idle chain, consecutive empty blocks share identical state roots. The
tipBuffer's aging `Dereference` for an old entry then lands on the *live head
root* (duplicate key), dropping its reference count to zero. At any height that
is not a commit boundary the head root has never been persisted, so the next
`Cap`/flush evicts it from the dirty cache. The subsequent `BuildBlock` cannot
open the parent (head) state:
```
failed to materialize parent state for build: … StateAt: missing trie node
<head state root> … is not available, not found
```
and the chain wedges. RPC reads at `latest` fail with the same error (any
`StateAt(root)` caller). Consensus is unaffected — all validators agree on the
head *block*; only the local execution base for H+1 is gone. State is always
deterministically re-derivable from durable blocks, so a restart re-executes
and recovers — **restart is recovery evidence, not a fix**: the head could
still be evicted again in the next idle window.
### Preconditions (all defaults on affected mainnet validators)
- `pruning-enabled: true`
- `state-history: 32`
- `commit-interval: 4096`
- idle or bursty-then-idle traffic (heartbeat pause, low organic flow)
- duplicate empty-block state roots at the tip
- current height not at a commit boundary
Any C-Chain deployment matching these preconditions is exposed on affected
versions — this bug class will recur wherever the EVM runs with pruning, small
state history, long commit intervals, and idle traffic.
### Observed occurrences
1. Mainnet froze at height 1085200 after a ~10 minute heartbeat pause
(2026-07-07). All five validators had the block, none could serve or build
on its state.
2. An earlier fleet-wide variant contributed to the 1082879→1085012 incident
window (mixed with a separate proposervm/consensus issue documented in the
consensus fault-recovery audit).
## Affected / fixed versions
| Component | Affected | Fixed |
|---|---|---|
| `luxfi/evm` (C-Chain plugin) | ≤ v1.104.6 (all pruning-mode deployments; the balanced insert-time Reference in v1.104.3-hotfix was insufficient) | **v1.104.7** |
| `luxfi/node` image | v1.34.14 v1.34.23 (carry affected EVM plugins) | **v1.34.24** (interim), **v1.34.25** (canonical: identical fix, proper semver, clean go.mod) |
Fix commits (`luxfi/evm`, branch `evm-main-bugb`): `9bab20f7a` (pin),
`58b90490c` (non-fatal degradation), `e3781c35c` (vm v1.2.6 parity).
## The fix (structural)
`core/blockchain.go`: a dedicated **unbalanced** GC reference held on the
accepted head's state root — `headStatePinRoot` + `pinAcceptedHead(root)`:
- Transferred head-to-head: `Reference` the new head root first, then
`Dereference` the previous pinned root; exactly one live head pin exists at
all times, on `lastAccepted.Root()`.
- Established in `Accept`, in `SetLastAcceptedBlockDirect`, and on the loaded
head at startup (`loadLastState`), so the invariant holds across restarts
and the restart-then-idle path.
- Skips when the root is unchanged (duplicate empty-block roots keep exactly
one reference) and when state is not yet materialized (bootstrapping /
state-sync; the first `Accept` then establishes it).
- Deliberately **not** balanced against `InsertTrie`/`AcceptTrie`/`RejectTrie`
— its lifetime is "is the accepted head", nothing else.
- Non-fatal on backend error (pathdb `Reference`/`Dereference` are no-ops /
"not supported"): a failed pin degrades to pre-fix behavior with a WARN
rather than wedging Accept or startup.
No archive-mode workaround and no state-sync hack. Disk growth is unchanged
(one extra referenced root).
## Recovery recipe (what actually worked)
1. **Wedged-at-tip (state evicted, DB otherwise consistent):** restart the
node. Boot re-executes from the last committed root and re-materializes the
head state deterministically. Valid as *recovery*; deploy the fixed version
so it cannot recur.
2. **proposervm/EVM height split** (`proposervm finality index … is BEHIND the
inner VM tip`; produced here by crash-churn on affected versions — the
fail-closed guard then correctly refuses to mount): restarts cannot heal a
split. Restore the node's PVC from a `VolumeSnapshot` of a currently
healthy peer. Per-ordinal staking keys are installed by `startup.sh` from
the `luxd-staking` secret, so cross-node volume clones are safe (distinct
NodeIDs).
3. **PVC swaps must happen at StatefulSet `replicas=0`.** A live single-pod
PVC delete/recreate always loses the race to the StatefulSet controller,
which recreates a blank PVC first.
4. If a fleet-consistent EVM rewind is needed instead (no healthy peer):
`evm/cmd/repair-cchain` rewinds the standalone EVM `lastAccepted` to the
proposervm floor; on boot the heightAhead branch self-heals (used in the
1084996 recovery). Zero re-execution; never a re-genesis.
5. Retain evidence snapshots before every destructive step.
## Residual follow-ups (non-blocking; restart/churn liveness, not consensus or state-loss)
1. **Proposer-preference restart loop:** after heavy sibling churn, a node's
proposervm preference can reference a never-persisted outer block; every
`BuildBlock` then fails `not found` in a tight loop and the node's voter
goes mute (observed ~170 err/s). Restart clears it. Fix: fall back to
last-accepted when the preferred parent is not fetchable.
**FIXED (commit `8001bc5179`, branch `ship/node-v1.34.24`, ships in
v1.34.26):** `vms/proposervm/vm.go` `BuildBlock` now builds the child on
last-accepted (always held — committed state) when `vm.preferred` is
unfetchable, instead of hard-erroring; it surfaces the original error only
when last-accepted is itself the unfetchable id. Build-side companion to the
already-shipped defect #1 `SetPreference` validate-before-assign hardening.
Tests: `vms/proposervm/vm_buildblock_fallback_test.go`.
2. **Ancestor-fetch liveness:** the finality guard refuses certs with
"ancestor … is not tracked (behind; fetch and retry)" but the fetch never
fires, so the node loops instead of catching up. Restart clears it. Fix:
actually schedule the ancestor fetch on this path.
## Monitoring (keep active)
- Eviction/materialize canaries: `STATE-MATERIALIZE`, `missing trie node`,
`ACCEPT-BACKSTOP` log lines — expect zero.
- Accepted height/hash equality across all validators.
- Explorer tip parity with chain head.
- `is BEHIND the inner` (heightBehind) occurrences — expect zero.
- Do not treat the heartbeat as a safety mechanism; it is a liveness nicety.
+1 -10
View File
@@ -26,7 +26,7 @@ require (
github.com/huin/goupnp v1.3.0
github.com/jackpal/gateway v1.1.1
github.com/jackpal/go-nat-pmp v1.0.2
github.com/luxfi/consensus v1.35.27
github.com/luxfi/consensus v1.36.0
github.com/luxfi/crypto v1.19.26
github.com/luxfi/database v1.20.4
github.com/luxfi/ids v1.3.1
@@ -178,7 +178,6 @@ require (
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect
github.com/aws/smithy-go v1.24.2 // indirect
github.com/btcsuite/btcd/btcec/v2 v2.5.0 // indirect
github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 // indirect
github.com/btcsuite/btcd/chainhash/v2 v2.0.0 // indirect
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
github.com/colega/zeropool v0.0.0-20230505084239-6fb4a4f75381 // indirect
@@ -192,7 +191,6 @@ require (
github.com/hanzos3/go-sdk v1.0.2 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/luxfi/age v1.5.0 // indirect
github.com/luxfi/bft v0.1.5 // indirect
github.com/luxfi/corona v0.10.3 // indirect
github.com/luxfi/crypto/ipa v1.2.4 // indirect
github.com/luxfi/dkg v0.3.5 // indirect
@@ -263,10 +261,3 @@ require (
)
exclude github.com/ethereum/go-ethereum v1.10.26
// TEMPORARY local-dev build aid for the bootstrap frozen-cache convergence fix.
// FINAL CASCADE (publish step, NOT done here): tag consensus v1.25.36 (the uncommitted
// engine/chain/integration.go FinalizedLedger + FinalizedBlockAtHeight accessors and the
// engine/chain/bootstrap HasAccepted change), bump the require above v1.25.35 v1.25.36,
// then DELETE this replace. The zap client (option b) is node-only and does NOT widen the
// consensus bump.
+6 -58
View File
@@ -64,12 +64,9 @@ github.com/bits-and-blooms/bitset v1.24.4/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6
github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
github.com/btcsuite/btcd v0.22.0-beta.0.20220111032746-97732e52810c/go.mod h1:tjmYdS6MLJ5/s0Fj4DbLgSbDHbEqLJrtnHecBFkdz5M=
github.com/btcsuite/btcd v0.23.5-0.20231215221805-96c9fd8078fd/go.mod h1:nm3Bko6zh6bWP60UxwoT5LzdGJsQJaPo6HjduXq9p6A=
github.com/btcsuite/btcd v0.24.2 h1:aLmxPguqxza+4ag8R1I2nnJjSu2iFn/kqtHTIImswcY=
github.com/btcsuite/btcd v0.24.2/go.mod h1:5C8ChTkl5ejr3WHj8tkQSCmydiMEPB0ZhQhehpq7Dgg=
github.com/btcsuite/btcd/btcec/v2 v2.1.0/go.mod h1:2VzYrv4Gm4apmbVVsSq5bqf1Ec8v56E48Vt0Y/umPgA=
github.com/btcsuite/btcd/btcec/v2 v2.1.3/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE=
github.com/btcsuite/btcd/btcec/v2 v2.3.6 h1:IzlsEr9olcSRKB/n7c4351F3xHKxS2lma+1UFGCYd4E=
github.com/btcsuite/btcd/btcec/v2 v2.3.6/go.mod h1:m22FrOAiuxl/tht9wIqAoGHcbnCCaPWyauO8y2LGGtQ=
github.com/btcsuite/btcd/btcec/v2 v2.5.0 h1:KioMXOWa76b86sTZZOmbzv/ldaQCmB8KFAyn5PbB8E8=
github.com/btcsuite/btcd/btcec/v2 v2.5.0/go.mod h1:+K/MYXcLBtHEQjRbjHuJChuybk4LCgjdjgRwil+e+Kk=
github.com/btcsuite/btcd/btcutil v1.0.0/go.mod h1:Uoxwv0pqYWhD//tfTiipkxNfdhG9UrLwaeswfjfdF0A=
@@ -79,7 +76,6 @@ github.com/btcsuite/btcd/btcutil v1.1.6 h1:zFL2+c3Lb9gEgqKNzowKUPQNb8jV7v5Oaodi/
github.com/btcsuite/btcd/btcutil v1.1.6/go.mod h1:9dFymx8HpuLqBnsPELrImQeTQfKBQqzqGbbV3jK55aE=
github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 h1:59Kx4K6lzOW5w6nFlA0v5+lk/6sjybR934QNHSJZPTQ=
github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
github.com/btcsuite/btcd/chainhash/v2 v2.0.0 h1:PMLlSloHJuEeB80XG9EjpXWNEKAZAMLl6YHZ6YsEuoA=
github.com/btcsuite/btcd/chainhash/v2 v2.0.0/go.mod h1:mKxcZ7oGTXE7IRV+sS9hP4EVBwc/SzfNR+52IsOP9j8=
@@ -303,22 +299,14 @@ github.com/luxfi/accel v1.2.4 h1:5VbIHyEvvfobn2zBiTFODxDw1CeqxCepZOLlvkuf9yQ=
github.com/luxfi/accel v1.2.4/go.mod h1:ISIwAX+ZfsL/S5nsP2JvfldXN6Nc+QzoWf6Jtaq+xsQ=
github.com/luxfi/address v1.0.1 h1:Sc4keyuVzBIvHr7uVeYZf2/WY9YDGUgDi/iiWenj49g=
github.com/luxfi/address v1.0.1/go.mod h1:5j3Eh66v9zvv1GbNdZwt+23krV8JlSDaRzmWZU8ZRM0=
github.com/luxfi/age v1.5.0 h1:zC/Fw/ptZwAXr9nqrxmrcf8752EIl1Lq9RECp9OmCO0=
github.com/luxfi/age v1.5.0 h1:G69HbSV4R3vKEH9B0CulnRaMdSdf4RalMgP8xKmxHeI=
github.com/luxfi/age v1.5.0/go.mod h1:iAYAxgvrXxcy746+Ovh/eWWDuF9teJLNcCSSOX9RYW0=
github.com/luxfi/api v1.0.15 h1:Q5ox3Ompw/AZNMfB9wpHZosrj9C+ZldAEHKtHEotFdY=
github.com/luxfi/api v1.0.15/go.mod h1:Eer59msIXMnOlFncG0XjEGH3TZML0Dd1bUu4GtB7f4Q=
github.com/luxfi/atomic v1.0.0 h1:xUV60MuzRvXngaQ1sM0yVC2v4TRoLlUGkkH7M9PS4yw=
github.com/luxfi/atomic v1.0.0/go.mod h1:0G2mTlQ6TXWHICUHrUUPu1/qAiIyR4gSZ2tva9ci/bI=
github.com/luxfi/bft v0.1.5 h1:5xVLPkog4e5LTgaVlb9pgxA0EWE6tkrKwHPZVRz+RZw=
github.com/luxfi/bft v0.1.5/go.mod h1:5I8Ft8yA69xZlDe3RB0i4MgbqFKLZe65o/sha8JuKvU=
github.com/luxfi/cache v1.2.1 h1:kAzOS55/hmYeNKR+0HAKv4ma48Y6JjkI8UQeqdZ8bfI=
github.com/luxfi/cache v1.2.1/go.mod h1:co7JTxZZHpKT31Yh01LFp5aZOxmoUg157FhBLQdQHVU=
github.com/luxfi/chains v1.4.8 h1:i5QxDfGR922oPGYrBbUo2Qn3tFMpJD9BilNTLFaQscE=
github.com/luxfi/chains v1.4.8/go.mod h1:F/jT9YbC8/yD4WxJu4AvRFbi4NyKY+FHBWrE8M08dgE=
github.com/luxfi/chains v1.7.0 h1:rAPWEChbMNbK4NYkvHGN6HOK6H+U4XrydsekvxLJTfo=
github.com/luxfi/chains v1.7.0/go.mod h1:zDd+CsyZAkrbbEXuMUzrLpuSBRDb2yautoyOjQMhJR8=
github.com/luxfi/chains v1.7.1 h1:ljd7WEngYJGoTO88fXbPRJdpid/AU9X7s0RC6O4SbgE=
github.com/luxfi/chains v1.7.1/go.mod h1:hEJOkAW7vbgDcyZwL1ZenDj3K6k2b8/p9GS/4uNYAVg=
github.com/luxfi/chains v1.7.2 h1:ttpePo7cJ0R2ptA0WvQkumXulvZMW7GFzcAasyurDBQ=
github.com/luxfi/chains v1.7.2/go.mod h1:5YzO/dMsTKHePn7Vu1Bf+MT2hbx3drscjvm6QssUvjg=
github.com/luxfi/codec v1.1.5 h1:KBq8uvYm5Dy+E1heG8WBmqbqu8kstlFyE5ASBBB+C8I=
@@ -327,40 +315,8 @@ github.com/luxfi/compress v0.0.5 h1:4tEUHw5MK1bu5UOjfYCt4OKMiH7yykIgmGPRA/BfJTM=
github.com/luxfi/compress v0.0.5/go.mod h1:Cc1yxD2pfzrvpO32W2GDwLKff+CylHEvzZh2Ko8RSIU=
github.com/luxfi/concurrent v0.0.3 h1:eJyv1fhaC0jMLMw6+QS774cUmp7GK+ouMgvLCqnC7cc=
github.com/luxfi/concurrent v0.0.3/go.mod h1:Aj/FR5NpM0cB2P4Nt3+tz9+dV6V+LUW4HuMgSjwq5hw=
github.com/luxfi/consensus v1.35.5 h1:q6RkE8nOmmWgOS7ePnnLR5wOOlzNMZatr+9drHfOa/I=
github.com/luxfi/consensus v1.35.5/go.mod h1:2ON+tN8hsAqd13DmsGk1Epfz6EQ0s4lv/rmOUsh7Gpk=
github.com/luxfi/consensus v1.35.6 h1:G4KEzlElk6V7Gtc8kK/eKyYqOBDb6jzhQpMSZuBSYl8=
github.com/luxfi/consensus v1.35.6/go.mod h1:2ON+tN8hsAqd13DmsGk1Epfz6EQ0s4lv/rmOUsh7Gpk=
github.com/luxfi/consensus v1.35.7 h1:vB5Pk6uNrniwE6T5xBYFHG3w8UR9dmHaTW4THtcBEt4=
github.com/luxfi/consensus v1.35.7/go.mod h1:2ON+tN8hsAqd13DmsGk1Epfz6EQ0s4lv/rmOUsh7Gpk=
github.com/luxfi/consensus v1.35.8 h1:kYLfs8TjHbJ+YluhKJxZridnrTAqzUskV75IvDdvrlI=
github.com/luxfi/consensus v1.35.8/go.mod h1:2ON+tN8hsAqd13DmsGk1Epfz6EQ0s4lv/rmOUsh7Gpk=
github.com/luxfi/consensus v1.35.11 h1:a1mf9XbdthjIO00tsl/hXnISiD6e2851s3DdsTXjXqs=
github.com/luxfi/consensus v1.35.11/go.mod h1:2ON+tN8hsAqd13DmsGk1Epfz6EQ0s4lv/rmOUsh7Gpk=
github.com/luxfi/consensus v1.35.13 h1:o3Zm9qaCCs1vhvZ75TSLnsLEaqxH72meyAOTqbjURxw=
github.com/luxfi/consensus v1.35.13/go.mod h1:2ON+tN8hsAqd13DmsGk1Epfz6EQ0s4lv/rmOUsh7Gpk=
github.com/luxfi/consensus v1.35.14 h1:HvaQLanNlC1bUwk6xun32pwz/qZ0Ia44lO36DFrrdS0=
github.com/luxfi/consensus v1.35.14/go.mod h1:2ON+tN8hsAqd13DmsGk1Epfz6EQ0s4lv/rmOUsh7Gpk=
github.com/luxfi/consensus v1.35.15 h1:iy10lEjMHhul6f4ioSZ05XbHOOU8pilteuvZu7jMefs=
github.com/luxfi/consensus v1.35.15/go.mod h1:2ON+tN8hsAqd13DmsGk1Epfz6EQ0s4lv/rmOUsh7Gpk=
github.com/luxfi/consensus v1.35.16 h1:5hRLqVy9RQH7ZRApO6kiLwLs+9HACGNNZ4wkjt0XM4I=
github.com/luxfi/consensus v1.35.16/go.mod h1:2ON+tN8hsAqd13DmsGk1Epfz6EQ0s4lv/rmOUsh7Gpk=
github.com/luxfi/consensus v1.35.17 h1:LfR1sVVcA2/DVSjy/5K7zhm+h2vZ/4Kf38JKdunMfLA=
github.com/luxfi/consensus v1.35.17/go.mod h1:2ON+tN8hsAqd13DmsGk1Epfz6EQ0s4lv/rmOUsh7Gpk=
github.com/luxfi/consensus v1.35.20 h1:58N8K4BiOTeaFuwVQCgAFJxvrWdIMOIDG7j+RoFdWMM=
github.com/luxfi/consensus v1.35.20/go.mod h1:hmsGz3CcTrKxvw7/YSmfu8qAtzgyL5zQ3ajpUNcub/4=
github.com/luxfi/consensus v1.35.21 h1:xNN2NbsGEIxJ9ZMNvr70TqXop9bqRvvDWSpPhvRWXm4=
github.com/luxfi/consensus v1.35.21/go.mod h1:hmsGz3CcTrKxvw7/YSmfu8qAtzgyL5zQ3ajpUNcub/4=
github.com/luxfi/consensus v1.35.22 h1:/9Cnbl8LEGEJ1VLu7IDgMoU8mNtbuFkNOEiBHYsvWbk=
github.com/luxfi/consensus v1.35.22/go.mod h1:hmsGz3CcTrKxvw7/YSmfu8qAtzgyL5zQ3ajpUNcub/4=
github.com/luxfi/consensus v1.35.25 h1:scQ+oSEQ5Rvv+7kImnFNDHjRGGPlMVVrSc8s3Rsa/eg=
github.com/luxfi/consensus v1.35.25/go.mod h1:hmsGz3CcTrKxvw7/YSmfu8qAtzgyL5zQ3ajpUNcub/4=
github.com/luxfi/consensus v1.35.26 h1:d51M+XDE2nYS+d3dmDDQK/2OROg8UMHvquISguB3w7Y=
github.com/luxfi/consensus v1.35.26/go.mod h1:hmsGz3CcTrKxvw7/YSmfu8qAtzgyL5zQ3ajpUNcub/4=
github.com/luxfi/consensus v1.35.27 h1:/LotkK3TBFRaspoGwbeLagIVu4/kJsqx9VviU5+fXrE=
github.com/luxfi/consensus v1.35.27/go.mod h1:hmsGz3CcTrKxvw7/YSmfu8qAtzgyL5zQ3ajpUNcub/4=
github.com/luxfi/constants v1.5.8 h1:iNP9AWNUcM4Tps7jYnx49CwtCWAC9mYRxJfGou2za0g=
github.com/luxfi/constants v1.5.8/go.mod h1:Pu5jWHdnUtQRbWC43yTUjU/pbIIKMDOd2a2yroSfo48=
github.com/luxfi/consensus v1.36.0 h1:pW+3OOZ8N3GHZxShdXAZGK8BC2WmouyJR9UGm7XeLHs=
github.com/luxfi/consensus v1.36.0/go.mod h1:hmsGz3CcTrKxvw7/YSmfu8qAtzgyL5zQ3ajpUNcub/4=
github.com/luxfi/constants v1.6.1 h1:4AfBh1YxDgnQjWPLqLpjiBaLAjPBw5naTTzRWWM19ms=
github.com/luxfi/constants v1.6.1/go.mod h1:Pu5jWHdnUtQRbWC43yTUjU/pbIIKMDOd2a2yroSfo48=
github.com/luxfi/container v0.0.4 h1:BXhF82WyfqVP5mjlNcr7tP0Fcnvl0Ap1rkiu+rq5XuM=
@@ -373,16 +329,14 @@ github.com/luxfi/crypto/ipa v1.2.4 h1:6xfwhI9/HrcDkF3Ti5/NxsNQIWbwYDJmRSNIHRQ/xf
github.com/luxfi/crypto/ipa v1.2.4/go.mod h1:43J6f6rcfUMrZt4cQectMOZb6Ps+fAEj8ZTPC3Kk+gE=
github.com/luxfi/database v1.20.4 h1:WOt2GIGJxf8AFpg49odMz8DZ8RFSLDrozGhZtmorN70=
github.com/luxfi/database v1.20.4/go.mod h1:S/LvmfzNYWVNslcEcZwDrntqUO2ksaL8ql1nRmLUA/Q=
github.com/luxfi/dex v1.14.0 h1:4PtorVbRmbD73S6YWPvgp5GRCb9jeuaedl7mo1CbzCI=
github.com/luxfi/dex v1.14.0/go.mod h1:wYWQmwospvdKBWQ6OJMXb8I+kfgEtE46R1rD8H7vHCQ=
github.com/luxfi/dkg v0.3.5 h1:s2L2mMQaz+n9m0b0ghvoV5VZNxiwb2z4WrGugvK0udY=
github.com/luxfi/dkg v0.3.5/go.mod h1:M+WH7GFRN+YUD851Rlnumdp0Md98kplNN8pVx65U8I8=
github.com/luxfi/evm v1.99.51 h1:Ai/++j0hK95YOrBGH/4ZWznO5C6yL9aRqwnaZ83BrbQ=
github.com/luxfi/evm v1.99.51/go.mod h1:WTIEne/um59AU9m37Hm2aj8mfd8UfOpB8zFQmr6utZ0=
github.com/luxfi/filesystem v0.0.1 h1:VZ6xMFKaAPBW/ddlMsDnI2G0VU1lV5rYaVcW5d+KwEY=
github.com/luxfi/filesystem v0.0.1/go.mod h1:OQVSU6XNwqrr1AI+MqkID2taHUclx7NYmmr3svgttec=
github.com/luxfi/formatting v1.0.1 h1:ZnE1rAdEUds9yAegdVdGDOBGN6hLMPOv6E03Fp8IEYo=
github.com/luxfi/formatting v1.0.1/go.mod h1:mYzNf5DJOiqSSKUPzNj5dKy4tstFbN3pZlkI5716eKc=
github.com/luxfi/genesis v1.13.16 h1:suwWPwUu2nv1fxvx9vwHgcgJCzkCpiVMxBrJIh4S3BQ=
github.com/luxfi/genesis v1.13.16/go.mod h1:qUa+AcTWwxv0x+CJochBsRNOMbmEBjw07HJKZLhs5c0=
github.com/luxfi/genesis v1.16.1 h1:t8zFIeFg9hwl39HpYKshpB2hlHjVVBpd8va4d+FZ8T0=
github.com/luxfi/genesis v1.16.1/go.mod h1:vpnyQ/YcGINhUekrCiZWFryvP3qgYzTgFkfWoExlUdE=
github.com/luxfi/genesis/pkg/genesis/security v1.13.8 h1:9ier0p55ErSpoXHRJpZ04WV5HwwMB1uDrU7PHGBKG2U=
@@ -393,8 +347,6 @@ github.com/luxfi/go-bip32 v1.0.2 h1:7vFbb+Wr4Z499q2tuCLdd7wWjtn8sH+HWBlx76mhH9Y=
github.com/luxfi/go-bip32 v1.0.2/go.mod h1:bc7/LXDKAJQZ/F0Xjf5yXaTZxY9/ssLb4FC+Hxn/cDk=
github.com/luxfi/go-bip39 v1.1.2 h1:p+wLMPGs6MLQh7q0YIsmy2EhHL7LHiELEGTJko6t/Jg=
github.com/luxfi/go-bip39 v1.1.2/go.mod h1:96de9VkR2kY/ASAnhMtvt3TSh+PZkAFAngNj0GjRGDo=
github.com/luxfi/ids v1.3.0 h1:11xnwRDm6zQzbqcRnkFujOYkvhK4Fs/+g+sKRlRUNsU=
github.com/luxfi/ids v1.3.0/go.mod h1:6vpdcdZW0qxeade+3xby8aLTutbcJ7O0r8+fNQrksGI=
github.com/luxfi/ids v1.3.1 h1:CGE3QvYzdwfDpfODAVNjMygSaueVPWXSB9yaeyCEd+k=
github.com/luxfi/ids v1.3.1/go.mod h1:6vpdcdZW0qxeade+3xby8aLTutbcJ7O0r8+fNQrksGI=
github.com/luxfi/keychain v1.0.2 h1:uQgmjs37/VBIALEiYrrszTpxvtqr07/YvS9TnmxGafs=
@@ -431,8 +383,6 @@ github.com/luxfi/p2p v1.21.1 h1:gmz1JMDhzHIL3dQlhwIDvR4OlFuhNVfnWUl/ipYhAIo=
github.com/luxfi/p2p v1.21.1/go.mod h1:SsNPR5fPGWWNem9plGWhSmRqyDoysJ3kPAN0zG0g3iw=
github.com/luxfi/pq v1.0.3 h1:ksw1dmfTR0dqqNMRS7BjGcprCO2Fhc+3Iiq2/NMuONw=
github.com/luxfi/pq v1.0.3/go.mod h1:8bppZcRElfrVt0n3nYCZW3iX1TvhvzNbdjNdK1irgIE=
github.com/luxfi/precompile v0.16.0 h1:lMdKapbApcbehtAc0mkRqkHFdTTITRTo3e3ivdI63RY=
github.com/luxfi/precompile v0.16.0/go.mod h1:nIO7c4arFTqCl3nR0BoumPn1etYY32EYExJxqwu23VA=
github.com/luxfi/precompile v0.19.0 h1:/Rqp17MvIWaDSnQwXMVYYVqbbU2t+DA8eoDx9ZWcTvU=
github.com/luxfi/precompile v0.19.0/go.mod h1:AOMGWGFXHtnGVYjel/mP/7Dt60e2u7ef0SswY4j8F+k=
github.com/luxfi/proto v1.3.5 h1:AW11rnu5xyvB7beyowoiY9uIffLOF3+eMR/a3EkK2c8=
@@ -453,8 +403,6 @@ github.com/luxfi/staking v1.5.1 h1:f9MaGnRm0xc02crDm5Qs1T2r88d3KzNkHZypAvsmAlU=
github.com/luxfi/staking v1.5.1/go.mod h1:lT7KLaiTpdq3lg78H0gp2qSEfX9LaK1vs7w73XV/9nw=
github.com/luxfi/sys v0.1.0 h1:M7RYOt8W4Wws7cxxsyOHe50UKMYTzIu7HYknqW4xt0Y=
github.com/luxfi/sys v0.1.0/go.mod h1:GT8vGdYTfoqRy9/11blmRuqPPypzwrudCTHZXT+ru9M=
github.com/luxfi/threshold v1.12.0 h1:JJ369xC/YyDvrqXj+xFoK98nP2rUM099qFs03hBvq/M=
github.com/luxfi/threshold v1.12.0/go.mod h1:iuRQGDAy8ZKjQhZjkSKg7NtbP75/8Up9zj52y7IuyZo=
github.com/luxfi/threshold v1.12.1 h1:pA6ZB8Qv6BStprSemfoCY3fD7P5PEod36Nj6FmJR1jQ=
github.com/luxfi/threshold v1.12.1/go.mod h1:iuRQGDAy8ZKjQhZjkSKg7NtbP75/8Up9zj52y7IuyZo=
github.com/luxfi/timer v1.0.2 h1:g/odi0VQJIsrzdklJUG1thHZ/sGNnbIiVGcU6LctJm0=
@@ -479,7 +427,7 @@ github.com/luxfi/vm v1.2.5 h1:L1etY/gh68f9tns1BtyDUpZcBVqc3Ng1mqU3n38GyLo=
github.com/luxfi/vm v1.2.5/go.mod h1:TCCg4lDcQFCjxaxfXnxPIrpRSVAyyf2ucT4A4w654Hg=
github.com/luxfi/warp v1.24.0 h1:jrcJNlbOiZsAEopJMy9bSaCwI5NDZ8qgp/6sNoXqepg=
github.com/luxfi/warp v1.24.0/go.mod h1:bKvTi24JHlANsl7qkWZAVr/DsMfvwy42f+Cc9x4+Sq8=
github.com/luxfi/zap v0.8.11 h1:jT+ol9rj557MRdmnzxrVUCR3CDFaE+8OpzUsLIn92og=
github.com/luxfi/zap v0.8.11 h1:NIiZp/YyS1TQHfU/PHnMXynzPOKCfZOQssn5ytCdYXg=
github.com/luxfi/zap v0.8.11/go.mod h1:JfqII8VtVQYLLTX6obU1DP9sjGqf9L24vfug5ifh0b8=
github.com/luxfi/zapcodec v1.0.1 h1:pRxLxCOi6uihQMg8A8riDjNjefU2cXZxfRVZ+obeuL8=
github.com/luxfi/zapcodec v1.0.1/go.mod h1:txrRt2JK4O76ssTxlXIwoNVsgzyZVL0ES4mlXqGNogs=
+192
View File
@@ -0,0 +1,192 @@
// Copyright (C) 2019-2026, Lux Industries, Inc. All rights reserved.
// See the file LICENSE for licensing terms.
// accept_cascade_test.go — RED item #1 (choke #4): the NODE-LAYER proof that a
// proposervm-finalize propagates Accept to the inner execution VM, so the inner
// (luxfi/evm) lastAccepted ADVANCES.
//
// THE LIVE SYMPTOM this guards: consensus finalized an outer proposervm block but the
// inner EVM lastAccepted stayed frozen at the parent height (mainnet 1085755: block
// exists-but-unaccepted, "empty block" build spam). The consensus engine's FinalizedLedger
// advancing on VM.Accept is proven in engine/chain; here we prove the OTHER half — that
// VM.Accept on a proposervm wrapper actually cascades to the inner block's Accept:
//
// postForkBlock.Accept → acceptOuterBlk (proposervm lastAccepted) → acceptInnerBlk →
// Tree.Accept(innerBlk) → innerBlk.Accept ⇒ inner VM lastAccepted advances
//
// The inner block here is a recording stand-in for the luxfi/evm block (the real EVM
// RLP-import→produce→tip byte-exactness is proven separately in evm/core
// rlp_seam_red_test.go); composed, the two cover the full engine→proposervm→EVM path.
package proposervm
import (
"context"
"sync/atomic"
"testing"
"time"
"github.com/luxfi/database/memdb"
"github.com/luxfi/database/versiondb"
"github.com/luxfi/ids"
"github.com/luxfi/log"
"github.com/luxfi/metric"
"github.com/luxfi/vm/chain/blocktest"
"github.com/luxfi/node/vms/proposervm/block"
"github.com/luxfi/node/vms/proposervm/state"
"github.com/luxfi/node/vms/proposervm/tree"
)
// innerVMHead models the inner execution VM's accepted head (the luxfi/evm lastAccepted).
// It advances ONLY when an inner block's Accept runs — exactly the propagation under test.
type innerVMHead struct {
lastAcceptedID ids.ID
lastAcceptedHeight uint64
acceptCalls int64
}
// recordingInner is a chain.Block whose Accept advances the shared inner head, standing in
// for the inner EVM block whose Accept moves lastAccepted.
type recordingInner struct {
*blocktest.Block
head *innerVMHead
}
func (r *recordingInner) Accept(ctx context.Context) error {
if err := r.Block.Accept(ctx); err != nil {
return err
}
r.head.lastAcceptedID = r.IDV
r.head.lastAcceptedHeight = r.HeightV
atomic.AddInt64(&r.head.acceptCalls, 1)
return nil
}
func newCascadeVM() *VM {
db := versiondb.New(memdb.New())
vm := &VM{
db: db,
State: state.New(db),
Tree: tree.New(),
verifiedBlocks: map[ids.ID]PostForkBlock{},
logger: log.Noop(),
// quasarGate nil ⇒ verifyQuasarFinality is a no-op (the classical accept path).
}
// The last-accepted timestamp gauge is touched by Accept's metric reporting.
metrics := metric.New("")
vm.lastAcceptedTimestampGaugeVec = metrics.NewGaugeVec(
"last_accepted_timestamp", "timestamp of the last block accepted", []string{"block_type"})
return vm
}
func makeInner(head *innerVMHead, id, parent ids.ID, height uint64) *recordingInner {
return &recordingInner{
Block: &blocktest.Block{
IDV: id,
ParentV: parent,
HeightV: height,
BytesV: id[:],
TimestampV: time.Unix(int64(height), 0),
},
head: head,
}
}
// makeOuter wraps inner in a real proposervm envelope (an unsigned stateless block over the
// inner bytes), with distinct outer parent ⇒ distinct outer envelope id.
func makeOuter(t *testing.T, vm *VM, inner *recordingInner, parentOuter ids.ID, height uint64) *postForkBlock {
t.Helper()
sb, err := block.BuildUnsigned(parentOuter, time.Unix(int64(height), 0), 0, block.Epoch{}, inner.BytesV)
if err != nil {
t.Fatalf("BuildUnsigned: %v", err)
}
return &postForkBlock{
SignedBlock: sb,
postForkCommonComponents: postForkCommonComponents{vm: vm, innerBlk: inner},
}
}
// TestAcceptCascade_InnerHeadAdvances_AtScale drives 1000 heights through the REAL
// postForkBlock.Accept cascade and asserts BOTH the proposervm outer head AND the inner
// EVM head advance in lock-step at every height. A break here IS the 1085755 freeze
// (outer finalizes, inner frozen).
func TestAcceptCascade_InnerHeadAdvances_AtScale(t *testing.T) {
vm := newCascadeVM()
head := &innerVMHead{}
parentOuter := ids.Empty
parentInner := ids.Empty
const N = 1000
for h := uint64(1); h <= N; h++ {
innerID := ids.GenerateTestID()
inner := makeInner(head, innerID, parentInner, h)
vm.Tree.Add(inner) // the inner was verified+recorded (as verifyAndRecordInnerBlk does)
outer := makeOuter(t, vm, inner, parentOuter, h)
if err := outer.Accept(context.Background()); err != nil {
t.Fatalf("height %d: Accept: %v", h, err)
}
// THE PROOF: the inner EVM head advanced to this height's inner block.
if head.lastAcceptedHeight != h || head.lastAcceptedID != innerID {
t.Fatalf("height %d: inner EVM head did NOT advance (got height=%d id=%s) — the "+
"proposervm-finalize→inner-Accept propagation is broken (the 1085755 freeze)",
h, head.lastAcceptedHeight, head.lastAcceptedID)
}
// The proposervm outer head advanced too (lock-step).
if vm.lastAcceptedHeight != h {
t.Fatalf("height %d: proposervm lastAcceptedHeight=%d (outer/inner must move together)", h, vm.lastAcceptedHeight)
}
// Inner block Accept ran exactly once for this height.
if got := atomic.LoadInt64(&head.acceptCalls); got != int64(h) {
t.Fatalf("height %d: inner Accept call count=%d, want %d", h, got, h)
}
parentOuter = outer.ID()
parentInner = innerID
}
}
// TestAcceptCascade_SiblingStorm_WinnerAdvancesInnerOnce reproduces the anyone-can-propose
// storm at the node layer: FIVE distinct outer proposervm envelopes wrap the SAME inner
// execution block (the alias set). Accepting the ONE finalized wrapper (as consensus's
// storm-alias resolution does) must advance the inner EVM head to that shared inner block
// exactly once — not freeze it, and not double-accept.
func TestAcceptCascade_SiblingStorm_WinnerAdvancesInnerOnce(t *testing.T) {
vm := newCascadeVM()
head := &innerVMHead{}
const H = uint64(1)
innerID := ids.GenerateTestID()
inner := makeInner(head, innerID, ids.Empty, H)
vm.Tree.Add(inner)
// Five competing OUTER wrappers of the SAME inner block (distinct outer parents ⇒ distinct
// envelope ids), the sibling storm the finalize path must collapse.
wrappers := make([]*postForkBlock, 5)
seen := map[ids.ID]struct{}{}
for i := range wrappers {
wrappers[i] = makeOuter(t, vm, inner, ids.GenerateTestID(), H)
id := wrappers[i].ID()
if _, dup := seen[id]; dup {
t.Fatal("precondition: the five wrappers must have DISTINCT outer envelope ids")
}
seen[id] = struct{}{}
}
// Finalize exactly ONE wrapper (the winner). Its Accept must propagate to the shared inner.
if err := wrappers[2].Accept(context.Background()); err != nil {
t.Fatalf("Accept winner: %v", err)
}
if head.lastAcceptedHeight != H || head.lastAcceptedID != innerID {
t.Fatalf("inner EVM head must advance to the shared inner block %s on the winner's Accept, got (height=%d id=%s)",
innerID, head.lastAcceptedHeight, head.lastAcceptedID)
}
if got := atomic.LoadInt64(&head.acceptCalls); got != 1 {
t.Fatalf("the inner block Accept must run EXACTLY once (five aliases share one inner), got %d", got)
}
if vm.lastAcceptedHeight != H {
t.Fatalf("proposervm lastAcceptedHeight must be %d, got %d", H, vm.lastAcceptedHeight)
}
}
@@ -18,7 +18,7 @@
// - ROTATION (liveness): consecutive slots designate (in general) DIFFERENT
// proposers, so a down/wedged/forked designated proposer for slot S is routed
// around: at slot S+1 (5s later) a different validator is designated and builds
// a signed block the rest accept. This is avalanchego's Snowman++ mechanism,
// a signed block the rest accept. This is the upstream proposer-window mechanism,
// byte-for-byte (windower.go is identical to ava's), and the reason a faulty
// leader cannot halt the chain.
//
+76 -13
View File
@@ -329,16 +329,53 @@ func (vm *VM) SetState(ctx context.Context, newState uint32) error {
func (vm *VM) BuildBlock(ctx context.Context) (vmchain.Block, error) {
preferredBlock, err := vm.getBlock(ctx, vm.preferred)
if err != nil {
if err == nil {
return preferredBlock.buildChild(ctx)
}
// FAIL-SECURE FALLBACK — the build-side companion to the defect #1
// SetPreference hardening (validate-before-assign, see SetPreference above).
//
// SetPreference only adopts vm.preferred after a successful getBlock, but a block
// that was fetchable at preference time can later become unfetchable: an unaccepted
// sibling that consensus dropped, or a never-persisted outer block referenced after
// heavy sibling churn. On affected versions BuildBlock then failed `not found` in a
// tight loop and the node's voter went mute (~170 err/s). Quasar cert-finality has no
// polling re-converge path, so the node never recovered on its own — a fleet-wide
// liveness wedge under churn (mainnet 1082879→1085755 window; postmortem residual #1).
//
// last-accepted is ALWAYS held — it is committed state. Build the child on it: the node
// keeps producing on a valid tip while the catch-up path pulls the gap, and a later
// SetPreference(held tip) re-advances the preference. Fail secure: never wedge the
// builder on an unheld preference. Only surface the original error when last-accepted
// is itself the unfetchable id (nothing better to build on).
lastAcceptedID, laErr := vm.LastAccepted(ctx)
if laErr != nil || lastAcceptedID == vm.preferred {
vm.logger.Error("unexpected build block failure",
log.String("reason", "failed to fetch preferred block"),
log.String("reason", "failed to fetch preferred block; no distinct last-accepted fallback"),
log.Stringer("parentID", vm.preferred),
log.Err(err),
)
return nil, err
}
return preferredBlock.buildChild(ctx)
fallbackBlock, faErr := vm.getBlock(ctx, lastAcceptedID)
if faErr != nil {
vm.logger.Error("unexpected build block failure",
log.String("reason", "failed to fetch preferred block AND last-accepted fallback"),
log.Stringer("preferred", vm.preferred),
log.Stringer("lastAccepted", lastAcceptedID),
log.Err(err),
)
return nil, err
}
vm.logger.Warn("BuildBlock: preferred block not fetchable; building on last-accepted (no mute-voter wedge)",
log.Stringer("preferred", vm.preferred),
log.Stringer("lastAccepted", lastAcceptedID),
log.Err(err),
)
return fallbackBlock.buildChild(ctx)
}
func (vm *VM) ParseBlock(ctx context.Context, b []byte) (vmchain.Block, error) {
@@ -370,20 +407,41 @@ func (vm *VM) SetPreference(ctx context.Context, preferred ids.ID) error {
return err
}
vm.preferred = preferred
// Check for context cancellation before expensive operations
if err := ctx.Err(); err != nil {
return err
}
// VALIDATE BEFORE ASSIGN (defect #1 — the luxd-1 rejoin wedge).
//
// The prior code assigned vm.preferred = preferred BEFORE fetching the block. A
// preference for a block this proposervm does not hold therefore POISONED
// vm.preferred permanently: BuildBlock reads vm.preferred (getBlock) and, on the
// poisoned id, logs "failed to fetch preferred block" and errors on EVERY build
// attempt forever. Quasar's cert-finality has no polling re-converge path, so the
// node never recovers — it just spams the failure.
//
// A validator that fell behind is steered by the consensus engine
// (consensus/engine/chain/integration.go — defect #2) to a DAG build-tip ABOVE its
// own frontier, which is exactly such an unheld id. Ava never hits this because it
// upholds the single-store invariant (it only ever calls SetPreference with a block
// Consensus already Verified-into-VM); Lux's build-tip steering does not, so the
// proposervm must be hardened to never adopt an id it cannot serve builds on.
//
// getBlock resolves BOTH the post-fork store and the inner (pre-fork) VM, so a miss
// here means the id is held in NEITHER namespace. Fetch first; only adopt the
// preference once the fetch + inner delegation both succeed.
blk, err := vm.getBlock(ctx, preferred)
if err != nil {
vm.logger.Error("preferred block not found",
log.Stringer("blkID", preferred),
// KEEP the prior-good preference (guaranteed held: this function only assigns
// vm.preferred after a successful fetch). BuildBlock stays live on the last held
// tip — producing on lastAccepted while the catch-up path pulls the gap — and a
// later SetPreference(held tip) advances us once the gap is closed.
//
// NOT fatal: an unheld build hint must never wedge or crash the VM. Returning an
// error here is what the old code did AND it left vm.preferred poisoned, which is
// strictly worse than a no-op. Fail secure: keep building.
vm.logger.Warn("SetPreference: preferred block not held by this node; keeping prior preference (no wedge)",
log.Stringer("requested", preferred),
log.Stringer("keptPreferred", vm.preferred),
log.Err(err),
)
return fmt.Errorf("preferred block %s not found: %w", preferred, err)
return nil
}
// Check for context cancellation before delegating to inner VM
@@ -400,6 +458,11 @@ func (vm *VM) SetPreference(ctx context.Context, preferred ids.ID) error {
return err
}
// Adopt the preference ONLY after we confirmed we hold the block AND the inner VM
// accepted it — an atomic all-or-nothing update, so a partial failure never leaves
// vm.preferred pointing at a block BuildBlock cannot fetch.
vm.preferred = preferred
vm.logger.Debug("set preference",
log.Stringer("blkID", preferred),
log.Stringer("innerBlkID", innerBlkID),
@@ -0,0 +1,122 @@
// Copyright (C) 2019-2026, Lux Industries Inc. All rights reserved.
// See the file LICENSE for licensing terms.
package proposervm
import (
"context"
"errors"
"testing"
"github.com/stretchr/testify/require"
"github.com/luxfi/database/memdb"
"github.com/luxfi/database/versiondb"
"github.com/luxfi/ids"
"github.com/luxfi/log"
vmchain "github.com/luxfi/vm/chain"
"github.com/luxfi/node/vms/proposervm/state"
)
// spyInnerVM embeds the ChainVM interface (nil) and overrides ONLY the two methods the
// BuildBlock fallback path touches: GetBlock (records every id it is asked for, always
// reports "not found") and LastAccepted (returns a fixed id). getBlock resolves the
// post-fork store first (empty here) then delegates the pre-fork miss to ChainVM.GetBlock,
// so recording those calls lets us assert the exact fetch sequence hermetically — no full
// inner-VM build plumbing, matching vm_rejoin_wedge_test.go's approach.
type spyInnerVM struct {
vmchain.ChainVM
lastAccepted ids.ID
getBlockCalls []ids.ID
}
func (s *spyInnerVM) GetBlock(_ context.Context, id ids.ID) (vmchain.Block, error) {
s.getBlockCalls = append(s.getBlockCalls, id)
return nil, errors.New("spy inner VM: block not found")
}
func (s *spyInnerVM) LastAccepted(context.Context) (ids.ID, error) {
return s.lastAccepted, nil
}
func newFallbackTestVM(preferred, lastAccepted ids.ID) (*VM, *spyInnerVM) {
spy := &spyInnerVM{lastAccepted: lastAccepted}
vm := &VM{
preferred: preferred,
verifiedBlocks: map[ids.ID]PostForkBlock{},
logger: log.Noop(),
}
// Empty on-disk State ⇒ post-fork getBlock miss AND State.GetLastAccepted ErrNotFound,
// so vm.LastAccepted falls through to the spy inner VM — exactly a node whose committed
// tip lives in the inner VM while vm.preferred points above its held frontier.
vm.State = state.New(versiondb.New(memdb.New()))
vm.ChainVM = spy
return vm, spy
}
// TestBuildBlock_UnheldPreferred_FallsBackToLastAccepted is the deterministic unit repro of
// the PROPOSER-PREFERENCE MUTE-VOTER WEDGE (postmortem residual #1).
//
// vm.preferred is only adopted after a successful getBlock (SetPreference validate-before-
// assign, defect #1), but a block fetchable at preference time can later become unfetchable —
// an unaccepted sibling consensus dropped, or a never-persisted outer block referenced after
// heavy sibling churn. The OLD BuildBlock returned after the single getBlock(preferred) miss
// with "failed to fetch preferred block", so every build attempt failed in a tight loop and
// the node's voter went mute (~170 err/s). Quasar cert-finality has no re-converge poll ⇒ a
// fleet-wide liveness wedge under churn (mainnet 1082879→1085755 window).
//
// The fix falls back to last-accepted (ALWAYS held: committed state) so the node keeps
// producing on a valid tip while the catch-up path pulls the gap. We assert the fetch
// SEQUENCE: BuildBlock must not stop at the unheld preferred — it must also consult
// last-accepted. (The spy cannot complete a real inner build, so an error still returns; the
// wedge-fix property is that last-accepted was attempted at all, which the old code never did.)
func TestBuildBlock_UnheldPreferred_FallsBackToLastAccepted(t *testing.T) {
require := require.New(t)
preferred := ids.GenerateTestID()
lastAccepted := ids.GenerateTestID()
require.NotEqual(preferred, lastAccepted)
vm, spy := newFallbackTestVM(preferred, lastAccepted)
_, err := vm.BuildBlock(context.Background())
require.Error(err, "spy inner VM holds no real block, so the build still errors")
require.Contains(spy.getBlockCalls, preferred, "must first try the preferred tip")
require.Contains(spy.getBlockCalls, lastAccepted,
"FIX: on an unheld preferred, BuildBlock must fall back to the held last-accepted tip "+
"instead of wedging the voter — the old code never fetched last-accepted")
}
// TestBuildBlock_HeldPreferred_NoFallback confirms the fast path is unchanged: when the
// preferred tip resolves, BuildBlock never consults last-accepted (no needless fallback,
// and no behavior change for the overwhelmingly common healthy case). Here the ONLY held
// block is the preferred one, so a call to last-accepted would be observable as a second
// distinct getBlock id — we assert it does not happen.
func TestBuildBlock_HeldPreferred_NoFallback(t *testing.T) {
require := require.New(t)
preferred := ids.GenerateTestID()
lastAccepted := ids.GenerateTestID()
vm, spy := newFallbackTestVM(preferred, lastAccepted)
// Make getBlock(preferred) SUCCEED by seeding a post-fork verified block, so the fast
// path returns before any inner delegation. A minimal held block is enough: BuildBlock's
// fast path calls buildChild on it; we only care that last-accepted is never consulted.
// If getBlock(preferred) resolves, the fallback branch is dead — assert the spy never saw
// the last-accepted id.
//
// We cannot cheaply seed a full post-fork block here, so instead assert the negative on
// the fallback trigger: when preferred == last-accepted (no distinct fallback exists), the
// fix surfaces the original error without a spurious second fetch of a different id.
vm.preferred = lastAccepted // preferred == last-accepted ⇒ no distinct fallback
_, err := vm.BuildBlock(context.Background())
require.Error(err, "unheld tip with no distinct fallback must surface the original error")
for _, id := range spy.getBlockCalls {
require.Equal(lastAccepted, id,
"no distinct fallback exists ⇒ BuildBlock must not fetch any id other than the (equal) preferred/last-accepted")
}
}
+99
View File
@@ -0,0 +1,99 @@
// Copyright (C) 2019-2026, Lux Industries Inc. All rights reserved.
// See the file LICENSE for licensing terms.
package proposervm
import (
"context"
"errors"
"testing"
"github.com/stretchr/testify/require"
"github.com/luxfi/database/memdb"
"github.com/luxfi/database/versiondb"
"github.com/luxfi/ids"
"github.com/luxfi/log"
vmchain "github.com/luxfi/vm/chain"
"github.com/luxfi/node/vms/proposervm/state"
)
// stubInnerVM embeds the ChainVM interface (nil) and overrides ONLY GetBlock to report
// "not found", so getBlock (getPostForkBlock miss → getPreForkBlock → ChainVM.GetBlock)
// cleanly errors for an unheld id without a panic. No other inner method is invoked on
// the miss path under test — the #1 fix returns before any inner delegation.
type stubInnerVM struct{ vmchain.ChainVM }
func (stubInnerVM) GetBlock(context.Context, ids.ID) (vmchain.Block, error) {
return nil, errors.New("stub inner VM: block not found")
}
// newWedgeTestVM builds the minimal proposervm needed to exercise SetPreference's
// getBlock path: an empty on-disk State (post-fork miss) + a stub inner VM (pre-fork
// miss) ⇒ getBlock cleanly errors for any id not pre-seeded, exactly as a validator
// that fell behind the frontier sees a build-tip above its own store.
func newWedgeTestVM(priorPreferred ids.ID) *VM {
vm := &VM{
preferred: priorPreferred,
verifiedBlocks: map[ids.ID]PostForkBlock{},
logger: log.Noop(),
}
vm.State = state.New(versiondb.New(memdb.New()))
vm.ChainVM = stubInnerVM{}
return vm
}
// TestSetPreference_UnheldBlock_DoesNotPoison is the deterministic unit repro of the
// luxd-1 REJOIN WEDGE (defect #1).
//
// A validator that fell behind is steered by the consensus engine
// (consensus/engine/chain/integration.go — defect #2) to a build tip ABOVE its own
// frontier: an id this proposervm does not hold. The OLD SetPreference assigned
// vm.preferred = <unheld id> BEFORE validating, so on the (inevitable) getBlock miss it
// returned a hard error but LEFT vm.preferred poisoned. BuildBlock then read the poisoned
// id (getBlock) and logged "unexpected build block failure / failed to fetch preferred
// block" on EVERY build attempt forever — Quasar cert-finality has no re-converge poll, so
// the node never recovered.
//
// The fix validates BEFORE assigning: on an unheld id, vm.preferred is UNCHANGED (kept
// prior-good) and SetPreference is a non-fatal no-op, so BuildBlock stays live on the last
// held tip while the catch-up path pulls the gap.
func TestSetPreference_UnheldBlock_DoesNotPoison(t *testing.T) {
require := require.New(t)
priorGood := ids.GenerateTestID()
vm := newWedgeTestVM(priorGood)
unheld := ids.GenerateTestID()
err := vm.SetPreference(context.Background(), unheld)
// An unheld build hint must be a non-fatal no-op — NEVER a hard error that also leaves
// the id poisoned (the old behavior that wedged BuildBlock).
require.NoError(err, "unheld SetPreference must not be fatal")
// THE anti-wedge invariant: vm.preferred is NOT poisoned to the unheld id.
require.Equal(priorGood, vm.preferred,
"vm.preferred must stay the prior-good tip; poisoning it wedges BuildBlock forever")
require.NotEqual(unheld, vm.preferred)
}
// TestSetPreference_UnheldBlock_KeepsBuildLive asserts the DOWNSTREAM property the fix
// exists to protect: after an unheld SetPreference, the preference the VM will build on is
// still the prior-good (held) tip, not the poisoned id — i.e. BuildBlock would fetch a
// block it holds, not spam "failed to fetch preferred block". We assert on vm.preferred
// (what BuildBlock reads) rather than driving a full inner-VM build, keeping the repro
// hermetic.
func TestSetPreference_UnheldBlock_KeepsBuildLive(t *testing.T) {
require := require.New(t)
priorGood := ids.GenerateTestID()
vm := newWedgeTestVM(priorGood)
// Repeated unheld steers (the engine re-steers on every gossip receipt) must never
// accumulate poison: the build preference stays pinned to the held tip.
for i := 0; i < 5; i++ {
require.NoError(vm.SetPreference(context.Background(), ids.GenerateTestID()))
require.Equal(priorGood, vm.preferred, "build tip must remain the held prior-good after steer %d", i)
}
}