mirror of
https://github.com/luxfi/node.git
synced 2026-07-27 03:39:39 +00:00
fix(chains): luxd-2 behind-node freeze — caught-up by ACCEPTANCE not store + convergence off the finalized ledger
A validator holding gossiped-but-unaccepted blocks (in the VM store, lastAccepted below them) concluded caught-up at its stale height forever — the caught-up check conflated store presence (Has/heldHeight via vm.GetBlock) with acceptance. Fix (consensus v1.25.36): Has→Accepted (a stored-but-unaccepted named frontier descends+ACCEPTS through the cert-gated per-height guard); recognize convergence off the in-process consensus finalized ledger, not the frozen VM.LastAccepted cache; un-freeze the zap client's lastAcceptedID on Accept. Proven on mainnet luxd-2 (1082780→1082796). C1/VerifyWeighted/cert-gate intact. blue→red→blue→red.
This commit is contained in:
+137
-37
@@ -256,7 +256,7 @@ func (b *blockHandler) FrontierTip(ctx context.Context) (ids.ID, chainbootstrap.
|
||||
// re-executes the descent and re-enters consensus, where ConsensusQuorum alone governs).
|
||||
// With the P-ready gate above, this judgement is over the TRUE FULL staked set, so a tip the
|
||||
// quorum actively names is a real frontier (when it equals our own held tip, a ⅔-of-responders
|
||||
// supermajority is AT our height, so we ARE at the network frontier — the loop's Has()-shortcut
|
||||
// supermajority is AT our height, so we ARE at the network frontier — the loop's Accepted()-shortcut
|
||||
// then correctly concludes caught-up). A Byzantine minority reporting an unheld forged sibling
|
||||
// cannot reach the ⅔ agreement, so it is never named (C1) and — crucially — never BLOCKS the
|
||||
// honest named tip either (we do NOT require holding every reported tip to accept the named
|
||||
@@ -267,7 +267,10 @@ func (b *blockHandler) FrontierTip(ctx context.Context) (ids.ID, chainbootstrap.
|
||||
log.Stringer("tip", frontier.ID),
|
||||
log.Uint64("namedHeight", frontier.Height),
|
||||
log.Int("responders", frontier.Responders),
|
||||
log.Bool("held", b.Has(ctx, frontier.ID)))
|
||||
// ACCEPTED, not merely held: a gossiped-but-unaccepted frontier (the freeze) logs
|
||||
// accepted=false here, so the loop's Accepted()-shortcut DESCENDS instead of completing
|
||||
// at the stale tip. The diagnostic now distinguishes "in the store" from "finalized".
|
||||
log.Bool("accepted", b.Accepted(ctx, frontier.ID)))
|
||||
return frontier.ID, chainbootstrap.FrontierNamed
|
||||
case errors.Is(err, ErrInsufficientBootstrapResponses):
|
||||
// Fewer than MinResponses configured beacons have RESPONDED — not a partition-capture-safe
|
||||
@@ -296,14 +299,14 @@ func (b *blockHandler) FrontierTip(ctx context.Context) (ids.ID, chainbootstrap.
|
||||
// policy.CaughtUp is true ONLY when the floor is met AND no responder reports an accepted tip
|
||||
// above our height AND we hold every reported tip (its full safety argument lives on the
|
||||
// method). When it holds, the network frontier IS our own accepted tip: return it NAMED so
|
||||
// the loop's existing Has()-shortcut (bootstrapper "named a tip we already hold → synced")
|
||||
// the loop's existing Accepted()-shortcut (bootstrapper "named a tip we have ACCEPTED → synced")
|
||||
// concludes caught-up and we go Ready at our OWN height — never below it, never at a stale
|
||||
// height. A genuinely stale node has an honest responder ahead (or lacks its tip) → CaughtUp
|
||||
// is false → it keeps syncing; an eclipse hiding the ahead-nodes drops below the floor →
|
||||
// CaughtUp is false → it fails safe. Distinct from the transient finality-skew retry the loop
|
||||
// runs on a bare FrontierNoQuorum ("connected but momentarily split" — keep polling): this is
|
||||
// "split AND I am at the top of every responder" — provably DONE, not waiting.
|
||||
if haveLast && policy.CaughtUp(replies, lastH, b.heldHeight) {
|
||||
if haveLast && policy.CaughtUp(replies, lastH, b.acceptedHeight) {
|
||||
b.logger.Debug("bootstrap frontier: CAUGHT UP at own tip (floor met, no responder ahead, hold every reported tip)",
|
||||
log.Stringer("chainID", b.chainID),
|
||||
log.Stringer("tip", lastID),
|
||||
@@ -331,13 +334,16 @@ func (b *blockHandler) logFrontierInputs(weights map[ids.NodeID]uint64, connecte
|
||||
total += w
|
||||
}
|
||||
for _, r := range replies {
|
||||
h, held := b.heldHeight(r.Tip)
|
||||
h, accepted := b.acceptedHeight(r.Tip)
|
||||
b.logger.Debug("bootstrap frontier reply",
|
||||
log.Stringer("chainID", b.chainID),
|
||||
log.Stringer("from", r.NodeID),
|
||||
log.Stringer("reportedTip", r.Tip),
|
||||
log.Uint64("beaconWeight", r.Weight),
|
||||
log.Bool("held", held),
|
||||
// ACCEPTED (on our finalized chain), not merely present in the store: a beacon tip we
|
||||
// hold-but-have-not-accepted logs accepted=false with resolvedHeight 0 — the smoking gun
|
||||
// that distinguishes a genuine catch-up from the stored-but-unaccepted freeze.
|
||||
log.Bool("accepted", accepted),
|
||||
log.Uint64("resolvedHeight", h))
|
||||
}
|
||||
b.logger.Debug("bootstrap frontier inputs",
|
||||
@@ -539,8 +545,47 @@ func (b *blockHandler) ParseBlock(ctx context.Context, raw []byte) (cblock.Block
|
||||
return b.vm.ParseBlock(ctx, raw)
|
||||
}
|
||||
|
||||
// LastAccepted implements chainbootstrap.Chain: the local last-accepted id + height.
|
||||
// LastAccepted implements chainbootstrap.Chain: the node's ACCEPTED tip id + height, read
|
||||
// from the IN-PROCESS consensus finalized ledger — the SINGLE advancing source of truth the
|
||||
// loop drives convergence off (THE one-source decomplect). It is NOT read from the VM cache.
|
||||
//
|
||||
// THE FREEZE-STALL (red HIGH-1): the descent DOES accept the run-up (the consensus finalized
|
||||
// ledger AND the coreth on-disk store both advance), but the ZAP VM client caches its
|
||||
// LastAccepted at Initialize and a fire-and-forget Accept never refreshes it (client.go) —
|
||||
// so VM.LastAccepted is FROZEN for the process life. Reading lastH off that frozen cache made
|
||||
// every pass after the first re-descend from the stale height while AcceptBootstrapBlock
|
||||
// no-oped each block (height ≤ the ADVANCED finalizedHeight) → advanced=false → ErrStalled:
|
||||
// "unfreezes but stalls". AcceptBootstrapBlock's own contiguity guard already trusts the
|
||||
// in-process ledger (consensus.GetFinalizedHeight); driving lastH/the caught-up oracle off the
|
||||
// SAME ledger is the decomplect — the loop trusts the ledger it is building, not a VM cache.
|
||||
func (b *blockHandler) LastAccepted(ctx context.Context) (ids.ID, uint64, error) {
|
||||
return b.finalizedTip(ctx)
|
||||
}
|
||||
|
||||
// finalizedTip returns the in-process consensus finalized ledger position (the ADVANCING
|
||||
// source), falling back to the VM's last-accepted ONLY when the ledger is unset — the
|
||||
// empty-genesis boot window before the first finalize (consensus SyncState seeds the ledger
|
||||
// from a NON-empty last-accepted; an empty node leaves it unset until block 1 finalizes), or a
|
||||
// degenerate/test handler with no engine. The VM fallback is correct there: at genesis the VM
|
||||
// last-accepted IS the anchor and has not yet had a chance to go stale (it only diverges from
|
||||
// the ledger once blocks finalize, at which point the ledger takes over). This mirrors the
|
||||
// engine's own M2 first-block anchor (bootstrap_accept.go localLastAccepted), keeping ONE rule.
|
||||
func (b *blockHandler) finalizedTip(ctx context.Context) (ids.ID, uint64, error) {
|
||||
if b.engine != nil {
|
||||
if tip, h, set := b.engine.FinalizedLedger(); set {
|
||||
return tip, h, nil
|
||||
}
|
||||
}
|
||||
return b.vmLastAccepted(ctx)
|
||||
}
|
||||
|
||||
// vmLastAccepted reads the VM's last-accepted id + height. It is the genesis-anchor fallback
|
||||
// for finalizedTip (used only before the consensus ledger is seeded), NOT a convergence
|
||||
// signal — a converging node's height MUST come from the finalized ledger, never this cache.
|
||||
func (b *blockHandler) vmLastAccepted(ctx context.Context) (ids.ID, uint64, error) {
|
||||
if b.vm == nil {
|
||||
return ids.Empty, 0, nil
|
||||
}
|
||||
id, err := b.vm.LastAccepted(ctx)
|
||||
if err != nil {
|
||||
return ids.Empty, 0, err
|
||||
@@ -555,31 +600,89 @@ func (b *blockHandler) LastAccepted(ctx context.Context) (ids.ID, uint64, error)
|
||||
return id, blk.Height(), nil
|
||||
}
|
||||
|
||||
// Has implements chainbootstrap.Chain: whether the node already holds block id (so the
|
||||
// loop can detect it has reached the frontier).
|
||||
func (b *blockHandler) Has(ctx context.Context, id ids.ID) bool {
|
||||
if id == ids.Empty || b.vm == nil {
|
||||
return false
|
||||
}
|
||||
_, err := b.vm.GetBlock(ctx, id)
|
||||
return err == nil
|
||||
// Accepted implements chainbootstrap.Chain: reports whether id is on the node's ACCEPTED chain
|
||||
// (finalized) — NOT merely PRESENT in the block store. This is the loop's caught-up predicate.
|
||||
// THE FREEZE was a store-vs-acceptance conflation: the prior Has() returned true for a frontier
|
||||
// GOSSIPED into the store but UNACCEPTED (height ABOVE last-accepted), short-circuiting the loop
|
||||
// to caught-up at the stale last-accepted and never descending to accept it. A stored-but-
|
||||
// unaccepted block returns false here, so the loop descends and DRIVES its acceptance.
|
||||
func (b *blockHandler) Accepted(ctx context.Context, id ids.ID) bool {
|
||||
_, ok := b.acceptedHeightCtx(ctx, id)
|
||||
return ok
|
||||
}
|
||||
|
||||
// heldHeight is the height ORACLE the BootstrapPolicy.CaughtUp determination injects: it returns a
|
||||
// block's canonical height from the LOCAL store, ok=false when the node does NOT hold it. CaughtUp
|
||||
// uses it to (c) require the node to HOLD every reported tip and (b) read those held tips' heights
|
||||
// to confirm none is above the node's accepted height. It NEVER fetches over the network — a tip
|
||||
// the node lacks simply makes the node not-caught-up (it syncs), the safe direction. It is Has()
|
||||
// plus the height the determination needs, in one local lookup.
|
||||
func (b *blockHandler) heldHeight(id ids.ID) (uint64, bool) {
|
||||
// acceptedHeight is the ACCEPTANCE oracle injected into BootstrapPolicy.CaughtUp (as heightOf):
|
||||
// it returns id's height and TRUE iff the node has ACCEPTED id (id is on the node's finalized
|
||||
// chain), and (0,false) when id is merely PRESENT IN THE STORE but NOT accepted — the store-vs-
|
||||
// acceptance distinction the luxd-2 freeze hinged on. CaughtUp uses it to (c) require the node to
|
||||
// have ACCEPTED every reported tip and (b) read those tips' heights to confirm none is above the
|
||||
// node's accepted height; a stored-but-unaccepted tip now makes CaughtUp FALSE (the node is behind
|
||||
// in ACCEPTANCE → it syncs). It NEVER fetches over the network — an unaccepted/unheld tip simply
|
||||
// makes the node not-caught-up, the safe direction.
|
||||
func (b *blockHandler) acceptedHeight(id ids.ID) (uint64, bool) {
|
||||
return b.acceptedHeightCtx(context.Background(), id)
|
||||
}
|
||||
|
||||
// acceptedHeightCtx is the ctx-honoring core of the acceptance oracle (shared by Accepted and
|
||||
// acceptedHeight). Authoritative and VM-internals-light: the accepted head id+height come from
|
||||
// the IN-PROCESS consensus finalized ledger (LastAccepted → finalizedTip — the ADVANCING source,
|
||||
// never the frozen VM cache), the height-bound (id ABOVE finalizedHeight ⇒ not yet accepted —
|
||||
// the gossiped-ahead freeze case, regardless of store presence) is the primary anchor, and the
|
||||
// in-process per-height ledger is the fork-sibling oracle for a block at/below the finalized head.
|
||||
func (b *blockHandler) acceptedHeightCtx(ctx context.Context, id ids.ID) (uint64, bool) {
|
||||
if id == ids.Empty || b.vm == nil {
|
||||
return 0, false
|
||||
}
|
||||
blk, err := b.vm.GetBlock(context.Background(), id)
|
||||
lastID, lastH, err := b.LastAccepted(ctx)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
return blk.Height(), true
|
||||
if id == lastID {
|
||||
return lastH, true // the finalized head itself — the authoritative anchor
|
||||
}
|
||||
blk, err := b.vm.GetBlock(ctx, id)
|
||||
if err != nil {
|
||||
return 0, false // not even in the store → certainly not accepted
|
||||
}
|
||||
h := blk.Height()
|
||||
if h > lastH {
|
||||
// ABOVE our finalized head: a block GOSSIPED into the store ahead of acceptance. THE FREEZE
|
||||
// — present, but NOT accepted. NOT caught up; the node must descend and accept it.
|
||||
return 0, false
|
||||
}
|
||||
// h <= lastH and id != lastID: id is at a height we have finalized PAST. It is accepted IFF it
|
||||
// is the block our per-height ledger finalized at h. Ask the IN-PROCESS finalized ledger (the
|
||||
// same source LastAccepted reads), which replaces the dead coreth height index — block.ChainVM.
|
||||
// GetBlockIDAtHeight is unhandled over ZAP, so the old VM-index call returned nothing on the real
|
||||
// C-Chain and this whole branch was dead there. The in-process ledger answers authoritatively for
|
||||
// every height finalized THIS session.
|
||||
if canonical, ok := b.finalizedBlockAtHeight(h); ok {
|
||||
if canonical != id {
|
||||
return 0, false // a stored sibling/fork at a finalized height — NOT on the accepted chain
|
||||
}
|
||||
return h, true
|
||||
}
|
||||
// The per-height ledger does not know h: it is a height BELOW the boot seed (consensus SyncState
|
||||
// seeds only the boot height + advances upward; older heights are never re-seeded). We have
|
||||
// finalized PAST h and we hold id, so on a BFT-final chain (one finalized block per height) id is
|
||||
// on our accepted chain. A node CAN gossip-hold a non-finalized sibling at such a height with NO
|
||||
// safety break (it received a losing fork via gossip) — but that costs nothing HERE: this oracle
|
||||
// only feeds the caught-up decision, which acts on the ⅔-NAMED frontier, and C1 (⅔-by-stake
|
||||
// frontier naming) guarantees a forged/losing sibling is NEVER the named frontier; the descent's
|
||||
// content-addressing + the per-height accept guard reject any off-chain block during execution.
|
||||
// So treating a held sub-boot-seed block as accepted is the safe over-approximation.
|
||||
return h, true
|
||||
}
|
||||
|
||||
// finalizedBlockAtHeight returns the block the IN-PROCESS consensus ledger finalized at height h
|
||||
// (ok=false when the node has not finalized h this session — a height below the boot seed — or has
|
||||
// no engine). It is the authoritative fork-sibling oracle that replaces the dead coreth height
|
||||
// index; degrading to ok=false simply routes acceptedHeightCtx to its height-bound anchor.
|
||||
func (b *blockHandler) finalizedBlockAtHeight(h uint64) (ids.ID, bool) {
|
||||
if b.engine == nil {
|
||||
return ids.Empty, false
|
||||
}
|
||||
return b.engine.FinalizedBlockAtHeight(h)
|
||||
}
|
||||
|
||||
// AcceptBootstrapBlock implements chainbootstrap.Chain: re-execute + finalize a fetched
|
||||
@@ -626,22 +729,19 @@ func (b *blockHandler) BootstrapFailure() error {
|
||||
// quorum" (self-heal → keep waiting). Distinct from BootstrapFailed (a terminal/structural fail).
|
||||
func (b *blockHandler) BootstrapConnecting() bool { return b.bootstrapConnecting.Load() }
|
||||
|
||||
// BootstrapHeight reports the node's current locally-accepted height — the PROGRESS
|
||||
// signal monitorBootstrap uses (H2) to tell a slow-but-advancing sync (reset the stall
|
||||
// timer) from a genuine no-progress stall (fail). Best-effort: 0 if unknown.
|
||||
// BootstrapHeight reports the node's current ACCEPTED height — the PROGRESS signal
|
||||
// monitorBootstrap uses (H2) to tell a slow-but-advancing sync (reset the stall timer) from a
|
||||
// genuine no-progress stall (fail). It reads the IN-PROCESS finalized ledger (finalizedTip), the
|
||||
// SAME advancing source as the convergence oracle — NOT the VM cache, which a fire-and-forget ZAP
|
||||
// Accept leaves FROZEN: reading the frozen cache here made monitorBootstrap's progress probe see
|
||||
// zero advance even while the descent finalized block after block, so a deep but healthy sync
|
||||
// looked like a stall. Best-effort: 0 if unknown.
|
||||
func (b *blockHandler) BootstrapHeight() uint64 {
|
||||
if b.vm == nil {
|
||||
return 0
|
||||
}
|
||||
id, err := b.vm.LastAccepted(context.Background())
|
||||
if err != nil || id == ids.Empty {
|
||||
return 0
|
||||
}
|
||||
blk, err := b.vm.GetBlock(context.Background(), id)
|
||||
_, h, err := b.finalizedTip(context.Background())
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return blk.Height()
|
||||
return h
|
||||
}
|
||||
|
||||
// deliverBootstrapFrontier routes a frontier reply (TAGGED with the responding beacon)
|
||||
@@ -716,7 +816,7 @@ func (b *blockHandler) runBootstrapThenPoll(ctx context.Context) {
|
||||
//
|
||||
// GO-LIVE includes the CAUGHT-UP path. A tip-holding producer on a mixed-height co-restart names
|
||||
// no frontier (the responders split below ⅔) yet is at the top of every responder — FrontierTip
|
||||
// returns its OWN held tip NAMED (BootstrapPolicy.CaughtUp), the loop's Has()-shortcut concludes
|
||||
// returns its OWN held tip NAMED (BootstrapPolicy.CaughtUp), the loop's Accepted()-shortcut concludes
|
||||
// caught-up, and bs.Run returns nil → it goes Ready at its own tip. Without that path the producer
|
||||
// would fail safe DOWN at its own tip (the regression this round fixes), the opposite of the
|
||||
// stale-go-live bug.
|
||||
|
||||
+210
-28
@@ -72,29 +72,48 @@ const (
|
||||
errBSUnknown = bsErr("unknown bytes")
|
||||
)
|
||||
|
||||
// bsTestVM models a node VM: it can PARSE any block on the chain (the wire codec is
|
||||
// deterministic), but only GETS blocks it has ACCEPTED (stored). SetPreference marks a
|
||||
// block stored — the engine calls it right after Accept, so Has/LastAccepted advance
|
||||
// exactly as the node syncs.
|
||||
// bsTestVM models a node VM with the REAL store-vs-acceptance distinction (the prior mock
|
||||
// conflated them — GetBlock returned only accepted blocks). It can PARSE any block on the chain
|
||||
// (the wire codec is deterministic). It GETS a block that is either ACCEPTED or merely STORED (a
|
||||
// block GOSSIPED into the store ahead of acceptance — the luxd-2 freeze precondition). `accepted`
|
||||
// is the FINALIZED chain; `acceptedByHeight` is the canonical height→id index the real coreth VM
|
||||
// maintains (GetBlockIDAtHeight); `stored` holds present-but-unaccepted blocks. SetPreference
|
||||
// marks a block accepted — the engine calls it right after Accept, so LastAccepted/the index
|
||||
// advance exactly as the node syncs.
|
||||
type bsTestVM struct {
|
||||
all map[ids.ID]*bsTestBlock
|
||||
byBytes map[string]*bsTestBlock
|
||||
accepted map[ids.ID]bool
|
||||
lastAcc ids.ID
|
||||
all map[ids.ID]*bsTestBlock
|
||||
byBytes map[string]*bsTestBlock
|
||||
accepted map[ids.ID]bool // FINALIZED chain
|
||||
acceptedByHeight map[uint64]ids.ID // canonical height→id index (coreth GetBlockIDAtHeight)
|
||||
stored map[ids.ID]bool // present in the store but NOT (yet) accepted (gossiped-ahead)
|
||||
lastAcc ids.ID
|
||||
|
||||
// frozenLastAccepted models the REAL ZAP VM client (vms/rpcchainvm/zap/client.go before the
|
||||
// option-b refresh): a fire-and-forget Accept advances the plugin's on-disk store (so GetBlock /
|
||||
// the accepted map still advance) but NEVER the cached LastAccepted id, which stays frozen at the
|
||||
// boot snapshot for the process life. When true, SetPreference advances the store but does NOT
|
||||
// move lastAcc — the exact freeze red HIGH-1 traced. A node whose bootstrap height/caught-up
|
||||
// signals read this frozen cache "unfreezes but stalls"; reading the consensus finalized ledger
|
||||
// instead converges. The mock bsTestVM otherwise advanced lastAcc on SetPreference, which HID the
|
||||
// bug (units passed while the real client did not advance).
|
||||
frozenLastAccepted bool
|
||||
}
|
||||
|
||||
func newBSVM(chain []*bsTestBlock) *bsTestVM {
|
||||
vm := &bsTestVM{
|
||||
all: map[ids.ID]*bsTestBlock{},
|
||||
byBytes: map[string]*bsTestBlock{},
|
||||
accepted: map[ids.ID]bool{},
|
||||
all: map[ids.ID]*bsTestBlock{},
|
||||
byBytes: map[string]*bsTestBlock{},
|
||||
accepted: map[ids.ID]bool{},
|
||||
acceptedByHeight: map[uint64]ids.ID{},
|
||||
stored: map[ids.ID]bool{},
|
||||
}
|
||||
for _, b := range chain {
|
||||
vm.all[b.id] = b
|
||||
vm.byBytes[string(b.bytes)] = b
|
||||
}
|
||||
// Empty node: only genesis is accepted/stored.
|
||||
// Empty node: only genesis is accepted.
|
||||
vm.accepted[chain[0].id] = true
|
||||
vm.acceptedByHeight[chain[0].height] = chain[0].id
|
||||
vm.lastAcc = chain[0].id
|
||||
return vm
|
||||
}
|
||||
@@ -106,18 +125,38 @@ func newBSVMAt(chain []*bsTestBlock, m int) *bsTestVM {
|
||||
vm := newBSVM(chain)
|
||||
for i := 1; i <= m; i++ {
|
||||
vm.accepted[chain[i].id] = true
|
||||
vm.acceptedByHeight[chain[i].height] = chain[i].id
|
||||
}
|
||||
vm.lastAcc = chain[m].id
|
||||
return vm
|
||||
}
|
||||
|
||||
// store marks blocks PRESENT in the block store WITHOUT accepting them — models blocks GOSSIPED
|
||||
// into the store ahead of acceptance (the luxd-2 freeze precondition: GetBlock returns them, but
|
||||
// they are not on the accepted chain and not in the canonical height index).
|
||||
func (m *bsTestVM) store(blocks ...*bsTestBlock) {
|
||||
for _, b := range blocks {
|
||||
m.stored[b.id] = true
|
||||
}
|
||||
}
|
||||
|
||||
func (m *bsTestVM) BuildBlock(context.Context) (cblock.Block, error) { return nil, errBSNoBuild }
|
||||
func (m *bsTestVM) GetBlock(_ context.Context, id ids.ID) (cblock.Block, error) {
|
||||
if m.accepted[id] {
|
||||
if m.accepted[id] || m.stored[id] {
|
||||
return m.all[id], nil
|
||||
}
|
||||
return nil, errBSNotStored
|
||||
}
|
||||
|
||||
// GetBlockIDAtHeight is the canonical ACCEPTED height→id index (coreth's authoritative oracle): it
|
||||
// returns the FINALIZED block at a height, never a stored-but-unaccepted one. ids.Empty when no
|
||||
// block is accepted at h — exactly how the real VM reports "nothing finalized there".
|
||||
func (m *bsTestVM) GetBlockIDAtHeight(_ context.Context, h uint64) (ids.ID, error) {
|
||||
if id, ok := m.acceptedByHeight[h]; ok {
|
||||
return id, nil
|
||||
}
|
||||
return ids.Empty, nil
|
||||
}
|
||||
func (m *bsTestVM) ParseBlock(_ context.Context, b []byte) (cblock.Block, error) {
|
||||
if blk, ok := m.byBytes[string(b)]; ok {
|
||||
return blk, nil
|
||||
@@ -127,7 +166,15 @@ func (m *bsTestVM) ParseBlock(_ context.Context, b []byte) (cblock.Block, error)
|
||||
func (m *bsTestVM) LastAccepted(context.Context) (ids.ID, error) { return m.lastAcc, nil }
|
||||
func (m *bsTestVM) SetPreference(_ context.Context, id ids.ID) error {
|
||||
m.accepted[id] = true
|
||||
m.lastAcc = id
|
||||
if blk, ok := m.all[id]; ok {
|
||||
m.acceptedByHeight[blk.height] = id
|
||||
}
|
||||
delete(m.stored, id) // accepting a stored-ahead block promotes it onto the finalized chain
|
||||
if !m.frozenLastAccepted {
|
||||
m.lastAcc = id
|
||||
}
|
||||
// else: model the ZAP client — the plugin (store/accepted above) advances, but the cached
|
||||
// LastAccepted id stays FROZEN at the boot snapshot for the process life.
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -462,7 +509,60 @@ func TestNodeBootstrap_EmptyNodeConvergesViaTransport(t *testing.T) {
|
||||
last, _ := vm.LastAccepted(ctx)
|
||||
require.Equal(t, chain[N].id, last, "empty node must sync to the beacon-agreed tip (height %d)", N)
|
||||
require.Equal(t, 1, chain[N].accepts, "tip block must be VM-accepted exactly once")
|
||||
require.True(t, bh.Has(ctx, chain[N].id), "node must hold the tip after sync")
|
||||
require.True(t, bh.Accepted(ctx, chain[N].id), "node must hold the tip after sync")
|
||||
}
|
||||
|
||||
// TestRED_FrozenVMLastAccepted_ConvergesOffFinalizedLedger is the regression guard for red
|
||||
// HIGH-1: the convergence-recognition must ride the IN-PROCESS consensus finalized ledger, NOT
|
||||
// the VM's LastAccepted cache — which the real ZAP client FREEZES at the boot snapshot for the
|
||||
// process life (a fire-and-forget Accept advances the plugin but never the cache).
|
||||
//
|
||||
// THE GAP THE PRIOR MOCK COULD NOT EXPRESS. bsTestVM.SetPreference advanced lastAcc, so every
|
||||
// unit "passed" while the real client did NOT advance it. Here frozenLastAccepted models the real
|
||||
// client faithfully: a STALE node boots at M, the descent finalizes M+1..N (the consensus
|
||||
// finalized ledger AND the store both advance), but VM.LastAccepted stays pinned at M. With the
|
||||
// pre-fix wiring (lastH / Accepted / BootstrapHeight read the frozen cache) the loop re-descends
|
||||
// every pass while AcceptBootstrapBlock no-ops each already-finalized block (advanced=false) →
|
||||
// ErrStalled: "unfreezes but stalls". The fix drives those signals off the finalized ledger (the
|
||||
// same source AcceptBootstrapBlock's contiguity guard trusts), so the node converges in ONE Run.
|
||||
//
|
||||
// RED-before / GREEN-after: with LastAccepted/Accepted/BootstrapHeight reading the frozen cache
|
||||
// this require.NoError FAILS (runBS returns ErrStalled); reading the finalized ledger it PASSES.
|
||||
func TestRED_FrozenVMLastAccepted_ConvergesOffFinalizedLedger(t *testing.T) {
|
||||
const N = 40
|
||||
const M = 24 // a STALE node booted at height M; the descent must finalize M+1..N
|
||||
chain, byID := buildBSChain(N, -1)
|
||||
vm := newBSVMAt(chain, M)
|
||||
vm.frozenLastAccepted = true // the ZAP freeze: VM.LastAccepted stays pinned at M for the run
|
||||
bh, chainID, beacons := newBSHandlerAndEngine(t, vm, 4)
|
||||
|
||||
bh.net = &bsBeaconNet{bh: bh, chainID: chainID, connected: beacons, byID: byID, tip: chain[N], serveAncestors: true}
|
||||
bh.msgCreator = bsMsgBuilder{}
|
||||
|
||||
ctx := context.Background()
|
||||
require.NoError(t, runBS(t, bh),
|
||||
"must converge off the advancing finalized ledger even though VM.LastAccepted is frozen")
|
||||
|
||||
// Test premise still holds: the VM cache NEVER moved (faithful to the real ZAP client).
|
||||
frozen, err := vm.LastAccepted(ctx)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, chain[M].id, frozen,
|
||||
"premise: VM.LastAccepted must stay frozen at the boot height M — the bug only matters because it does")
|
||||
|
||||
// Yet every bootstrap signal advanced to the true frontier N off the consensus finalized ledger.
|
||||
tip, h, set := bh.engine.FinalizedLedger()
|
||||
require.True(t, set)
|
||||
require.Equal(t, chain[N].id, tip, "consensus finalized ledger must have advanced to the frontier")
|
||||
require.Equal(t, uint64(N), h)
|
||||
|
||||
lastID, lastH, err := bh.LastAccepted(ctx)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, chain[N].id, lastID, "Chain.LastAccepted must read the advancing finalized tip, not the frozen cache")
|
||||
require.Equal(t, uint64(N), lastH)
|
||||
require.Equal(t, uint64(N), bh.BootstrapHeight(), "BootstrapHeight (monitorBootstrap progress) must advance off the ledger")
|
||||
require.True(t, bh.Accepted(ctx, chain[N].id), "caught-up oracle must recognize the finalized frontier")
|
||||
// And a block ABOVE the finalized head is still correctly NOT accepted (the freeze direction).
|
||||
require.False(t, bh.Accepted(ctx, ids.GenerateTestID()), "an unknown/ahead id must never read as accepted")
|
||||
}
|
||||
|
||||
// TestRED_ForgedFrontierFromNonBeaconRejected is the HEADLINE C1 proof at the node layer.
|
||||
@@ -514,9 +614,9 @@ func TestRED_ForgedFrontierFromNonBeaconRejected(t *testing.T) {
|
||||
require.ErrorIs(t, err, chainbootstrap.ErrBeaconsUnreachable, "eclipsed node must fail safe, not false-complete")
|
||||
last, _ := vm.LastAccepted(ctx)
|
||||
require.Equal(t, honest[0].id, last, "C1: node must stay at genesis — the forged chain was NOT synced")
|
||||
require.False(t, bh.Has(ctx, forged[N].id), "C1: node must NOT hold the forged tip")
|
||||
require.False(t, bh.Accepted(ctx, forged[N].id), "C1: node must NOT hold the forged tip")
|
||||
for _, f := range forged[1:] {
|
||||
require.False(t, bh.Has(ctx, f.id), "C1: no forged block may be finalized")
|
||||
require.False(t, bh.Accepted(ctx, f.id), "C1: no forged block may be finalized")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -542,7 +642,7 @@ func TestRED_HonestBeaconQuorumIgnoresMaliciousFrontier(t *testing.T) {
|
||||
require.NoError(t, runBS(t, bh))
|
||||
last, _ := vm.LastAccepted(ctx)
|
||||
require.Equal(t, honest[N].id, last, "node must sync the beacon-agreed canonical tip")
|
||||
require.False(t, bh.Has(ctx, forged[N].id), "node must ignore the malicious forged frontier")
|
||||
require.False(t, bh.Accepted(ctx, forged[N].id), "node must ignore the malicious forged frontier")
|
||||
}
|
||||
|
||||
// TestRED_SubQuorumCannotNameFrontier: a minority of configured beacons cannot name the
|
||||
@@ -611,7 +711,7 @@ func TestNodeBootstrap_StaleNodeWaitsForBeaconConnect(t *testing.T) {
|
||||
|
||||
last, _ := vm.LastAccepted(ctx)
|
||||
require.Equal(t, chain[N].id, last, "CANARY: stale node must converge to the beacon frontier N=%d (NOT false-complete at the stale M=%d)", N, M)
|
||||
require.True(t, bh.Has(ctx, chain[N].id), "node must hold the beacon tip after sync")
|
||||
require.True(t, bh.Accepted(ctx, chain[N].id), "node must hold the beacon tip after sync")
|
||||
require.Greater(t, net.peerInfoCalls, net.connectAfterCalls, "the loop must have WAITED through the connecting passes before naming the frontier")
|
||||
}
|
||||
|
||||
@@ -723,7 +823,7 @@ func TestRED_PeersTrackNetNotChain_StaleNodeConverges(t *testing.T) {
|
||||
last, _ := vm.LastAccepted(ctx)
|
||||
require.Equal(t, chain[N].id, last,
|
||||
"CANARY SUCCESS: stale node converges to the producer frontier N=%d, not stuck at M=%d", N, M)
|
||||
require.True(t, bh.Has(ctx, chain[N].id), "node must hold the producer tip after sync")
|
||||
require.True(t, bh.Accepted(ctx, chain[N].id), "node must hold the producer tip after sync")
|
||||
}
|
||||
|
||||
// TestRED_EmptyStakedSetFailsSafe covers EDGE (4): a native chain whose STAKED validator set
|
||||
@@ -773,7 +873,7 @@ func TestRED_EmptyStakedSetFailsSafe(t *testing.T) {
|
||||
"empty staked set must fail safe at the connect deadline, never trust endpoints")
|
||||
last, _ := vm.LastAccepted(ctx)
|
||||
require.Equal(t, chain[5].id, last, "node must stay at its stale height — no endpoint-named frontier adopted")
|
||||
require.False(t, bh.Has(ctx, chain[N].id), "no frontier tip may be adopted from a non-staked endpoint")
|
||||
require.False(t, bh.Accepted(ctx, chain[N].id), "no frontier tip may be adopted from a non-staked endpoint")
|
||||
}
|
||||
|
||||
// TestRED_TipSplitConvergesToCommonHeight is THE MAINNET CANARY (luxd-2 on v1.30.78) the
|
||||
@@ -844,8 +944,8 @@ func TestRED_TipSplitConvergesToCommonHeight(t *testing.T) {
|
||||
last, _ := vm.LastAccepted(ctx)
|
||||
require.Equal(t, chain[top].id, last,
|
||||
"CANARY SUCCESS: ratchet through the ⅔-common N to the true finalized tip TOP=%d (NOT stuck at M=%d)", top, M)
|
||||
require.True(t, bh.Has(ctx, chain[N].id), "node must hold the ⅔-common N it passed through")
|
||||
require.True(t, bh.Has(ctx, chain[top].id), "node must hold the true finalized tip TOP after ratcheting to it")
|
||||
require.True(t, bh.Accepted(ctx, chain[N].id), "node must hold the ⅔-common N it passed through")
|
||||
require.True(t, bh.Accepted(ctx, chain[top].id), "node must hold the true finalized tip TOP after ratcheting to it")
|
||||
}
|
||||
|
||||
// TestRED_ForgedHighTipFromMinorityNotNamed is the C1 proof for the ancestor-tolerant path: a
|
||||
@@ -913,8 +1013,8 @@ func TestRED_ForgedHighTipFromMinorityNotNamed(t *testing.T) {
|
||||
require.NoError(t, runBS(t, bh), "node converges to the real finalized frontier, never the forged tip")
|
||||
last, _ := vm.LastAccepted(ctx)
|
||||
require.Equal(t, chain[top].id, last, "C1: node syncs to the real TOP (the true finalized frontier)")
|
||||
require.False(t, bh.Has(ctx, forged.id), "C1: the forged block must NOT be finalized — at any point in the ratchet")
|
||||
require.True(t, bh.Has(ctx, chain[N].id), "node holds the real ⅔-common N it passed through")
|
||||
require.False(t, bh.Accepted(ctx, forged.id), "C1: the forged block must NOT be finalized — at any point in the ratchet")
|
||||
require.True(t, bh.Accepted(ctx, chain[N].id), "node holds the real ⅔-common N it passed through")
|
||||
}
|
||||
|
||||
// TestNodeBootstrap_ExactQuorumUsesFastPath is the no-regression proof: when a ⅔-by-stake
|
||||
@@ -1621,11 +1721,93 @@ func TestNodeBootstrap_BehindNodeNotHeldTips_SyncsToFrontier(t *testing.T) {
|
||||
bh.msgCreator = bsMsgBuilder{}
|
||||
ctx := context.Background()
|
||||
|
||||
require.False(t, bh.Has(ctx, chain[Top].id), "precondition: the producers' tip Top is NOT held by the behind node")
|
||||
require.False(t, bh.Accepted(ctx, chain[Top].id), "precondition: the producers' tip Top is NOT held by the behind node")
|
||||
require.NoError(t, runBS(t, bh), "behind node must descend the not-held frontier and converge")
|
||||
last, _ := vm.LastAccepted(ctx)
|
||||
require.Equal(t, chain[Top].id, last, "behind node must reach Top=%d, never stop at stale M=%d", Top, M)
|
||||
require.True(t, bh.Has(ctx, chain[Top].id), "node must hold the producer tip after sync")
|
||||
require.True(t, bh.Accepted(ctx, chain[Top].id), "node must hold the producer tip after sync")
|
||||
}
|
||||
|
||||
// TestNodeBootstrap_StoredButUnacceptedFrontier_DrivesAcceptance is the NODE-LAYER reproduction of
|
||||
// THE mainnet luxd-2 freeze with production ground truth — the case the prior mocks could not even
|
||||
// express (their VM conflated store and acceptance). A behind node at M holds blocks M+1..Top
|
||||
// ALREADY IN ITS STORE (gossiped / a prior incomplete sync) but UNACCEPTED, and the ⅔-stake
|
||||
// producers name the frontier at Top — a tip the node HOLDS but has NOT accepted. THE BUG: the
|
||||
// node-side Has()/heldHeight reported "present" for those stored-but-unaccepted blocks, the loop's
|
||||
// Has()-shortcut concluded caught-up at the stale M, and the node went Ready at M forever (self-
|
||||
// reinforcing across restarts — every reboot re-concluded caught-up). THE FIX: Accepted(Top) is
|
||||
// false (present ≠ accepted), so the node descends and ACCEPTS M+1..Top — through the per-height-
|
||||
// guarded cert/Verify path, accepting the blocks already in the store — reaching Top, never Ready
|
||||
// at the stale M.
|
||||
func TestNodeBootstrap_StoredButUnacceptedFrontier_DrivesAcceptance(t *testing.T) {
|
||||
const M = 40
|
||||
const Top = 56 // gap 16, the exact ground-truth skew (1082780 → 1082796)
|
||||
chain, byID := buildBSChain(Top, -1)
|
||||
vm := newBSVMAt(chain, M)
|
||||
// GROUND TRUTH: M+1..Top are PRESENT IN THE STORE (gossiped ahead) but UNACCEPTED.
|
||||
vm.store(chain[M+1 : Top+1]...)
|
||||
|
||||
p1, p2, p3 := ids.GenerateTestNodeID(), ids.GenerateTestNodeID(), ids.GenerateTestNodeID()
|
||||
bh, chainID := newBSHandlerWeighted(t, vm, map[ids.NodeID]uint64{p1: 100, p2: 100, p3: 100})
|
||||
var pReady atomic.Bool
|
||||
pReady.Store(true) // P-chain synced: full staked set
|
||||
bh.primaryNetworkReady = pReady.Load
|
||||
bh.net = &bsBeaconNet{
|
||||
bh: bh, chainID: chainID, connected: []ids.NodeID{p1, p2, p3},
|
||||
byID: byID, tip: chain[Top], serveAncestors: true,
|
||||
}
|
||||
bh.msgCreator = bsMsgBuilder{}
|
||||
ctx := context.Background()
|
||||
|
||||
// Precondition — the EXACT freeze state: the node HOLDS the named frontier in its store yet has
|
||||
// NOT accepted it (last-accepted is M). The store-vs-acceptance distinction the conflation missed.
|
||||
require.True(t, vm.stored[chain[Top].id], "precondition: the frontier must be present in the store (gossiped ahead)")
|
||||
require.False(t, bh.Accepted(ctx, chain[Top].id), "precondition: the held-in-store frontier must NOT be accepted (last-accepted is M)")
|
||||
|
||||
require.NoError(t, runBS(t, bh), "node must descend+accept the stored-but-unaccepted frontier, not freeze at M")
|
||||
last, _ := vm.LastAccepted(ctx)
|
||||
require.Equal(t, chain[Top].id, last,
|
||||
"FREEZE: node must reach Top=%d, never go Ready at the stale M=%d while holding M+1..Top unaccepted", Top, M)
|
||||
require.True(t, bh.Accepted(ctx, chain[Top].id), "the named frontier must be ACCEPTED after sync")
|
||||
}
|
||||
|
||||
// TestBootstrap_AcceptedHeight_StoreVsAcceptance pins the ACCEPTANCE oracle the whole fix turns on:
|
||||
// acceptedHeight / Accepted report a block ACCEPTED only when it is on the FINALIZED chain — never
|
||||
// merely because it is PRESENT in the store. This is the store-vs-acceptance distinction the luxd-2
|
||||
// freeze conflated, isolated from the network so the predicate itself is the unit under test.
|
||||
func TestBootstrap_AcceptedHeight_StoreVsAcceptance(t *testing.T) {
|
||||
const M = 30
|
||||
const Top = 40
|
||||
chain, _ := buildBSChain(Top, -1)
|
||||
vm := newBSVMAt(chain, M) // accepted 0..M
|
||||
vm.store(chain[M+1 : Top+1]...) // M+1..Top GOSSIPED into the store but UNACCEPTED
|
||||
bh := &blockHandler{logger: log.NewNoOpLogger(), vm: vm}
|
||||
ctx := context.Background()
|
||||
|
||||
// (a) the accepted head and a finalized ancestor resolve as ACCEPTED at their canonical heights.
|
||||
if h, ok := bh.acceptedHeight(chain[M].id); !ok || h != uint64(M) {
|
||||
t.Fatalf("accepted head must resolve accepted at height %d, got (%d,%v)", M, h, ok)
|
||||
}
|
||||
if h, ok := bh.acceptedHeight(chain[M-5].id); !ok || h != uint64(M-5) {
|
||||
t.Fatalf("a finalized ancestor must resolve accepted at height %d, got (%d,%v)", M-5, h, ok)
|
||||
}
|
||||
// (b) stored-but-unaccepted blocks (ABOVE the accepted head) are present in the store yet NOT
|
||||
// accepted — the freeze conflation. acceptedHeight AND Accepted must both report not-accepted.
|
||||
for _, h := range []int{M + 1, Top} {
|
||||
if _, err := vm.GetBlock(ctx, chain[h].id); err != nil {
|
||||
t.Fatalf("precondition: chain[%d] must be present in the store", h)
|
||||
}
|
||||
if _, ok := bh.acceptedHeight(chain[h].id); ok {
|
||||
t.Fatalf("FREEZE: stored-but-unaccepted chain[%d] must NOT resolve as accepted", h)
|
||||
}
|
||||
if bh.Accepted(ctx, chain[h].id) {
|
||||
t.Fatalf("FREEZE: Accepted(stored-but-unaccepted chain[%d]) must be false", h)
|
||||
}
|
||||
}
|
||||
// (c) an absent block is not accepted.
|
||||
if bh.Accepted(ctx, ids.GenerateTestID()) {
|
||||
t.Fatal("an absent block must not be accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNodeBootstrap_BehindNode_EmptyAncestry_StaysBootstrapping (variant B). A behind node names the
|
||||
@@ -1659,7 +1841,7 @@ func TestNodeBootstrap_BehindNode_EmptyAncestry_StaysBootstrapping(t *testing.T)
|
||||
last, _ := vm.LastAccepted(ctx)
|
||||
require.Equal(t, chain[M].id, last,
|
||||
"behind node must NOT advance and must NOT go Ready when the gap cannot be fetched — stays at M=%d (Bootstrapping)", M)
|
||||
require.False(t, bh.Has(ctx, chain[Top].id), "node must NOT hold the unfetched frontier")
|
||||
require.False(t, bh.Accepted(ctx, chain[Top].id), "node must NOT hold the unfetched frontier")
|
||||
}
|
||||
|
||||
// TestRED_BehindNode_PeerConnectDelay_NeverFalseCompletesAtStale (variant C). A behind node boots
|
||||
|
||||
@@ -373,15 +373,18 @@ func (p *BootstrapPolicy) AcceptsFrontier(ctx context.Context, replies []BeaconR
|
||||
// last-ACCEPTED block, so an un-finalized N+1 a producer is merely processing is never reported
|
||||
// — the ±1 pending-tip skew cannot fake "ahead", and a producer one ACCEPTED block ahead
|
||||
// correctly defeats caught-up so the node syncs that block.)
|
||||
// - (c) the node HOLDS every reported tip — heightOf returns ok ONLY for a block the node has, so
|
||||
// a tip the node lacks (someone genuinely ahead, or a same-height sibling/fork it never
|
||||
// finalized) makes the conclusion fail. The node declares caught-up only to blocks it holds.
|
||||
// - (c) the node has ACCEPTED every reported tip — heightOf returns ok ONLY for a block on the
|
||||
// node's FINALIZED chain, so a tip the node lacks OR merely holds-in-store-but-has-not-accepted
|
||||
// (someone genuinely ahead, a gossiped-ahead block, or a same-height sibling/fork it never
|
||||
// finalized) makes the conclusion fail. The node declares caught-up only to blocks it ACCEPTED.
|
||||
//
|
||||
// heightOf resolves a tip's height from blocks the NODE HOLDS (ok=false when not held), injected so
|
||||
// the trust DECISION stays free of any VM/block dependency — the same separation as AncestrySource.
|
||||
// It is NEVER a network fetch: a tip the node lacks simply makes it not-caught-up (the safe
|
||||
// direction — it syncs). Because (c) requires the node to hold every reported tip, the heights (b)
|
||||
// compares are all read locally and are the blocks' canonical (content-addressed) heights.
|
||||
// heightOf resolves a tip's height from the node's ACCEPTED chain (ok=false when the tip is not
|
||||
// accepted — including a block merely PRESENT in the store but unaccepted, the luxd-2 freeze case),
|
||||
// injected so the trust DECISION stays free of any VM/block dependency — the same separation as
|
||||
// AncestrySource. It is NEVER a network fetch: an unaccepted/absent tip simply makes the node
|
||||
// not-caught-up (the safe direction — it syncs). Because (c) requires the node to have ACCEPTED
|
||||
// every reported tip, the heights (b) compares are the blocks' canonical (content-addressed)
|
||||
// heights read from the finalized chain — store presence can never fake "caught up".
|
||||
func (p *BootstrapPolicy) CaughtUp(replies []BeaconReply, lastAccepted uint64, heightOf func(ids.ID) (uint64, bool)) bool {
|
||||
responders, responderWeight, stakeOnTip, _ := p.tallyResponders(replies)
|
||||
if !p.floorMet(responders, responderWeight) {
|
||||
|
||||
@@ -161,7 +161,7 @@ func TestBootstrapTrust_A_MassRecoverySucceeds(t *testing.T) {
|
||||
require.NoError(t, runBS(t, bh), "mass-recovery node must converge")
|
||||
last, _ := vm.LastAccepted(ctx)
|
||||
require.Equal(t, chain[N].id, last, "RECOVERED: converged to the frontier N=%d despite 2 of 5 validators down", N)
|
||||
require.True(t, bh.Has(ctx, chain[N].id))
|
||||
require.True(t, bh.Accepted(ctx, chain[N].id))
|
||||
}
|
||||
|
||||
// ----- B: ONE-BEACON CAPTURE REJECTED ---------------------------------------
|
||||
|
||||
@@ -365,6 +365,12 @@ func (vm *pChainHeightVM) LastAccepted(ctx context.Context) (ids.ID, error) {
|
||||
return vm.inner.LastAccepted(ctx)
|
||||
}
|
||||
|
||||
// NOTE: this wrapper deliberately does NOT forward GetBlockIDAtHeight. The bootstrap acceptance
|
||||
// oracle's fork-sibling check reads the IN-PROCESS consensus finalized ledger
|
||||
// (blockHandler.finalizedBlockAtHeight → engine.FinalizedBlockAtHeight), NOT a VM height index —
|
||||
// because the VM index is dead over ZAP (the zap server has no MsgGetBlockIDAtHeight handler, so
|
||||
// the real C-Chain returns nothing). A forwarder here would only re-expose that dead path.
|
||||
|
||||
// SetPreference delegates to the inner VM.
|
||||
func (vm *pChainHeightVM) SetPreference(ctx context.Context, id ids.ID) error {
|
||||
return vm.inner.SetPreference(ctx, id)
|
||||
|
||||
@@ -26,7 +26,7 @@ require (
|
||||
github.com/huin/goupnp v1.3.0
|
||||
github.com/jackpal/gateway v1.1.1
|
||||
github.com/jackpal/go-nat-pmp v1.0.2
|
||||
github.com/luxfi/consensus v1.25.35
|
||||
github.com/luxfi/consensus v1.25.36
|
||||
github.com/luxfi/crypto v1.19.26
|
||||
github.com/luxfi/database v1.20.4
|
||||
github.com/luxfi/ids v1.2.15
|
||||
@@ -190,6 +190,7 @@ require (
|
||||
github.com/hanzos3/go-sdk v1.0.2 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/luxfi/age v1.5.0 // indirect
|
||||
github.com/luxfi/bft v0.1.5 // indirect
|
||||
github.com/luxfi/corona v0.7.9 // indirect
|
||||
github.com/luxfi/crypto/ipa v1.2.4 // indirect
|
||||
github.com/luxfi/kms v1.11.7 // indirect
|
||||
@@ -259,3 +260,10 @@ require (
|
||||
)
|
||||
|
||||
exclude github.com/ethereum/go-ethereum v1.10.26
|
||||
|
||||
// TEMPORARY — local-dev build aid for the bootstrap frozen-cache convergence fix.
|
||||
// FINAL CASCADE (publish step, NOT done here): tag consensus v1.25.36 (the uncommitted
|
||||
// engine/chain/integration.go FinalizedLedger + FinalizedBlockAtHeight accessors and the
|
||||
// engine/chain/bootstrap Has→Accepted change), bump the require above v1.25.35 → v1.25.36,
|
||||
// then DELETE this replace. The zap client (option b) is node-only and does NOT widen the
|
||||
// consensus bump.
|
||||
|
||||
@@ -304,6 +304,8 @@ github.com/luxfi/api v1.0.15 h1:Q5ox3Ompw/AZNMfB9wpHZosrj9C+ZldAEHKtHEotFdY=
|
||||
github.com/luxfi/api v1.0.15/go.mod h1:Eer59msIXMnOlFncG0XjEGH3TZML0Dd1bUu4GtB7f4Q=
|
||||
github.com/luxfi/atomic v1.0.0 h1:xUV60MuzRvXngaQ1sM0yVC2v4TRoLlUGkkH7M9PS4yw=
|
||||
github.com/luxfi/atomic v1.0.0/go.mod h1:0G2mTlQ6TXWHICUHrUUPu1/qAiIyR4gSZ2tva9ci/bI=
|
||||
github.com/luxfi/bft v0.1.5 h1:5xVLPkog4e5LTgaVlb9pgxA0EWE6tkrKwHPZVRz+RZw=
|
||||
github.com/luxfi/bft v0.1.5/go.mod h1:5I8Ft8yA69xZlDe3RB0i4MgbqFKLZe65o/sha8JuKvU=
|
||||
github.com/luxfi/cache v1.2.1 h1:kAzOS55/hmYeNKR+0HAKv4ma48Y6JjkI8UQeqdZ8bfI=
|
||||
github.com/luxfi/cache v1.2.1/go.mod h1:co7JTxZZHpKT31Yh01LFp5aZOxmoUg157FhBLQdQHVU=
|
||||
github.com/luxfi/chains v1.4.6 h1:+cptoH0TpcH4kVhwcWsdPP2h6Sz1fokCTo+hJJDtV4s=
|
||||
@@ -316,6 +318,8 @@ github.com/luxfi/concurrent v0.0.3 h1:eJyv1fhaC0jMLMw6+QS774cUmp7GK+ouMgvLCqnC7c
|
||||
github.com/luxfi/concurrent v0.0.3/go.mod h1:Aj/FR5NpM0cB2P4Nt3+tz9+dV6V+LUW4HuMgSjwq5hw=
|
||||
github.com/luxfi/consensus v1.25.35 h1:j/smfWj2LT0LKbaWGl7PYDTPY2fpVO3AE8535vvR9R4=
|
||||
github.com/luxfi/consensus v1.25.35/go.mod h1:cerfisfzmUJv8gbMcjcQUSdxlLpfn/+LdLY+4D95Nw0=
|
||||
github.com/luxfi/consensus v1.25.36 h1:tErdCx7FBY/JsoG2hY7HgPkpYGzDEZtgi7Vm56tTRd4=
|
||||
github.com/luxfi/consensus v1.25.36/go.mod h1:cerfisfzmUJv8gbMcjcQUSdxlLpfn/+LdLY+4D95Nw0=
|
||||
github.com/luxfi/constants v1.5.8 h1:iNP9AWNUcM4Tps7jYnx49CwtCWAC9mYRxJfGou2za0g=
|
||||
github.com/luxfi/constants v1.5.8/go.mod h1:Pu5jWHdnUtQRbWC43yTUjU/pbIIKMDOd2a2yroSfo48=
|
||||
github.com/luxfi/container v0.0.4 h1:BXhF82WyfqVP5mjlNcr7tP0Fcnvl0Ap1rkiu+rq5XuM=
|
||||
|
||||
@@ -39,7 +39,14 @@ type Client struct {
|
||||
conn *zapwire.Conn
|
||||
logger log.Logger
|
||||
|
||||
// Cached state from Initialize
|
||||
// lastAcceptedID caches the plugin's last-accepted block id. SEEDED at Initialize and
|
||||
// REFRESHED on every successful block Accept (setLastAccepted) so LastAccepted() honors the
|
||||
// block.ChainVM contract — return the ACTUAL last-accepted, not a frozen Initialize snapshot.
|
||||
// Before this refresh the cache froze for the process life: a fire-and-forget Accept advanced
|
||||
// the plugin (coreth on-disk) but never the cache, so GetAcceptedFrontier served a stale tip and
|
||||
// any consumer reading VM.LastAccepted was misled. Guarded by lastAcceptedMu because Accept (the
|
||||
// consensus accept goroutine) and LastAccepted (the network/bootstrap goroutines) race.
|
||||
lastAcceptedMu sync.RWMutex
|
||||
lastAcceptedID ids.ID
|
||||
|
||||
// dbServer is the ZAP-native rpcdb server spawned in Initialize that
|
||||
@@ -135,19 +142,29 @@ func (c *Client) Initialize(ctx context.Context, init block.Init) error {
|
||||
return fmt.Errorf("zap decode initialize response: %w", err)
|
||||
}
|
||||
|
||||
c.lastAcceptedID, err = ids.ToID(resp.LastAcceptedID)
|
||||
seedID, err := ids.ToID(resp.LastAcceptedID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.setLastAccepted(seedID)
|
||||
|
||||
c.logger.Info("VM initialized via ZAP",
|
||||
"height", resp.Height,
|
||||
"lastAcceptedID", c.lastAcceptedID,
|
||||
"lastAcceptedID", seedID,
|
||||
)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// setLastAccepted refreshes the cached last-accepted id under the write lock. Called at
|
||||
// Initialize (seed) and on every successful zapBlock.Accept so the cache tracks the plugin's
|
||||
// real accepted tip instead of freezing at the boot snapshot.
|
||||
func (c *Client) setLastAccepted(id ids.ID) {
|
||||
c.lastAcceptedMu.Lock()
|
||||
c.lastAcceptedID = id
|
||||
c.lastAcceptedMu.Unlock()
|
||||
}
|
||||
|
||||
// Shutdown implements chain.ChainVM
|
||||
func (c *Client) Shutdown(ctx context.Context) error {
|
||||
_, _, err := c.conn.Call(ctx, zapwire.MsgShutdown, nil)
|
||||
@@ -376,8 +393,11 @@ func (c *Client) SetPreference(ctx context.Context, blkID ids.ID) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// LastAccepted implements chain.ChainVM
|
||||
// LastAccepted implements chain.ChainVM. Returns the cache refreshed on every Accept (NOT a
|
||||
// frozen Initialize snapshot) under the read lock.
|
||||
func (c *Client) LastAccepted(ctx context.Context) (ids.ID, error) {
|
||||
c.lastAcceptedMu.RLock()
|
||||
defer c.lastAcceptedMu.RUnlock()
|
||||
return c.lastAcceptedID, nil
|
||||
}
|
||||
|
||||
@@ -643,8 +663,13 @@ func (b *zapBlock) Accept(ctx context.Context) error {
|
||||
defer zapwire.PutBuffer(buf)
|
||||
req.Encode(buf)
|
||||
|
||||
_, _, err := b.client.conn.Call(ctx, zapwire.MsgBlockAccept, buf.Bytes())
|
||||
return err
|
||||
if _, _, err := b.client.conn.Call(ctx, zapwire.MsgBlockAccept, buf.Bytes()); err != nil {
|
||||
return err
|
||||
}
|
||||
// Refresh the cache so LastAccepted() reflects this accept instead of freezing at the
|
||||
// Initialize snapshot. Only on SUCCESS — a failed accept did not advance the plugin.
|
||||
b.client.setLastAccepted(b.id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *zapBlock) Reject(ctx context.Context) error {
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
// Copyright (C) 2026, Lux Industries Inc. All rights reserved.
|
||||
// See the file LICENSE for licensing terms.
|
||||
|
||||
package zap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
zapwire "github.com/luxfi/api/zap"
|
||||
"github.com/luxfi/consensus/engine/chain/block"
|
||||
"github.com/luxfi/ids"
|
||||
"github.com/luxfi/log"
|
||||
luxruntime "github.com/luxfi/runtime"
|
||||
)
|
||||
|
||||
// TestLastAccepted_RefreshedOnAccept is the option-(b) regression guard: the ZAP client's
|
||||
// LastAccepted() must reflect the most-recently-ACCEPTED block, not a frozen Initialize snapshot.
|
||||
//
|
||||
// THE BUG (red HIGH-1 root): zapBlock.Accept fired the MsgBlockAccept wire call but never refreshed
|
||||
// c.lastAcceptedID, which was written ONLY at Initialize — so LastAccepted() returned the boot id
|
||||
// for the process life even as the plugin (coreth on-disk) advanced. Every consumer of
|
||||
// VM.LastAccepted was misled: GetAcceptedFrontier served a stale tip to peers, and the bootstrap
|
||||
// caught-up/height signals (before the option-a ledger decomplect) re-descended forever.
|
||||
//
|
||||
// This drives a real in-process ZAP server: Initialize seeds the cache, a successful Accept must
|
||||
// refresh it to the accepted id, and a FAILED Accept must leave it unchanged (a failed accept did
|
||||
// not advance the plugin).
|
||||
func TestLastAccepted_RefreshedOnAccept(t *testing.T) {
|
||||
seedID := ids.ID{0x5e, 0xed}
|
||||
acceptedID := ids.ID{0xac, 0xce, 0x97, 0xed}
|
||||
failID := ids.ID{0xfa, 0x11}
|
||||
|
||||
var failAccept bool
|
||||
addr, stop := startTestServer(t, zapwire.HandlerFunc(func(_ context.Context, msgType zapwire.MessageType, _ []byte) (zapwire.MessageType, []byte, error) {
|
||||
switch msgType {
|
||||
case zapwire.MsgInitialize:
|
||||
var zeroParent ids.ID
|
||||
resp := &zapwire.InitializeResponse{
|
||||
LastAcceptedID: seedID[:],
|
||||
LastAcceptedParentID: zeroParent[:],
|
||||
Height: 0,
|
||||
Bytes: []byte{},
|
||||
Timestamp: time.Now().UnixNano(),
|
||||
}
|
||||
buf := zapwire.GetBuffer()
|
||||
defer zapwire.PutBuffer(buf)
|
||||
resp.Encode(buf)
|
||||
out := make([]byte, len(buf.Bytes()))
|
||||
copy(out, buf.Bytes())
|
||||
return zapwire.MsgInitialize, out, nil
|
||||
case zapwire.MsgBlockAccept:
|
||||
if failAccept {
|
||||
return 0, nil, errors.New("synthesized accept failure")
|
||||
}
|
||||
return zapwire.MsgBlockAccept, []byte{}, nil
|
||||
default:
|
||||
return 0, nil, errors.New("unexpected message")
|
||||
}
|
||||
}))
|
||||
defer stop()
|
||||
|
||||
conn, err := zapwire.Dial(context.Background(), addr, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Dial: %v", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
c := NewClient(conn, log.NewNoOpLogger())
|
||||
if err := c.Initialize(context.Background(), block.Init{
|
||||
Runtime: &luxruntime.Runtime{NetworkID: 1337},
|
||||
Genesis: []byte("{}"),
|
||||
}); err != nil {
|
||||
t.Fatalf("Initialize: %v", err)
|
||||
}
|
||||
|
||||
// Seeded at Initialize.
|
||||
if got, _ := c.LastAccepted(context.Background()); got != seedID {
|
||||
t.Fatalf("after Initialize: LastAccepted = %s, want seed %s", got, seedID)
|
||||
}
|
||||
|
||||
// A successful Accept REFRESHES the cache (the fix — previously it stayed frozen at seed).
|
||||
if err := (&zapBlock{client: c, id: acceptedID}).Accept(context.Background()); err != nil {
|
||||
t.Fatalf("Accept: %v", err)
|
||||
}
|
||||
if got, _ := c.LastAccepted(context.Background()); got != acceptedID {
|
||||
t.Fatalf("after Accept: LastAccepted = %s, want accepted %s (cache did not refresh — the freeze)", got, acceptedID)
|
||||
}
|
||||
|
||||
// A FAILED Accept must NOT move the cache (the plugin did not advance).
|
||||
failAccept = true
|
||||
if err := (&zapBlock{client: c, id: failID}).Accept(context.Background()); err == nil {
|
||||
t.Fatal("expected the synthesized accept failure to surface")
|
||||
}
|
||||
if got, _ := c.LastAccepted(context.Background()); got != acceptedID {
|
||||
t.Fatalf("after FAILED Accept: LastAccepted = %s, want unchanged %s", got, acceptedID)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user