diff --git a/cmd/repair-proposervm/main.go b/cmd/repair-proposervm/main.go new file mode 100644 index 000000000..bfbc9fe3e --- /dev/null +++ b/cmd/repair-proposervm/main.go @@ -0,0 +1,352 @@ +// Copyright (C) 2019-2026, Lux Industries Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +// Command repair-proposervm is a ONE-TIME, fail-closed surgical tool to reconcile +// a proposervm chain-state DB onto a known-canonical outer block at a single height. +// +// Motivation (incident 1082814, Lux mainnet C-Chain): a sub-quorum proposervm +// envelope A (3-of-5 ACCEPT) was locally accepted on some nodes while the network +// finalized the supermajority sibling B (4-of-5) that wraps the IDENTICAL inner EVM +// block. The two siblings differ ONLY in the proposervm outer envelope; the inner +// EVM state is byte-identical (no EVM divergence). On restart those nodes re-seed +// finality from their persisted proposervm lastAccepted=A and fatal on B's cert +// (EQUIVOCATION). This tool swaps the persisted record of `height` from A to the +// canonical B, leaving the inner EVM completely untouched (no EVM rollback). +// +// It is NOT a blind hex edit: it opens the exact same typed proposervm state +// (luxfi/node/vms/proposervm/state) over the exact same nested keyspace luxd uses +// (chainID -> "vm" -> "proposervm" -> versiondb -> chain/block/height), and writes +// via the state's own PutBlock / SetBlockIDAtHeight / SetLastAccepted so the on-disk +// bytes are identical to what luxd itself wrote for B on the canonical node. +// +// proposervm invariant honored: proLastAcceptedHeight must never be < the inner VM's +// last-accepted height (vm.repairAcceptedChainByHeight). The inner EVM is at `height` +// (it accepted the shared inner block under A), so the recovery target is the +// canonical block AT `height` (B), never height-1 — keeping outer==inner height. +// +// Modes: +// +// inspect : read-only. Print lastAccepted, height index at H and H-1, and the +// outer block currently recorded at H. +// export : read-only. Read the outer block recorded at H and write its raw +// stateless bytes to --block-file (run against a canonical node's DB). +// repair : read-write, fail-closed. Parse --block-file (canonical B), assert it is +// the expected block, assert the DB is in the expected bad state (lastAccepted +// and heightIndex[H] both == the expected sub-quorum block A, and B's parent +// == heightIndex[H-1]); then PutBlock(B), SetBlockIDAtHeight(H,B), +// SetLastAccepted(B), Commit. Idempotent: if already on B, it no-ops. +// +// The DB uses an exclusive LOCK; luxd MUST be stopped on the target before `repair`. +package main + +import ( + "errors" + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/luxfi/database" + databasefactory "github.com/luxfi/database/factory" + "github.com/luxfi/database/prefixdb" + "github.com/luxfi/database/versiondb" + "github.com/luxfi/ids" + "github.com/luxfi/log" + "github.com/luxfi/metric" + + "github.com/luxfi/node/vms/proposervm/block" + "github.com/luxfi/node/vms/proposervm/state" +) + +// The proposervm nests its state under these fixed prefixes inside the chain DB. +// chainDB = prefixdb(chainID[:], baseDB) [chains.ChainDBManager.GetDatabase] +// vmDB = prefixdb("vm", chainDB) [chains.ChainDBManager.GetVMDatabase / VMDBPrefix] +// ppvmDB = versiondb(prefixdb("proposervm", vmDB)) [proposervm.VM.Initialize dbPrefix] +// state.New(ppvmDB) -> "chain"/"block"/"height" sub-prefixes +var ( + vmDBPrefix = []byte("vm") + proposervmDBPrefix = []byte("proposervm") +) + +var ( + dbPath string + dbType string + chainIDStr string + height uint64 + blockFile string + expectBlockID string // canonical B (target) + expectCurID string // sub-quorum A (the bad state we expect to overwrite) + yes bool +) + +func main() { + root := &cobra.Command{ + Use: "repair-proposervm", + Short: "Surgical, fail-closed reconcile of a proposervm chain-state onto a canonical outer block at one height", + } + root.PersistentFlags().StringVar(&dbPath, "db-path", "", "zapdb root (e.g. /data/db/mainnet/db) (required)") + root.PersistentFlags().StringVar(&dbType, "db-type", "zapdb", "database type") + root.PersistentFlags().StringVar(&chainIDStr, "chain-id", "2wRdZGeca1qkxzNCq88NWDF5nJ5A9o623vRJKd3FsjRYvuVvvt", "blockchain ID (proposervm chain)") + root.PersistentFlags().Uint64Var(&height, "height", 1082814, "contested height") + root.MarkPersistentFlagRequired("db-path") + + inspect := &cobra.Command{Use: "inspect", Short: "read-only: print proposervm finality state at the height", RunE: runInspect} + + export := &cobra.Command{Use: "export", Short: "read-only: write the outer block recorded at the height to --block-file", RunE: runExport} + export.Flags().StringVar(&blockFile, "block-file", "", "output file for the canonical outer block bytes (required)") + export.MarkFlagRequired("block-file") + + probe := &cobra.Command{Use: "probe", Short: "read-only: look up an arbitrary block by ID (is it present in the store?)", RunE: runProbe} + probe.Flags().StringVar(&expectBlockID, "block-id", "", "block ID to look up (required)") + probe.MarkFlagRequired("block-id") + + dump := &cobra.Command{Use: "dump", Short: "read-only: write an arbitrary block (by ID) raw stateless bytes to --block-file", RunE: runDump} + dump.Flags().StringVar(&expectBlockID, "block-id", "wDMUyGyaKcC2Vng8i8ngU5f83XEtHZx5hqCSe5tMTwLAPagmo", "block ID to dump (canonical B)") + dump.Flags().StringVar(&blockFile, "block-file", "", "output file for the block bytes (required)") + dump.MarkFlagRequired("block-file") + + repair := &cobra.Command{Use: "repair", Short: "fail-closed: swap height's outer block from A to canonical B", RunE: runRepair} + repair.Flags().StringVar(&blockFile, "block-file", "", "canonical outer block (B) bytes, from `export` (required)") + repair.Flags().StringVar(&expectBlockID, "expect-block", "wDMUyGyaKcC2Vng8i8ngU5f83XEtHZx5hqCSe5tMTwLAPagmo", "expected canonical block ID (B)") + repair.Flags().StringVar(&expectCurID, "expect-current", "2U2pR3DHCNEFDLnMq2uraNVkThRWgETDd468hR26yGHBQuAnNy", "expected current sub-quorum block ID (A) to be overwritten") + repair.Flags().BoolVar(&yes, "yes", false, "confirm the write") + repair.MarkFlagRequired("block-file") + + root.AddCommand(inspect, export, probe, dump, repair) + if err := root.Execute(); err != nil { + fmt.Fprintln(os.Stderr, "ERROR:", err) + os.Exit(1) + } +} + +// openState opens the proposervm typed state over the exact nested keyspace luxd uses. +// Returns the state, the proposervm versiondb (for Commit), and the base DB (for Close). +func openState(readOnly bool) (state.State, *versiondb.Database, database.Database, ids.ID, error) { + chainID, err := ids.FromString(chainIDStr) + if err != nil { + return nil, nil, nil, ids.Empty, fmt.Errorf("bad chain-id: %w", err) + } + logger := log.New("cmd", "repair-proposervm") + gatherer := metric.NewRegistry() + base, err := databasefactory.New(dbType, dbPath, readOnly, nil, gatherer, logger, "repair", "db") + if err != nil { + return nil, nil, nil, ids.Empty, fmt.Errorf("open db %q: %w", dbPath, err) + } + chainDB := prefixdb.New(chainID[:], base) + vmDB := prefixdb.New(vmDBPrefix, chainDB) + ppvmDB := versiondb.New(prefixdb.New(proposervmDBPrefix, vmDB)) + return state.New(ppvmDB), ppvmDB, base, chainID, nil +} + +func idAt(st state.State, h uint64) string { + id, err := st.GetBlockIDAtHeight(h) + if errors.Is(err, database.ErrNotFound) { + return "" + } + if err != nil { + return "" + } + return id.String() +} + +func runInspect(_ *cobra.Command, _ []string) error { + st, _, base, _, err := openState(true) + if err != nil { + return err + } + defer base.Close() + la, err := st.GetLastAccepted() + if err != nil { + return fmt.Errorf("GetLastAccepted: %w", err) + } + fmt.Printf("proposervm lastAccepted = %s\n", la) + fmt.Printf("heightIndex[%d] = %s\n", height, idAt(st, height)) + fmt.Printf("heightIndex[%d] = %s\n", height-1, idAt(st, height-1)) + if id, err := st.GetBlockIDAtHeight(height); err == nil { + if blk, err := st.GetBlock(id); err == nil { + fmt.Printf("block@%d: id=%s parent=%s bytes=%d\n", height, blk.ID(), blk.ParentID(), len(blk.Bytes())) + } + } + return nil +} + +func runProbe(_ *cobra.Command, _ []string) error { + want, err := ids.FromString(expectBlockID) + if err != nil { + return fmt.Errorf("bad --block-id: %w", err) + } + // RW so Badger replays its WAL: a verified/built-but-unflushed block may live + // only in the memtable. Disposable copy only. + st, _, base, _, err := openState(false) + if err != nil { + return err + } + defer base.Close() + blk, err := st.GetBlock(want) + if errors.Is(err, database.ErrNotFound) { + fmt.Printf("NOT-PRESENT: block %s is not in this store\n", want) + return nil + } + if err != nil { + return fmt.Errorf("GetBlock(%s): %w", want, err) + } + fmt.Printf("PRESENT: id=%s parent=%s bytes=%d\n", blk.ID(), blk.ParentID(), len(blk.Bytes())) + return nil +} + +func runDump(_ *cobra.Command, _ []string) error { + want, err := ids.FromString(expectBlockID) + if err != nil { + return fmt.Errorf("bad --block-id: %w", err) + } + // RW so Badger replays its WAL: a verified-but-unflushed block may live only in + // the memtable. We never call a state WRITER here (read + write output file only), + // and this runs against an idle (luxd-stopped) pod DB; the contested sibling B was + // verified by this node when it saw the conflicting cert, so it is present by ID. + st, _, base, _, err := openState(false) + if err != nil { + return err + } + defer base.Close() + blk, err := st.GetBlock(want) + if errors.Is(err, database.ErrNotFound) { + return fmt.Errorf("block %s is NOT present in this store (cannot dump)", want) + } + if err != nil { + return fmt.Errorf("GetBlock(%s): %w", want, err) + } + if blk.ID() != want { + return fmt.Errorf("self-ID mismatch: requested=%s parsed=%s (refusing)", want, blk.ID()) + } + if err := os.WriteFile(blockFile, blk.Bytes(), 0o644); err != nil { + return fmt.Errorf("write %s: %w", blockFile, err) + } + fmt.Printf("DUMPED id=%s parent=%s bytes=%d -> %s\n", blk.ID(), blk.ParentID(), len(blk.Bytes()), blockFile) + return nil +} + +func runExport(_ *cobra.Command, _ []string) error { + // Open read-WRITE so Badger replays its value-log/WAL: the canonical block at + // the contested height was the LAST write before the chain went idle, so on a + // crash-consistent snapshot copy it may live only in the memtable/WAL, not yet + // in an SST. A read-only open skips recovery and could miss it. We never call a + // state writer here (we only read + write the output file), and this only ever + // runs against a DISPOSABLE snapshot copy of a canonical node — never the live node. + st, _, base, _, err := openState(false) + if err != nil { + return err + } + defer base.Close() + id, err := st.GetBlockIDAtHeight(height) + if err != nil { + return fmt.Errorf("GetBlockIDAtHeight(%d): %w", height, err) + } + blk, err := st.GetBlock(id) + if err != nil { + return fmt.Errorf("GetBlock(%s): %w", id, err) + } + if blk.ID() != id { + return fmt.Errorf("block id mismatch: index=%s block=%s", id, blk.ID()) + } + if err := os.WriteFile(blockFile, blk.Bytes(), 0o644); err != nil { + return fmt.Errorf("write %s: %w", blockFile, err) + } + la, _ := st.GetLastAccepted() + fmt.Printf("EXPORTED height=%d id=%s parent=%s bytes=%d -> %s\n", height, blk.ID(), blk.ParentID(), len(blk.Bytes()), blockFile) + fmt.Printf(" (source lastAccepted=%s heightIndex[%d-1]=%s)\n", la, height, idAt(st, height-1)) + return nil +} + +func runRepair(_ *cobra.Command, _ []string) error { + wantB, err := ids.FromString(expectBlockID) + if err != nil { + return fmt.Errorf("bad --expect-block: %w", err) + } + wantA, err := ids.FromString(expectCurID) + if err != nil { + return fmt.Errorf("bad --expect-current: %w", err) + } + raw, err := os.ReadFile(blockFile) + if err != nil { + return fmt.Errorf("read %s: %w", blockFile, err) + } + blk, err := block.ParseWithoutVerification(raw) + if err != nil { + return fmt.Errorf("parse block file: %w", err) + } + // (1) the supplied block must be exactly the canonical target B. + if blk.ID() != wantB { + return fmt.Errorf("block-file id %s != --expect-block %s (refusing)", blk.ID(), wantB) + } + + st, ppvmDB, base, _, err := openState(false) + if err != nil { + return err + } + defer base.Close() + + la, err := st.GetLastAccepted() + if err != nil { + return fmt.Errorf("GetLastAccepted: %w", err) + } + curAtH, err := st.GetBlockIDAtHeight(height) + if err != nil { + return fmt.Errorf("GetBlockIDAtHeight(%d): %w", height, err) + } + + // (2) idempotency: if already on B, do nothing. + if la == wantB && curAtH == wantB { + fmt.Printf("ALREADY-CANONICAL: lastAccepted and heightIndex[%d] already == B (%s); no-op\n", height, wantB) + return nil + } + + // (3) fail-closed: only proceed from the exact expected bad state (A at H, lastAccepted A). + if la != wantA { + return fmt.Errorf("refusing: lastAccepted=%s is neither A(%s) nor B(%s) — unexpected state", la, wantA, wantB) + } + if curAtH != wantA { + return fmt.Errorf("refusing: heightIndex[%d]=%s != A(%s) — unexpected state", height, curAtH, wantA) + } + + // (4) B must extend the SAME finalized prefix: B.parent == the block recorded at H-1. + parentAtH1, err := st.GetBlockIDAtHeight(height - 1) + if err != nil { + return fmt.Errorf("GetBlockIDAtHeight(%d): %w", height-1, err) + } + if blk.ParentID() != parentAtH1 { + return fmt.Errorf("refusing: B.parent=%s != heightIndex[%d]=%s (B does not extend the local finalized prefix)", blk.ParentID(), height-1, parentAtH1) + } + if _, err := st.GetBlock(parentAtH1); err != nil { + return fmt.Errorf("refusing: parent %s (height %d) not present in store: %w", parentAtH1, height-1, err) + } + + fmt.Printf("PLAN: height=%d %s (A) -> %s (B)\n", height, wantA, wantB) + fmt.Printf(" current lastAccepted = %s\n", la) + fmt.Printf(" B.parent = %s == heightIndex[%d] OK\n", blk.ParentID(), height-1) + if !yes { + return errors.New("dry-run only: re-run with --yes to write") + } + + // (5) apply via the state's own typed writers (identical on-disk bytes to luxd). + if err := st.PutBlock(blk); err != nil { + return fmt.Errorf("PutBlock(B): %w", err) + } + if err := st.SetBlockIDAtHeight(height, blk.ID()); err != nil { + return fmt.Errorf("SetBlockIDAtHeight(%d,B): %w", height, err) + } + if err := st.SetLastAccepted(blk.ID()); err != nil { + return fmt.Errorf("SetLastAccepted(B): %w", err) + } + if err := ppvmDB.Commit(); err != nil { + return fmt.Errorf("commit: %w", err) + } + + // (6) re-read to confirm. + la2, _ := st.GetLastAccepted() + fmt.Printf("DONE: lastAccepted=%s heightIndex[%d]=%s heightIndex[%d]=%s\n", la2, height, idAt(st, height), height-1, idAt(st, height-1)) + if la2 != wantB || idAt(st, height) != wantB.String() { + return fmt.Errorf("post-write verification FAILED") + } + fmt.Println("VERIFIED: proposervm now records canonical B at the contested height; inner EVM untouched.") + return nil +} diff --git a/vms/proposervm/height_repair_test.go b/vms/proposervm/height_repair_test.go index df354eb97..69b4895df 100644 --- a/vms/proposervm/height_repair_test.go +++ b/vms/proposervm/height_repair_test.go @@ -5,53 +5,54 @@ package proposervm import "testing" -// TestPlanHeightRepair locks the init-time reconciliation policy between the -// proposervm finality index and the inner VM's accepted tip. The load-bearing -// case is repairResetBehindIndex: proposervm BEHIND the inner (e.g. the devnet-C -// "index 7 < inner 8" truncated snapshot) must map to a recoverable reset, NOT -// the pre-fix behavior that returned a fatal init error and bricked the whole -// chain. A regression back to fatal would have to delete this case and fail here. -func TestPlanHeightRepair(t *testing.T) { +// TestClassifyHeightRepair locks the init-time reconciliation policy between the +// proposervm finality index and the inner VM's accepted tip. +// +// The load-bearing case is heightBehind: proposervm BELOW the inner (e.g. the +// devnet-C "index 7 < inner 8" from a snapshot restored inconsistently across +// the proposervm and EVM databases). It MUST classify as heightBehind — which +// the caller turns into a LOUD, actionable fatal — and must NEVER be treated as +// heightMatch (a no-op that leaves the node inconsistent) or heightAhead (a +// rollback). It must in particular never be "self-healed" by dropping the +// finality pointer: that leaves proposervm.LastAccepted() in the inner-id +// namespace and permanently wedges bootstrap/catch-up/live at the inner tip. +// A regression back to that silent reset would have to reclassify this case and +// fail here. +func TestClassifyHeightRepair(t *testing.T) { tests := []struct { - name string - proHeight, innerHeight, fork uint64 - want repairAction + name string + proHeight, innerHeight uint64 + want heightRelation }{ - {"heights match => nothing", 8, 8, 0, repairNone}, - {"heights match at genesis", 0, 0, 0, repairNone}, + {"heights match => nothing", 8, 8, heightMatch}, + {"heights match at genesis", 0, 0, heightMatch}, - // The bug-3 case: proposervm index behind the inner tip must self-heal - // (reset + re-bootstrap), never fatally brick init. - {"behind by one (devnet-C 7<8)", 7, 8, 0, repairResetBehindIndex}, - {"behind by many", 100, 4242, 10, repairResetBehindIndex}, - {"behind, fork above inner still resets", 5, 9, 20, repairResetBehindIndex}, + // The bug-3 case: proposervm index behind the inner tip is unrecoverable + // locally and must be a loud fatal, never a silent reset or a rollback. + {"behind by one (devnet-C 7<8)", 7, 8, heightBehind}, + {"behind by many", 100, 4242, heightBehind}, + {"behind at genesis boundary", 0, 1, heightBehind}, - // Proposervm ahead: roll back to the inner height when the target is - // at/above the fork. - {"ahead, fork below inner => rollback", 9, 7, 3, repairRollBackToInner}, - {"ahead by one, fork at genesis", 9, 8, 0, repairRollBackToInner}, - {"ahead, fork equals inner => rollback", 9, 7, 7, repairRollBackToInner}, - - // Proposervm ahead but the rollback target is below the fork => forget - // all proposervm indices (pre-fork territory). - {"ahead, fork above inner => forget past fork", 9, 5, 7, repairForgetPastFork}, - {"ahead, rollback below fork", 100, 10, 11, repairForgetPastFork}, + // Proposervm ahead: the inner rolled back; roll the proposervm back. + {"ahead by one", 9, 8, heightAhead}, + {"ahead by many", 4242, 100, heightAhead}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := planHeightRepair(tt.proHeight, tt.innerHeight, tt.fork) + got := classifyHeightRepair(tt.proHeight, tt.innerHeight) if got != tt.want { - t.Fatalf("planHeightRepair(%d,%d,%d) = %d, want %d", - tt.proHeight, tt.innerHeight, tt.fork, got, tt.want) + t.Fatalf("classifyHeightRepair(%d,%d) = %d, want %d", + tt.proHeight, tt.innerHeight, got, tt.want) } }) } - // Explicit safety assertion: the behind-index case must NEVER resolve to a - // no-op or a rollback (both of which would leave the node inconsistent or, - // as before, crash) — it must always be the reset action. - if got := planHeightRepair(7, 8, 0); got != repairResetBehindIndex { - t.Fatalf("behind-index (7<8) must self-heal via repairResetBehindIndex, got %d", got) + // Explicit safety assertion: the behind-index case must be heightBehind (loud + // fatal), NEVER heightMatch (silent inconsistency) or heightAhead (rollback). + // This is the exact regression RED flagged: a silent finality-pointer reset + // here wedges the node forever. + if got := classifyHeightRepair(7, 8); got != heightBehind { + t.Fatalf("behind-index (7<8) must classify heightBehind (loud fatal), got %d", got) } } diff --git a/vms/proposervm/vm.go b/vms/proposervm/vm.go index 4dd9bbb19..307a17619 100644 --- a/vms/proposervm/vm.go +++ b/vms/proposervm/vm.go @@ -618,43 +618,47 @@ func (vm *VM) CreateHandlers(ctx context.Context) (map[string]http.Handler, erro return handlers, nil } -// repairAction is the reconciliation the proposervm applies at init when its -// finality index and the inner VM's accepted tip disagree in height. -type repairAction int +// heightRelation classifies how the proposervm finality index relates to the +// inner VM's accepted tip at init. It is the PURE part of the reconciliation and +// deliberately does NOT depend on the fork height (only the AHEAD case needs the +// fork height, and it is read lazily in that branch so the other paths gain no +// new failure mode). +type heightRelation int const ( - // repairNone: the proposervm and inner heights match; nothing to do. - repairNone repairAction = iota - // repairRollBackToInner: the proposervm is AHEAD of the inner (and the - // rollback target is at/above the fork); roll the proposervm index back to - // the inner's height. - repairRollBackToInner - // repairForgetPastFork: the proposervm is ahead but the rollback target is - // BELOW the fork; drop all proposervm indices (pre-fork territory). - repairForgetPastFork - // repairResetBehindIndex: the proposervm index is BEHIND the inner tip — a - // truncated/inconsistent on-disk state (e.g. a partially-restored snapshot). - // The proposervm cannot fabricate the missing wrapper blocks, so drop the - // stale finality pointer and re-bootstrap rather than fatally bricking init. - repairResetBehindIndex + // heightMatch: proposervm and inner heights are equal; nothing to repair. + heightMatch heightRelation = iota + // heightAhead: the proposervm is AHEAD of the inner — the inner rolled back + // (or state-synced behind); the proposervm index is rolled back to the inner + // height (or, if the target is below the fork, forgotten entirely). + heightAhead + // heightBehind: the proposervm index is BEHIND the inner tip. This is an + // on-disk inconsistency (e.g. a snapshot restored inconsistently across the + // proposervm and inner-EVM databases). It is UNRECOVERABLE LOCALLY: the + // proposervm cannot fabricate the missing outer wrapper blocks for the heights + // (pro, inner], and it must NOT silently drop its finality pointer — doing so + // leaves proposervm.LastAccepted() reporting an INNER-namespace id whose + // ParentID is contiguity-incompatible with the network's OUTER wrappers, which + // permanently wedges bootstrap/catch-up/live at the inner tip (blocks at + // height <= tip are skipped, so the missing wrapper is never rebuilt). The + // only correct remedy is operator action (restore a consistent snapshot or + // full resync), so init fails LOUD with an actionable runbook instead. + heightBehind ) -// planHeightRepair is the PURE reconciliation decision for -// repairAcceptedChainByHeight: given the proposervm's last-accepted height, the -// inner VM's last-accepted height, and the proposervm fork height, it returns -// the action to apply. Separating this decision from the DB effects keeps it -// deterministically testable and makes the behind-index policy (which replaced a -// node-bricking fatal) explicit and regression-locked. -func planHeightRepair(proHeight, innerHeight, forkHeight uint64) repairAction { +// classifyHeightRepair is the PURE, deterministically-testable reconciliation +// decision. Keeping the behind-index case explicit here regression-locks the +// invariant that a behind index is treated as unrecoverable-locally (a LOUD +// fatal), never as a silent finality-pointer reset — a reset creates a silent +// permanent wedge that is strictly worse than the loud crash it would replace. +func classifyHeightRepair(proHeight, innerHeight uint64) heightRelation { switch { case proHeight == innerHeight: - return repairNone + return heightMatch case proHeight < innerHeight: - return repairResetBehindIndex - case forkHeight > innerHeight: - return repairForgetPastFork - default: // proHeight > innerHeight and forkHeight <= innerHeight - return repairRollBackToInner + return heightBehind + default: // proHeight > innerHeight + return heightAhead } } @@ -706,53 +710,56 @@ func (vm *VM) repairAcceptedChainByHeight(ctx context.Context) error { proLastAcceptedHeight := proLastAccepted.Height() innerLastAcceptedHeight := innerLastAccepted.Height() - // The fork height only matters when the proposervm is AHEAD (rolling back), - // but the state is already initialized here (we read a last-accepted above), - // so reading it up front is safe and keeps the reconciliation decision pure. + switch classifyHeightRepair(proLastAcceptedHeight, innerLastAcceptedHeight) { + case heightMatch: + // Heights match — nothing to repair. + return nil + + case heightBehind: + // INVARIANT VIOLATION and UNRECOVERABLE LOCALLY: the proposervm's finality + // index sits BELOW the inner VM's accepted tip. In a correct system this + // never happens — the proposervm last-accepted pointer and height index + // commit in the SAME versiondb batch as every inner accept, so they cannot + // lag. Reaching here means the on-disk proposervm state was truncated + // relative to the inner EVM — e.g. a snapshot restored inconsistently across + // the two databases (the devnet-C "index 7 < inner 8"). + // + // We FAIL LOUD rather than "self-heal", because there is no correct local + // heal: the proposervm cannot fabricate the missing outer wrapper blocks for + // heights (pro, inner]. In particular, dropping the finality pointer + // (DeleteLastAccepted) is NOT a heal — proposervm.LastAccepted() would then + // fall back to the inner-namespace id (see LastAccepted), whose ParentID is + // contiguity-incompatible with the network's OUTER wrappers, permanently + // wedging bootstrap (the first-block anchor), catch-up (the parent==tip + // guard) and live Verify (the parent lookup) at the inner tip — and since + // every path skips blocks at height <= the tip, the missing wrapper is never + // rebuilt. That silent wedge is strictly worse than this loud, actionable + // stop. The correct remedy is operator action; surface it explicitly. + return fmt.Errorf( + "proposervm finality index (height %d, id %s) is BEHIND the inner VM tip (height %d, id %s): "+ + "the on-disk proposervm state is truncated/inconsistent relative to the inner EVM "+ + "(e.g. a snapshot restored inconsistently across the proposervm and EVM databases). "+ + "This cannot be repaired locally — the proposervm cannot rebuild the missing outer wrapper "+ + "blocks. RECOVERY: restore a snapshot that is consistent across BOTH databases, or fully "+ + "resync this node from peers (wipe this chain's db and re-bootstrap). Refusing to auto-reset "+ + "the finality pointer, which would silently wedge this node at the inner tip forever", + proLastAcceptedHeight, proLastAcceptedID, innerLastAcceptedHeight, innerLastAcceptedID, + ) + } + + // heightAhead: the inner vm is BEHIND the proposer vm (the inner rolled back or + // state-synced behind), so roll the proposervm index back to the inner height. + // The fork height is only needed here, so read it lazily — the match/behind + // paths above never touch it, and so cannot gain a new failure mode from it. forkHeight, err := vm.State.GetForkHeight() if err != nil { return fmt.Errorf("failed to get fork height: %w", err) } - switch planHeightRepair(proLastAcceptedHeight, innerLastAcceptedHeight, forkHeight) { - case repairNone: - // Heights match — nothing to repair. - return nil - - case repairResetBehindIndex: - // INVARIANT VIOLATION, but a RECOVERABLE one: the proposervm's finality - // index sits BELOW the inner VM's accepted tip. In a correct system this - // never happens (the proposervm's last-accepted and height index commit in - // the SAME versiondb batch as every inner accept, so they cannot lag), so - // reaching here means the on-disk proposervm state was truncated relative - // to the inner — e.g. a crash-inconsistent or partially-restored snapshot - // (the devnet-C "index 7 < inner 8" brick). The proposervm cannot fabricate - // the missing outer wrapper blocks for heights (pro, inner], so it CANNOT - // heal forward locally. - // - // Previously this was a FATAL init error that bricked the WHOLE chain - // ("error creating required chain" -> exit 1), turning a recoverable - // index/snapshot inconsistency on ONE node into a hard outage that needed - // manual surgery. Instead, drop the stale proposervm finality pointer and - // let the node re-bootstrap: consensus re-fetches the canonical blocks (with - // their finality certs) from healthy peers through the catch-up transport - // and re-accepts them, rebuilding the proposervm index consistently on top - // of the inner tip. This is the SAME recovery repairForgetPastFork performs; - // here we reach it because the index is behind, not ahead. Loud + actionable - // so operators see exactly what healed and why. - vm.logger.Warn("proposervm finality index is behind the inner VM tip; resetting the proposervm index to re-bootstrap from peers (recoverable snapshot/index inconsistency, NOT a crash)", - log.Uint64("proposervmHeight", proLastAcceptedHeight), - log.Uint64("innerHeight", innerLastAcceptedHeight), - log.Stringer("innerLastAcceptedID", innerLastAcceptedID), - ) - if err := vm.State.DeleteLastAccepted(); err != nil { - return fmt.Errorf("failed to reset proposervm last accepted while healing behind-index: %w", err) - } - return vm.db.Commit() - - case repairForgetPastFork: - // We are rolling back past the fork, so we should just forget about all - // of our proposervm indices. + if forkHeight > innerLastAcceptedHeight { + // We are rolling back past the fork, so we should just forget about all of + // our proposervm indices. The inner tip is BELOW the fork, so it is a + // pre-fork block and proposervm.LastAccepted() correctly falls back to it. vm.logger.Info("repairing accepted chain by height: rolling back past the proposervm fork", log.Uint64("outerHeight", proLastAcceptedHeight), log.Uint64("innerHeight", innerLastAcceptedHeight), @@ -764,8 +771,6 @@ func (vm *VM) repairAcceptedChainByHeight(ctx context.Context) error { return vm.db.Commit() } - // repairRollBackToInner: the inner vm is behind the proposer vm, so roll the - // proposervm back to the inner's height. vm.logger.Info("repairing accepted chain by height", log.Uint64("outerHeight", proLastAcceptedHeight), log.Uint64("innerHeight", innerLastAcceptedHeight), diff --git a/vms/rpcchainvm/zap/client.go b/vms/rpcchainvm/zap/client.go index d10270f81..7ad5bc601 100644 --- a/vms/rpcchainvm/zap/client.go +++ b/vms/rpcchainvm/zap/client.go @@ -29,6 +29,15 @@ import ( var ( ErrNotConnected = errors.New("zap: not connected") ErrInvalidResponse = errors.New("zap: invalid response") + // errMalformedBlockID is returned when the VM plugin sends a block-id field + // over the ZAP boundary whose length is not exactly ids.IDLen (32 bytes). + // An empty (0-length) field is the plugin's way of signalling that it could + // NOT resolve the block — e.g. the requested block does not connect to this + // node's accepted chain (a fork / missing-parent condition). We surface that + // as this explicit typed error instead of the opaque "invalid hash length" + // from ids.ToID, and we NEVER coerce it to ids.Empty (a 32-byte zero id is a + // legitimate value; a 0-length field is not). + errMalformedBlockID = errors.New("zap: plugin returned malformed block id (does not connect to accepted chain)") ) // Compile-time check that Client implements chain.ChainVM @@ -142,7 +151,7 @@ func (c *Client) Initialize(ctx context.Context, init block.Init) error { return fmt.Errorf("zap decode initialize response: %w", err) } - seedID, err := ids.ToID(resp.LastAcceptedID) + seedID, err := blockIDFromZAP("lastAcceptedID", resp.LastAcceptedID) if err != nil { return err } @@ -262,6 +271,18 @@ func (c *Client) Version(ctx context.Context) (string, error) { return resp.Version, nil } +// blockIDFromZAP converts a block-id field returned by the VM plugin over the +// ZAP boundary into an ids.ID, guarding the malformed/empty case. A well-formed +// plugin always returns exactly ids.IDLen bytes; any other length (notably the +// 0-length "could not resolve" signal) yields errMalformedBlockID rather than +// the opaque ids.ToID "invalid hash length" error, and never a coerced zero id. +func blockIDFromZAP(field string, b []byte) (ids.ID, error) { + if len(b) != ids.IDLen { + return ids.Empty, fmt.Errorf("%w: %s was %d bytes, want %d", errMalformedBlockID, field, len(b), ids.IDLen) + } + return ids.ToID(b) +} + // BuildBlock implements chain.ChainVM func (c *Client) BuildBlock(ctx context.Context) (block.Block, error) { _, respData, err := c.conn.Call(ctx, zapwire.MsgBuildBlock, nil) @@ -278,11 +299,11 @@ func (c *Client) BuildBlock(ctx context.Context) (block.Block, error) { return nil, errorFromZAP(resp.Err) } - id, err := ids.ToID(resp.ID) + id, err := blockIDFromZAP("id", resp.ID) if err != nil { return nil, err } - parentID, err := ids.ToID(resp.ParentID) + parentID, err := blockIDFromZAP("parentID", resp.ParentID) if err != nil { return nil, err } @@ -321,11 +342,11 @@ func (c *Client) ParseBlock(ctx context.Context, blockBytes []byte) (block.Block return nil, errorFromZAP(resp.Err) } - id, err := ids.ToID(resp.ID) + id, err := blockIDFromZAP("id", resp.ID) if err != nil { return nil, err } - parentID, err := ids.ToID(resp.ParentID) + parentID, err := blockIDFromZAP("parentID", resp.ParentID) if err != nil { return nil, err } @@ -364,7 +385,7 @@ func (c *Client) GetBlock(ctx context.Context, blkID ids.ID) (block.Block, error return nil, errorFromZAP(resp.Err) } - parentID, err := ids.ToID(resp.ParentID) + parentID, err := blockIDFromZAP("parentID", resp.ParentID) if err != nil { return nil, err } @@ -592,7 +613,7 @@ func (c *Client) GetBlockIDAtHeight(ctx context.Context, height uint64) (ids.ID, return ids.Empty, errorFromZAP(resp.Err) } - blkID, err := ids.ToID(resp.BlkID) + blkID, err := blockIDFromZAP("blkID", resp.BlkID) if err != nil { return ids.Empty, err } diff --git a/vms/rpcchainvm/zap/client_parseblock_test.go b/vms/rpcchainvm/zap/client_parseblock_test.go new file mode 100644 index 000000000..a688104ec --- /dev/null +++ b/vms/rpcchainvm/zap/client_parseblock_test.go @@ -0,0 +1,155 @@ +// Copyright (C) 2026, Lux Industries Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package zap + +import ( + "context" + "errors" + "testing" + "time" + + zapwire "github.com/luxfi/api/zap" + "github.com/luxfi/consensus/engine/chain/block" + "github.com/luxfi/ids" + "github.com/luxfi/log" +) + +// TestParseBlock_MalformedBlockID_IsTypedAndQuarantined is the regression guard +// for the mainnet luxd-3 wedge (C-Chain height 1082797 fork). +// +// THE OBSERVED SYMPTOM: luxd-3 spammed +// +// warn ParseBlock failed, cannot vote correctly +// error="invalid hash length: expected 32 bytes but got 0" +// +// ROOT of that surfacing: the C-Chain EVM (coreth) runs as an rpcchainvm plugin +// across the ZAP boundary. When luxd-3 (sitting on a divergent, already-accepted +// fork block at 1082797) is asked via PushQuery to ParseBlock a CANONICAL block +// that does not connect to its forked chain, the plugin answers with a +// BlockResponse whose ID/ParentID are EMPTY (0-length) and whose Err is left at +// ErrorUnspecified. The old client passed that empty slice straight into +// ids.ToID, which returned the opaque "invalid hash length: expected 32 bytes +// but got 0" — masking the real "does-not-connect" condition. +// +// The wire codec (github.com/luxfi/api/zap BlockResponse) genuinely admits a +// 0-length ID/ParentID: both are length-prefixed []byte read via ReadBytes, and +// Err defaults to ErrorUnspecified, so an empty-id response slips past the +// `resp.Err != ErrorUnspecified` guard. This test reproduces that exact wire +// shape (not a hand-fabricated convenience) and asserts the hardened behavior: +// +// - a 0-length (or any non-32-byte) id yields the explicit typed +// errMalformedBlockID, NOT the opaque ids.ToID error; +// - ParseBlock returns a nil block (no state advance, no zero-id coercion); +// - ParseBlock never panics on the malformed field; +// - a well-formed 32-byte response — INCLUDING a 32-zero-byte ParentID, which +// is the legitimate ids.Empty value and must NOT be confused with the +// 0-length malformed case — still parses cleanly. +// +// Run under -race. +func TestParseBlock_MalformedBlockID_IsTypedAndQuarantined(t *testing.T) { + goodID := ids.ID{0x11, 0x22, 0x33} // a valid, non-empty 32-byte id + goodParent := ids.ID{0xaa, 0xbb, 0xcc} // a valid, non-empty 32-byte parent + zeroParent := ids.Empty // 32 ZERO bytes — a legitimate id value + + cases := []struct { + name string + resp *zapwire.BlockResponse + wantErr bool + }{ + { + // The literal luxd-3 wedge: empty id, no error code. + name: "empty id with no error code (the luxd-3 wedge shape)", + resp: &zapwire.BlockResponse{ID: nil, ParentID: goodParent[:], Bytes: []byte{0xde, 0xad}, Err: zapwire.ErrorUnspecified}, + wantErr: true, + }, + { + name: "empty parentID with no error code", + resp: &zapwire.BlockResponse{ID: goodID[:], ParentID: nil, Bytes: []byte{0xde, 0xad}, Err: zapwire.ErrorUnspecified}, + wantErr: true, + }, + { + // Malleability: any non-32 length must be rejected, not truncated/padded. + name: "short 5-byte id is rejected (length malleability guard)", + resp: &zapwire.BlockResponse{ID: []byte{1, 2, 3, 4, 5}, ParentID: goodParent[:], Bytes: []byte{0xde, 0xad}, Err: zapwire.ErrorUnspecified}, + wantErr: true, + }, + { + // Well-formed: 32-byte id + 32-ZERO-byte parent (legit ids.Empty) parses. + name: "well-formed 32-byte ids, zero-but-32-byte parent parses cleanly", + resp: &zapwire.BlockResponse{ID: goodID[:], ParentID: zeroParent[:], Bytes: []byte{0xde, 0xad}, Height: 7, Timestamp: time.Now().UnixNano(), Err: zapwire.ErrorUnspecified}, + wantErr: false, + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + c := newParseBlockTestClient(t, tc.resp) + + var ( + blk block.Block + err error + ) + // Must never panic regardless of the malformed field. + func() { + defer func() { + if r := recover(); r != nil { + t.Fatalf("ParseBlock panicked on malformed response: %v", r) + } + }() + blk, err = c.ParseBlock(context.Background(), []byte{0xde, 0xad}) + }() + + if tc.wantErr { + if !errors.Is(err, errMalformedBlockID) { + t.Fatalf("want typed errMalformedBlockID, got %v", err) + } + if blk != nil { + t.Fatalf("malformed id must yield a nil block (no state advance / no zero-id coercion), got %v", blk) + } + return + } + if err != nil { + t.Fatalf("well-formed response must parse, got err: %v", err) + } + if blk == nil { + t.Fatal("well-formed response must yield a non-nil block") + } + if blk.ID() != goodID { + t.Fatalf("block id = %s, want %s", blk.ID(), goodID) + } + if blk.Parent() != zeroParent { + t.Fatalf("parent id = %s, want %s (the legitimate 32-byte zero id)", blk.Parent(), zeroParent) + } + }) + } +} + +// newParseBlockTestClient spins an in-process ZAP server whose MsgParseBlock +// handler returns the supplied BlockResponse verbatim — modelling exactly what +// a VM plugin (coreth over rpcchainvm) puts on the wire — and returns a Client +// connected to it. +func newParseBlockTestClient(t *testing.T, resp *zapwire.BlockResponse) *Client { + t.Helper() + addr, stop := startTestServer(t, zapwire.HandlerFunc(func(_ context.Context, msgType zapwire.MessageType, _ []byte) (zapwire.MessageType, []byte, error) { + if msgType != zapwire.MsgParseBlock { + return 0, nil, errors.New("unexpected message") + } + buf := zapwire.GetBuffer() + defer zapwire.PutBuffer(buf) + resp.Encode(buf) + out := make([]byte, len(buf.Bytes())) + copy(out, buf.Bytes()) + return zapwire.MsgParseBlock, out, nil + })) + t.Cleanup(stop) + + conn, err := zapwire.Dial(context.Background(), addr, nil) + if err != nil { + t.Fatalf("Dial: %v", err) + } + t.Cleanup(func() { _ = conn.Close() }) + + return NewClient(conn, log.NewNoOpLogger()) +}