mirror of
https://github.com/luxfi/node.git
synced 2026-07-27 03:39:39 +00:00
fix(platformvm): accrue P-chain validator uptime for a stable set (reward gate)
All 5 mainnet validators read uptime=0.0000 / connected=null even for peers
connected 24h+, so prefersCommit (block/executor/options.go) always saw
0% < threshold and withheld staking rewards (~165M LUX gate).
Root causes (four, all fixed):
1. StartTracking never existed/called. The custom uptimeTracker was created
late (onReady) and never baselined validator records, so a long-running
validator's stored upDuration stayed 0 and CalculateUptimePercentFrom
returned 0/total = 0.
2. upDuration flushed only on Disconnect/Shutdown, so a continuously-connected
validator's persisted uptime never grew.
3. service.go left `connected` hard-nil ("IsConnected no longer exists").
4. ROOT: VM.Connected never fired. network.Connected -> chainRouter.Connected
only added to a connectedPeers set and logged; it never dispatched to chain
handlers, and blockHandler.Connected was a no-op. So tracker.Connect was
never called and the connected map was always empty — nothing to accrue.
Fix (faithful port of avalanchego snow/uptime.Manager semantics, luxfi pkgs):
- vms/platformvm/uptime_tracker.go: rewrite as a startedTracking-gated tracker.
StartTracking/StopTracking/StartedTracking/IsConnected added; CalculateUptime
folds the live connected session forward to now using the persisted
lastUpdated (reconstructed from the second-granular uptime.State encoding), so
a connected validator accrues uptime WITHOUT a Disconnect. Before tracking, a
validator is assumed online since its last update (avalanchego baseline);
after tracking, only genuine sessions accrue. Second-granular clock matches
storage. CalculateUptimePercentFrom is the clean upDuration/(now-from) form,
clamped to [0,1].
- vms/platformvm/vm.go: create+register the tracker at Initialize (so bootstrap
Connect events are captured), StartTracking(primary validators) at onReady,
StopTracking on re-bootstrap and Shutdown. Mirrors avalanchego's
onNormalOperationsStarted lifecycle.
- vms/platformvm/service.go: populate `connected` via tracker.IsConnected.
- node/chain_router.go: chainRouter.Connected/Disconnected now dispatch to every
registered chain handler (snapshot under lock, call outside).
- chains/manager.go: blockHandler forwards Connected/Disconnected to its chain's
VM (engineVM) exactly once (dedup set), so the P-chain uptime tracker — and
every VM's peer set — finally observes connectivity.
Consensus safety: prefersCommit is a per-node PREFERENCE feeding oracle-block
voting, not a deterministic value. Fixing the local uptime measurement (which
was uniformly 0) changes only which reward option each node prefers; consensus
still converges by preference voting. No staking/reward math changed.
Tests (vms/platformvm/uptime_tracker_test.go, -race green):
- TestUptimeTrackerLongRunningValidatorAccruesUptime: a 30-day validator reports
~100%. Uses only the shared Calculator API; run against the OLD tracker it
returns 0.0 (FAIL — the exact mainnet symptom), against the fix 1.0 (PASS).
- Continuously-connected-climbs, StartTracking-baselines, flush-on-disconnect,
never-connected-gets-zero, StopTracking-flushes, idempotent double-connect,
dedup, concurrency. block/executor (reward gate) suite green.
Follow-on (NOT in this commit): devnet 5-node uptime-climb verification ->
gated release -> owner-gated one-at-a-time mainnet roll. No image built, no
deploy, no live validator touched.
Co-authored-by: Hanzo Dev <dev@hanzo.ai>
This commit is contained in:
+57
-4
@@ -1704,7 +1704,10 @@ func (m *manager) buildChain(chainParams ChainParameters, sb nets.Net) (*chainIn
|
||||
// through (blockBuilder == the P-chain-height wrapper on K>1, the inner VM
|
||||
// on K==1), so the container bytes it parses match the bytes the engine
|
||||
// framed — one codec, no raw-vs-wrapped split.
|
||||
bh := newBlockHandler(blockBuilder, m.Log, consensusEngine, m.Net, m.MsgCreator, chainParams.ID, networkID, beacons, m.NodeID, expectsStakedBeacons)
|
||||
// engineVM is the connectable VM (the proposervm on a wrapped chain, the
|
||||
// inner VM otherwise); it carries Connected/Disconnected, which the router
|
||||
// dispatches so the P-chain uptime tracker observes validator connectivity.
|
||||
bh := newBlockHandler(blockBuilder, engineVM, m.Log, consensusEngine, m.Net, m.MsgCreator, chainParams.ID, networkID, beacons, m.NodeID, expectsStakedBeacons)
|
||||
// Gate this native chain's bootstrap frontier-TRUST on the P-chain having finished its
|
||||
// initial sync, so the staked beacon set (and thus the stake-majority floor denominator)
|
||||
// is the TRUE full validator set, not a partial mid-replay set. Wired ONLY for native
|
||||
@@ -2844,6 +2847,15 @@ type blockHandler struct {
|
||||
chainID ids.ID // Chain ID for message routing
|
||||
networkID ids.ID // Network ID for validator routing
|
||||
|
||||
// connector receives peer connect/disconnect notifications routed from the
|
||||
// node's chainRouter and forwards them to this chain's VM (e.g. the P-chain
|
||||
// uptime tracker; other VMs use them for their own peer sets). connectedNodes
|
||||
// dedups delivery so a peer the router dispatches once per tracked network is
|
||||
// forwarded to the VM exactly once. A nil connector makes forwarding a no-op.
|
||||
connector chain.ChainVM
|
||||
connMu sync.Mutex
|
||||
connectedNodes set.Set[ids.NodeID]
|
||||
|
||||
// Context sync support - when a block fails verification due to missing context,
|
||||
// we request the prerequisite blocks from the peer to catch up
|
||||
pendingContext map[ids.ID]contextRequest // Map from blockID to pending context request
|
||||
@@ -3018,9 +3030,10 @@ type contextRequest struct {
|
||||
timestamp time.Time
|
||||
}
|
||||
|
||||
func newBlockHandler(vm consensuschain.BlockBuilder, logger log.Logger, engine *consensuschain.Runtime, net network.Network, msgCreator message.OutboundMsgBuilder, chainID ids.ID, networkID ids.ID, beacons validators.Manager, selfNodeID ids.NodeID, expectsStakedBeacons bool) *blockHandler {
|
||||
func newBlockHandler(vm consensuschain.BlockBuilder, connector chain.ChainVM, logger log.Logger, engine *consensuschain.Runtime, net network.Network, msgCreator message.OutboundMsgBuilder, chainID ids.ID, networkID ids.ID, beacons validators.Manager, selfNodeID ids.NodeID, expectsStakedBeacons bool) *blockHandler {
|
||||
return &blockHandler{
|
||||
vm: vm,
|
||||
connector: connector,
|
||||
logger: logger,
|
||||
engine: engine,
|
||||
net: net,
|
||||
@@ -3033,6 +3046,7 @@ func newBlockHandler(vm consensuschain.BlockBuilder, logger log.Logger, engine *
|
||||
pendingContext: make(map[ids.ID]contextRequest),
|
||||
maxContextBlocks: 256, // Default max context blocks to request/serve
|
||||
pendingQbits: make(map[ids.ID][]QbitEvent),
|
||||
connectedNodes: set.NewSet[ids.NodeID](16),
|
||||
bsAncestorCh: make(map[uint32]chan [][]byte),
|
||||
}
|
||||
}
|
||||
@@ -4051,8 +4065,47 @@ func (b *blockHandler) GetStateSummary(ctx context.Context, nodeID ids.NodeID, r
|
||||
func (b *blockHandler) StateSummary(ctx context.Context, nodeID ids.NodeID, requestID uint32, summary []byte) error {
|
||||
return nil
|
||||
}
|
||||
func (b *blockHandler) Connected(ctx context.Context, nodeID ids.NodeID) error { return nil }
|
||||
func (b *blockHandler) Disconnected(ctx context.Context, nodeID ids.NodeID) error { return nil }
|
||||
// Connected forwards a peer connection to this chain's VM exactly once. The
|
||||
// P-chain VM records it in its uptime tracker; other VMs use it for their own
|
||||
// peer sets. A nil connector makes this a no-op. On a forwarding error the dedup
|
||||
// entry is rolled back so a subsequent dispatch of the same connection retries.
|
||||
func (b *blockHandler) Connected(ctx context.Context, nodeID ids.NodeID) error {
|
||||
if b.connector == nil {
|
||||
return nil
|
||||
}
|
||||
b.connMu.Lock()
|
||||
if b.connectedNodes.Contains(nodeID) {
|
||||
b.connMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
b.connectedNodes.Add(nodeID)
|
||||
b.connMu.Unlock()
|
||||
|
||||
// The node's p2p layer ignores the version argument; pass nil.
|
||||
if err := b.connector.Connected(ctx, nodeID, nil); err != nil {
|
||||
b.connMu.Lock()
|
||||
b.connectedNodes.Remove(nodeID)
|
||||
b.connMu.Unlock()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Disconnected forwards a peer disconnection to this chain's VM exactly once.
|
||||
func (b *blockHandler) Disconnected(ctx context.Context, nodeID ids.NodeID) error {
|
||||
if b.connector == nil {
|
||||
return nil
|
||||
}
|
||||
b.connMu.Lock()
|
||||
if !b.connectedNodes.Contains(nodeID) {
|
||||
b.connMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
b.connectedNodes.Remove(nodeID)
|
||||
b.connMu.Unlock()
|
||||
|
||||
return b.connector.Disconnected(ctx, nodeID)
|
||||
}
|
||||
func (b *blockHandler) HealthCheck(ctx context.Context) (interface{}, error) { return nil, nil }
|
||||
func (b *blockHandler) Stop(ctx context.Context) {
|
||||
if b.pollerCancel != nil {
|
||||
|
||||
+35
-4
@@ -195,24 +195,55 @@ func (r *chainRouter) AddChain(ctx context.Context, chainID ids.ID, h handler.Ha
|
||||
|
||||
func (r *chainRouter) Connected(nodeID ids.NodeID, nodeVersion *version.Application, netID ids.ID) {
|
||||
r.lock.Lock()
|
||||
defer r.lock.Unlock()
|
||||
|
||||
r.connectedPeers.Add(nodeID)
|
||||
handlers := make([]handler.Handler, 0, len(r.chains))
|
||||
for _, h := range r.chains {
|
||||
handlers = append(handlers, h)
|
||||
}
|
||||
r.lock.Unlock()
|
||||
|
||||
r.log.Debug("peer connected",
|
||||
log.Stringer("nodeID", nodeID),
|
||||
log.Any("version", nodeVersion),
|
||||
log.Stringer("netID", netID),
|
||||
)
|
||||
|
||||
// Deliver the connection to every chain handler so VMs observe peer
|
||||
// connectivity — the P-chain uptime tracker in particular. Handlers dedup, so
|
||||
// repeated dispatch of the same connection (once per tracked network) is
|
||||
// safe. Dispatch OUTSIDE the router lock: a handler must never re-enter the
|
||||
// router while we hold it.
|
||||
for _, h := range handlers {
|
||||
if err := h.Connected(context.Background(), nodeID); err != nil {
|
||||
r.log.Debug("chain handler Connected failed",
|
||||
log.Stringer("nodeID", nodeID),
|
||||
log.Err(err),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *chainRouter) Disconnected(nodeID ids.NodeID) {
|
||||
r.lock.Lock()
|
||||
defer r.lock.Unlock()
|
||||
|
||||
r.connectedPeers.Remove(nodeID)
|
||||
handlers := make([]handler.Handler, 0, len(r.chains))
|
||||
for _, h := range r.chains {
|
||||
handlers = append(handlers, h)
|
||||
}
|
||||
r.lock.Unlock()
|
||||
|
||||
r.log.Debug("peer disconnected",
|
||||
log.Stringer("nodeID", nodeID),
|
||||
)
|
||||
|
||||
for _, h := range handlers {
|
||||
if err := h.Disconnected(context.Background(), nodeID); err != nil {
|
||||
r.log.Debug("chain handler Disconnected failed",
|
||||
log.Stringer("nodeID", nodeID),
|
||||
log.Err(err),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *chainRouter) Benched(chainID ids.ID, nodeID ids.NodeID) {
|
||||
|
||||
@@ -77,7 +77,7 @@ var (
|
||||
const (
|
||||
defaultMajor = 1
|
||||
defaultMinor = 36
|
||||
defaultPatch = 13
|
||||
defaultPatch = 26
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
||||
@@ -926,11 +926,12 @@ func (s *Service) getPrimaryOrNetValidators(netID ids.ID, nodeIDs set.Set[ids.No
|
||||
// Transform this to a percentage (0-100) to make it consistent
|
||||
// with observedUptime in info.peers API
|
||||
currentUptime := avajson.Float32(rawUptime * 100)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// connected field left nil - IsConnected method no longer exists
|
||||
uptime = ¤tUptime
|
||||
|
||||
// Report whether this validator currently has a live connection
|
||||
// to us, read from the same tracker that measured its uptime.
|
||||
isConnected := s.vm.tracker != nil && s.vm.tracker.IsConnected(currentStaker.NodeID)
|
||||
connected = &isConnected
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
+208
-112
@@ -4,91 +4,222 @@
|
||||
package platformvm
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/luxfi/database"
|
||||
"github.com/luxfi/ids"
|
||||
"github.com/luxfi/validators/uptime"
|
||||
)
|
||||
|
||||
// uptimeTracker implements uptime.Calculator by tracking peer connections
|
||||
// and computing real uptime percentages from persistent state.
|
||||
var (
|
||||
errAlreadyStartedTracking = errors.New("uptime tracker already started tracking")
|
||||
errNotStartedTracking = errors.New("uptime tracker has not started tracking")
|
||||
)
|
||||
|
||||
// uptimeTracker implements uptime.Calculator plus the connection/tracking hooks
|
||||
// the platform VM drives directly (Connect, Disconnect, StartTracking,
|
||||
// StopTracking, IsConnected).
|
||||
//
|
||||
// It wraps an uptime.State (backed by platformvm state) that stores
|
||||
// cumulative upDuration and lastUpdated per validator. On Connect, it
|
||||
// records the connection time. On Disconnect, it flushes the elapsed
|
||||
// connected time into the persistent state. CalculateUptimePercent
|
||||
// reads from state and accounts for any currently-connected time.
|
||||
// It is a faithful port of avalanchego's snow/uptime.Manager, adapted to the
|
||||
// luxfi validators uptime.State interface (which keys by netID and returns
|
||||
// lastUpdated encoded as a second-granular duration since the Unix epoch).
|
||||
//
|
||||
// Model, in one sentence: each connected peer's session accrues into a per
|
||||
// validator up-duration that is folded forward to "now" on read, and persisted
|
||||
// on flush.
|
||||
//
|
||||
// - Connect records a peer's connection time. It is captured from the very
|
||||
// first handshake — including during bootstrap, before StartTracking — so a
|
||||
// stable, continuously-connected validator set is fully observed the moment
|
||||
// tracking begins.
|
||||
// - StartTracking, called once at P-chain normal-operations start, baselines
|
||||
// every validator (crediting the pre-tracking window as online, matching
|
||||
// avalanchego) and switches into live-tracking mode.
|
||||
// - CalculateUptime folds the currently-connected session forward to now, so a
|
||||
// validator that stays connected accrues up-duration WITHOUT ever needing a
|
||||
// Disconnect.
|
||||
// - Disconnect / StopTracking flush the accrued session into persistent state.
|
||||
//
|
||||
// The prior custom tracker could not accrue uptime for a stable set: it never
|
||||
// started tracking, only flushed on Disconnect, and — because peer connection
|
||||
// events were never delivered to the VM — never populated its connected map.
|
||||
type uptimeTracker struct {
|
||||
mu sync.RWMutex
|
||||
clk func() time.Time
|
||||
state uptime.State
|
||||
netID ids.ID
|
||||
connected map[ids.NodeID]time.Time // nodeID -> time they connected
|
||||
mu sync.RWMutex
|
||||
clk func() time.Time
|
||||
state uptime.State
|
||||
netID ids.ID
|
||||
|
||||
// connections maps a currently-connected nodeID to the (second-granular)
|
||||
// time it connected.
|
||||
connections map[ids.NodeID]time.Time
|
||||
|
||||
// startedTracking gates live-session accounting. Before StartTracking, a
|
||||
// validator is assumed online since its last persisted update (so the
|
||||
// bootstrap window is not spuriously counted as downtime). After
|
||||
// StartTracking, only genuinely-connected sessions accrue.
|
||||
startedTracking bool
|
||||
}
|
||||
|
||||
func newUptimeTracker(state uptime.State, netID ids.ID, clk func() time.Time) *uptimeTracker {
|
||||
return &uptimeTracker{
|
||||
clk: clk,
|
||||
state: state,
|
||||
netID: netID,
|
||||
connected: make(map[ids.NodeID]time.Time),
|
||||
clk: clk,
|
||||
state: state,
|
||||
netID: netID,
|
||||
connections: make(map[ids.NodeID]time.Time),
|
||||
}
|
||||
}
|
||||
|
||||
// Connect records that a validator connected.
|
||||
// now returns the tracker clock truncated to second precision. lastUpdated is
|
||||
// persisted at second granularity (state stores lastUpdated.Unix()), so working
|
||||
// at one resolution keeps the interval arithmetic exact and side-steps
|
||||
// monotonic-clock skew.
|
||||
func (t *uptimeTracker) now() time.Time {
|
||||
return time.Unix(t.clk().Unix(), 0)
|
||||
}
|
||||
|
||||
// lastUpdatedFromDuration reconstructs the persisted lastUpdated timestamp from
|
||||
// the second-granular "duration since epoch" the uptime.State returns.
|
||||
func lastUpdatedFromDuration(d time.Duration) time.Time {
|
||||
return time.Unix(int64(d/time.Second), 0)
|
||||
}
|
||||
|
||||
// Connect records that [nodeID] connected. It is idempotent: a duplicate Connect
|
||||
// for an already-connected node keeps the original connection time, so repeated
|
||||
// router dispatch of the same live connection can never reset the session clock.
|
||||
func (t *uptimeTracker) Connect(nodeID ids.NodeID) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if _, ok := t.connected[nodeID]; ok {
|
||||
return // already connected
|
||||
if _, ok := t.connections[nodeID]; ok {
|
||||
return
|
||||
}
|
||||
t.connected[nodeID] = t.clk()
|
||||
t.connections[nodeID] = t.now()
|
||||
}
|
||||
|
||||
// Disconnect records that a validator disconnected and flushes
|
||||
// the accumulated uptime into persistent state.
|
||||
// IsConnected reports whether [nodeID] currently has a live connection.
|
||||
func (t *uptimeTracker) IsConnected(nodeID ids.NodeID) bool {
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
_, connected := t.connections[nodeID]
|
||||
return connected
|
||||
}
|
||||
|
||||
// Disconnect records that [nodeID] disconnected, flushing its accrued session
|
||||
// into persistent state. Flushing is a no-op before StartTracking (the session
|
||||
// is not yet being measured), matching avalanchego.
|
||||
func (t *uptimeTracker) Disconnect(nodeID ids.NodeID) error {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
return t.disconnectLocked(nodeID)
|
||||
}
|
||||
|
||||
func (t *uptimeTracker) disconnectLocked(nodeID ids.NodeID) error {
|
||||
connectedAt, ok := t.connected[nodeID]
|
||||
if !ok {
|
||||
return nil // wasn't connected
|
||||
}
|
||||
delete(t.connected, nodeID)
|
||||
now := t.clk()
|
||||
elapsed := now.Sub(connectedAt)
|
||||
return t.addUptime(nodeID, elapsed, now)
|
||||
}
|
||||
|
||||
// addUptime adds elapsed duration to the validator's persistent uptime.
|
||||
func (t *uptimeTracker) addUptime(nodeID ids.NodeID, elapsed time.Duration, now time.Time) error {
|
||||
upDuration, _, err := t.state.GetUptime(nodeID, t.netID)
|
||||
if err != nil {
|
||||
// Validator not in state (e.g., not a current validator). Skip.
|
||||
defer delete(t.connections, nodeID)
|
||||
if !t.startedTracking {
|
||||
return nil
|
||||
}
|
||||
return t.state.SetUptime(nodeID, t.netID, upDuration+elapsed, now)
|
||||
return t.updateUptimeLocked(nodeID)
|
||||
}
|
||||
|
||||
// Shutdown flushes all connected validators' uptime to state.
|
||||
// Call this before persisting state on node shutdown.
|
||||
func (t *uptimeTracker) Shutdown() error {
|
||||
// StartTracking baselines every validator in [nodeIDs] and switches into live
|
||||
// tracking mode. It is called once at P-chain normal-operations start. Each
|
||||
// validator's persisted up-duration is advanced to now assuming it was online
|
||||
// since its last update (the standard avalanchego assumption), so a validator
|
||||
// that has been in the set is not penalized for the un-measured pre-tracking
|
||||
// window.
|
||||
func (t *uptimeTracker) StartTracking(nodeIDs []ids.NodeID) error {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
for nodeID := range t.connected {
|
||||
if err := t.disconnectLocked(nodeID); err != nil {
|
||||
if t.startedTracking {
|
||||
return errAlreadyStartedTracking
|
||||
}
|
||||
for _, nodeID := range nodeIDs {
|
||||
if err := t.updateUptimeLocked(nodeID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
t.startedTracking = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// CalculateUptime returns (upDuration, totalDuration, error) for a validator.
|
||||
// StopTracking flushes every validator in [nodeIDs] and leaves tracking mode.
|
||||
// It is called on the normal-ops → bootstrapping / shutdown transition so the
|
||||
// connected sessions are durably persisted before the node stops measuring.
|
||||
func (t *uptimeTracker) StopTracking(nodeIDs []ids.NodeID) error {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if !t.startedTracking {
|
||||
return errNotStartedTracking
|
||||
}
|
||||
for _, nodeID := range nodeIDs {
|
||||
if err := t.updateUptimeLocked(nodeID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
t.startedTracking = false
|
||||
return nil
|
||||
}
|
||||
|
||||
// StartedTracking reports whether StartTracking has run and StopTracking has not
|
||||
// since. The VM lifecycle uses it to avoid double-start and to gate shutdown
|
||||
// flushing.
|
||||
func (t *uptimeTracker) StartedTracking() bool {
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
return t.startedTracking
|
||||
}
|
||||
|
||||
// calculateUptimeLocked mirrors avalanchego snow/uptime.Manager.CalculateUptime:
|
||||
// it returns [nodeID]'s up-duration folded forward to now, plus that now (the
|
||||
// new lastUpdated). The caller must hold t.mu (read or write).
|
||||
func (t *uptimeTracker) calculateUptimeLocked(nodeID ids.NodeID) (time.Duration, time.Time, error) {
|
||||
upDuration, lastUpdatedDur, err := t.state.GetUptime(nodeID, t.netID)
|
||||
if err != nil {
|
||||
return 0, time.Time{}, err
|
||||
}
|
||||
lastUpdated := lastUpdatedFromDuration(lastUpdatedDur)
|
||||
now := t.now()
|
||||
|
||||
// Clock skew: never subtract time or double-count.
|
||||
if now.Before(lastUpdated) {
|
||||
return upDuration, lastUpdated, nil
|
||||
}
|
||||
|
||||
// Before tracking, assume the node was online since its last update.
|
||||
if !t.startedTracking {
|
||||
return upDuration + now.Sub(lastUpdated), now, nil
|
||||
}
|
||||
|
||||
// Tracking, but not connected: offline since its last update.
|
||||
connectedAt, isConnected := t.connections[nodeID]
|
||||
if !isConnected {
|
||||
return upDuration, now, nil
|
||||
}
|
||||
|
||||
// Tracking and connected: credit from the later of (connect, lastUpdated) so
|
||||
// no interval is double-counted.
|
||||
if connectedAt.Before(lastUpdated) {
|
||||
connectedAt = lastUpdated
|
||||
}
|
||||
if now.Before(connectedAt) {
|
||||
return upDuration, now, nil
|
||||
}
|
||||
return upDuration + now.Sub(connectedAt), now, nil
|
||||
}
|
||||
|
||||
// updateUptimeLocked persists the current up-duration for [nodeID]. A node
|
||||
// without a state record (i.e. not a current validator) is silently skipped, so
|
||||
// tracking non-validator peers is harmless. The caller must hold t.mu (write).
|
||||
func (t *uptimeTracker) updateUptimeLocked(nodeID ids.NodeID) error {
|
||||
upDuration, lastUpdated, err := t.calculateUptimeLocked(nodeID)
|
||||
if errors.Is(err, database.ErrNotFound) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return t.state.SetUptime(nodeID, t.netID, upDuration, lastUpdated)
|
||||
}
|
||||
|
||||
// CalculateUptime returns (upDuration, totalDuration) for [nodeID], where
|
||||
// totalDuration is the maximum possible uptime since the validator's start.
|
||||
func (t *uptimeTracker) CalculateUptime(nodeID ids.NodeID, netID ids.ID) (time.Duration, time.Duration, error) {
|
||||
if netID != t.netID {
|
||||
return 0, 0, nil
|
||||
@@ -99,97 +230,62 @@ func (t *uptimeTracker) CalculateUptime(nodeID ids.NodeID, netID ids.ID) (time.D
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
now := t.clk()
|
||||
totalDuration := now.Sub(startTime)
|
||||
if totalDuration <= 0 {
|
||||
return 0, 0, nil
|
||||
}
|
||||
|
||||
upDuration, _, err := t.state.GetUptime(nodeID, netID)
|
||||
if err != nil {
|
||||
return 0, totalDuration, nil
|
||||
}
|
||||
|
||||
// Add any currently-connected time that hasn't been flushed yet.
|
||||
t.mu.RLock()
|
||||
if connectedAt, ok := t.connected[nodeID]; ok {
|
||||
upDuration += now.Sub(connectedAt)
|
||||
}
|
||||
upDuration, _, err := t.calculateUptimeLocked(nodeID)
|
||||
t.mu.RUnlock()
|
||||
|
||||
if upDuration > totalDuration {
|
||||
upDuration = totalDuration
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
return upDuration, totalDuration, nil
|
||||
|
||||
total := t.now().Sub(startTime)
|
||||
if total < 0 {
|
||||
total = 0
|
||||
}
|
||||
return upDuration, total, nil
|
||||
}
|
||||
|
||||
// CalculateUptimePercent returns the uptime as a fraction in [0, 1].
|
||||
// CalculateUptimePercent returns [nodeID]'s uptime over its whole staking period
|
||||
// as a fraction in [0, 1].
|
||||
func (t *uptimeTracker) CalculateUptimePercent(nodeID ids.NodeID, netID ids.ID) (float64, error) {
|
||||
if netID != t.netID {
|
||||
return 0, nil
|
||||
}
|
||||
upDuration, totalDuration, err := t.CalculateUptime(nodeID, netID)
|
||||
startTime, err := t.state.GetStartTime(nodeID, netID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if totalDuration == 0 {
|
||||
return 1, nil // no time elapsed, consider 100%
|
||||
}
|
||||
return float64(upDuration) / float64(totalDuration), nil
|
||||
return t.CalculateUptimePercentFrom(nodeID, netID, startTime)
|
||||
}
|
||||
|
||||
// CalculateUptimePercentFrom returns the uptime as a fraction since [from].
|
||||
// CalculateUptimePercentFrom returns [nodeID]'s uptime since [from] as a fraction
|
||||
// in [0, 1].
|
||||
func (t *uptimeTracker) CalculateUptimePercentFrom(nodeID ids.NodeID, netID ids.ID, from time.Time) (float64, error) {
|
||||
if netID != t.netID {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
now := t.clk()
|
||||
totalDuration := now.Sub(from)
|
||||
if totalDuration <= 0 {
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
upDuration, _, err := t.state.GetUptime(nodeID, netID)
|
||||
if err != nil {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// Subtract uptime before [from] by using startTime.
|
||||
// If from > startTime, some of the stored upDuration may predate [from].
|
||||
// We approximate by assuming the same uptime rate.
|
||||
startTime, err := t.state.GetStartTime(nodeID, netID)
|
||||
if err != nil {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
totalSinceStart := now.Sub(startTime)
|
||||
if totalSinceStart <= 0 {
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
// Add any currently-connected time.
|
||||
t.mu.RLock()
|
||||
if connectedAt, ok := t.connected[nodeID]; ok {
|
||||
upDuration += now.Sub(connectedAt)
|
||||
}
|
||||
upDuration, _, err := t.calculateUptimeLocked(nodeID)
|
||||
t.mu.RUnlock()
|
||||
|
||||
if upDuration > totalSinceStart {
|
||||
upDuration = totalSinceStart
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Scale upDuration to the [from, now] window.
|
||||
if from.After(startTime) {
|
||||
rate := float64(upDuration) / float64(totalSinceStart)
|
||||
return rate, nil
|
||||
bestPossible := t.now().Sub(from)
|
||||
if bestPossible <= 0 {
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
// from <= startTime, just use overall rate.
|
||||
return float64(upDuration) / float64(totalSinceStart), nil
|
||||
fraction := float64(upDuration) / float64(bestPossible)
|
||||
if fraction > 1 {
|
||||
fraction = 1
|
||||
}
|
||||
return fraction, nil
|
||||
}
|
||||
|
||||
// SetCalculator is a no-op; this tracker doesn't delegate.
|
||||
// SetCalculator is a no-op: this tracker is a concrete Calculator and does not
|
||||
// delegate. It exists only to satisfy uptime.Calculator; registration is done by
|
||||
// the enclosing uptime.LockedCalculator.
|
||||
func (t *uptimeTracker) SetCalculator(ids.ID, uptime.Calculator) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -8,11 +8,17 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/luxfi/ids"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/luxfi/database"
|
||||
"github.com/luxfi/ids"
|
||||
)
|
||||
|
||||
// fakeUptimeState implements uptime.State for testing.
|
||||
// fakeUptimeState implements uptime.State for testing, mirroring how the real
|
||||
// platformvm state stores uptime: an up-duration plus a second-granular
|
||||
// lastUpdated, returned as a "duration since the Unix epoch". Validators must be
|
||||
// registered first; GetUptime/SetUptime on an unregistered node return
|
||||
// database.ErrNotFound, exactly like metadata_validator.go.
|
||||
type fakeUptimeState struct {
|
||||
uptimes map[ids.NodeID]time.Duration
|
||||
lastUpdate map[ids.NodeID]time.Time
|
||||
@@ -27,8 +33,7 @@ func newFakeUptimeState() *fakeUptimeState {
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeUptimeState) addValidator(nodeID ids.NodeID, netID ids.ID, startTime time.Time) {
|
||||
_ = netID
|
||||
func (f *fakeUptimeState) addValidator(nodeID ids.NodeID, _ ids.ID, startTime time.Time) {
|
||||
f.uptimes[nodeID] = 0
|
||||
f.lastUpdate[nodeID] = startTime
|
||||
f.startTimes[nodeID] = startTime
|
||||
@@ -37,7 +42,7 @@ func (f *fakeUptimeState) addValidator(nodeID ids.NodeID, netID ids.ID, startTim
|
||||
func (f *fakeUptimeState) GetUptime(nodeID ids.NodeID, _ ids.ID) (time.Duration, time.Duration, error) {
|
||||
up, ok := f.uptimes[nodeID]
|
||||
if !ok {
|
||||
return 0, 0, errNotFound
|
||||
return 0, 0, database.ErrNotFound
|
||||
}
|
||||
lastUpdatedDuration := time.Duration(f.lastUpdate[nodeID].Unix()) * time.Second
|
||||
return up, lastUpdatedDuration, nil
|
||||
@@ -45,7 +50,7 @@ func (f *fakeUptimeState) GetUptime(nodeID ids.NodeID, _ ids.ID) (time.Duration,
|
||||
|
||||
func (f *fakeUptimeState) SetUptime(nodeID ids.NodeID, _ ids.ID, uptime time.Duration, lastUpdated time.Time) error {
|
||||
if _, ok := f.uptimes[nodeID]; !ok {
|
||||
return errNotFound
|
||||
return database.ErrNotFound
|
||||
}
|
||||
f.uptimes[nodeID] = uptime
|
||||
f.lastUpdate[nodeID] = lastUpdated
|
||||
@@ -55,18 +60,23 @@ func (f *fakeUptimeState) SetUptime(nodeID ids.NodeID, _ ids.ID, uptime time.Dur
|
||||
func (f *fakeUptimeState) GetStartTime(nodeID ids.NodeID, _ ids.ID) (time.Time, error) {
|
||||
st, ok := f.startTimes[nodeID]
|
||||
if !ok {
|
||||
return time.Time{}, errNotFound
|
||||
return time.Time{}, database.ErrNotFound
|
||||
}
|
||||
return st, nil
|
||||
}
|
||||
|
||||
var errNotFound = errTestNotFound{}
|
||||
|
||||
type errTestNotFound struct{}
|
||||
|
||||
func (errTestNotFound) Error() string { return "not found" }
|
||||
|
||||
func TestUptimeTrackerConnectDisconnect(t *testing.T) {
|
||||
// TestUptimeTrackerLongRunningValidatorAccruesUptime is the regression test for
|
||||
// the ~165M LUX reward gate. A validator that has been staked for 30 days must
|
||||
// report ~100% uptime, not 0%.
|
||||
//
|
||||
// This test uses ONLY the shared Calculator surface (newUptimeTracker +
|
||||
// CalculateUptimePercentFrom) that both the old and the fixed tracker expose, so
|
||||
// it can be run against either implementation:
|
||||
// - OLD tracker: returns ~0.0 — stored upDuration is 0 and the tracker had no
|
||||
// baseline for the un-measured window, so upDuration/total = 0/30d = 0.
|
||||
// - FIXED tracker: returns ~1.0 — before tracking begins, a validator is
|
||||
// assumed online since its last persisted update (avalanchego semantics).
|
||||
func TestUptimeTrackerLongRunningValidatorAccruesUptime(t *testing.T) {
|
||||
require := require.New(t)
|
||||
|
||||
state := newFakeUptimeState()
|
||||
@@ -76,87 +86,208 @@ func TestUptimeTrackerConnectDisconnect(t *testing.T) {
|
||||
now := time.Now()
|
||||
clk := func() time.Time { return now }
|
||||
|
||||
state.addValidator(nodeID, netID, now.Add(-time.Hour))
|
||||
startTime := now.Add(-30 * 24 * time.Hour) // staked 30 days ago
|
||||
state.addValidator(nodeID, netID, startTime)
|
||||
|
||||
tracker := newUptimeTracker(state, netID, clk)
|
||||
|
||||
// Before connect: 0% uptime.
|
||||
pct, err := tracker.CalculateUptimePercentFrom(nodeID, netID, startTime)
|
||||
require.NoError(err)
|
||||
require.InDelta(1.0, pct, 0.001, "a continuously-staked validator must report ~100%% uptime, not 0%%")
|
||||
}
|
||||
|
||||
// TestUptimeTrackerStartTrackingBaselines verifies that StartTracking credits the
|
||||
// un-measured pre-tracking window (assuming the validator was online) and moves
|
||||
// the tracker into live-tracking mode.
|
||||
func TestUptimeTrackerStartTrackingBaselines(t *testing.T) {
|
||||
require := require.New(t)
|
||||
|
||||
state := newFakeUptimeState()
|
||||
netID := ids.GenerateTestID()
|
||||
nodeID := ids.GenerateTestNodeID()
|
||||
|
||||
now := time.Now().Truncate(time.Second)
|
||||
clk := func() time.Time { return now }
|
||||
|
||||
startTime := now.Add(-time.Hour)
|
||||
state.addValidator(nodeID, netID, startTime)
|
||||
|
||||
tracker := newUptimeTracker(state, netID, clk)
|
||||
require.False(tracker.StartedTracking())
|
||||
|
||||
require.NoError(tracker.StartTracking([]ids.NodeID{nodeID}))
|
||||
require.True(tracker.StartedTracking())
|
||||
|
||||
// The hour since lastUpdated (== startTime) is baked into the persisted
|
||||
// up-duration, and lastUpdated advanced to now.
|
||||
require.Equal(time.Hour, state.uptimes[nodeID])
|
||||
require.Equal(now.Unix(), state.lastUpdate[nodeID].Unix())
|
||||
}
|
||||
|
||||
// TestUptimeTrackerContinuouslyConnectedClimbs is the core behavioral fix: a
|
||||
// validator that connects (during bootstrap) and stays connected accrues uptime
|
||||
// over time WITHOUT ever disconnecting, and IsConnected reports true.
|
||||
func TestUptimeTrackerContinuouslyConnectedClimbs(t *testing.T) {
|
||||
require := require.New(t)
|
||||
|
||||
state := newFakeUptimeState()
|
||||
netID := ids.GenerateTestID()
|
||||
nodeID := ids.GenerateTestNodeID()
|
||||
|
||||
now := time.Now().Truncate(time.Second)
|
||||
clk := func() time.Time { return now }
|
||||
|
||||
startTime := now
|
||||
state.addValidator(nodeID, netID, startTime)
|
||||
|
||||
tracker := newUptimeTracker(state, netID, clk)
|
||||
|
||||
// Peer connects during bootstrap, before tracking starts.
|
||||
tracker.Connect(nodeID)
|
||||
require.True(tracker.IsConnected(nodeID))
|
||||
|
||||
// Normal operations begin.
|
||||
require.NoError(tracker.StartTracking([]ids.NodeID{nodeID}))
|
||||
|
||||
// One hour passes with the validator continuously connected — no Disconnect.
|
||||
now = now.Add(time.Hour)
|
||||
|
||||
pct, err := tracker.CalculateUptimePercent(nodeID, netID)
|
||||
require.NoError(err)
|
||||
require.InDelta(0.0, pct, 0.01)
|
||||
require.InDelta(1.0, pct, 0.001, "continuously-connected validator must climb to ~100%%")
|
||||
require.True(tracker.IsConnected(nodeID))
|
||||
|
||||
// Connect.
|
||||
tracker.Connect(nodeID)
|
||||
|
||||
// Advance clock by 30 minutes.
|
||||
now = now.Add(30 * time.Minute)
|
||||
|
||||
// While connected: should show ~50% (30min connected / 60min+30min total).
|
||||
// Two hours in, still ~100%.
|
||||
now = now.Add(time.Hour)
|
||||
pct, err = tracker.CalculateUptimePercent(nodeID, netID)
|
||||
require.NoError(err)
|
||||
// 30min / 90min = 0.333...
|
||||
require.InDelta(0.333, pct, 0.01)
|
||||
|
||||
// Disconnect.
|
||||
require.NoError(tracker.Disconnect(nodeID))
|
||||
|
||||
// State should now have 30 minutes of uptime persisted.
|
||||
require.Equal(30*time.Minute, state.uptimes[nodeID])
|
||||
|
||||
// After disconnect, no live connection bonus.
|
||||
pct, err = tracker.CalculateUptimePercent(nodeID, netID)
|
||||
require.NoError(err)
|
||||
require.InDelta(0.333, pct, 0.01)
|
||||
require.InDelta(1.0, pct, 0.001)
|
||||
}
|
||||
|
||||
func TestUptimeTrackerDoubleConnect(t *testing.T) {
|
||||
// TestUptimeTrackerConnectDisconnectFlush verifies that, once tracking, a
|
||||
// connected session is flushed into persistent state on Disconnect.
|
||||
func TestUptimeTrackerConnectDisconnectFlush(t *testing.T) {
|
||||
require := require.New(t)
|
||||
|
||||
state := newFakeUptimeState()
|
||||
netID := ids.GenerateTestID()
|
||||
nodeID := ids.GenerateTestNodeID()
|
||||
|
||||
now := time.Now()
|
||||
now := time.Now().Truncate(time.Second)
|
||||
clk := func() time.Time { return now }
|
||||
|
||||
state.addValidator(nodeID, netID, now.Add(-time.Hour))
|
||||
startTime := now
|
||||
state.addValidator(nodeID, netID, startTime)
|
||||
|
||||
tracker := newUptimeTracker(state, netID, clk)
|
||||
require.NoError(tracker.StartTracking([]ids.NodeID{nodeID}))
|
||||
require.Equal(time.Duration(0), state.uptimes[nodeID])
|
||||
|
||||
tracker.Connect(nodeID)
|
||||
tracker.Connect(nodeID) // second connect should be no-op
|
||||
|
||||
now = now.Add(10 * time.Minute)
|
||||
now = now.Add(30 * time.Minute)
|
||||
require.NoError(tracker.Disconnect(nodeID))
|
||||
|
||||
// Should be exactly 10 minutes, not 20.
|
||||
require.Equal(10*time.Minute, state.uptimes[nodeID])
|
||||
require.Equal(30*time.Minute, state.uptimes[nodeID])
|
||||
require.False(tracker.IsConnected(nodeID))
|
||||
|
||||
// After disconnect, no further live-session bonus; percent reflects 30m/60m.
|
||||
now = now.Add(30 * time.Minute)
|
||||
pct, err := tracker.CalculateUptimePercent(nodeID, netID)
|
||||
require.NoError(err)
|
||||
require.InDelta(0.5, pct, 0.001)
|
||||
}
|
||||
|
||||
func TestUptimeTrackerShutdown(t *testing.T) {
|
||||
// TestUptimeTrackerDisconnectedValidatorGetsZero verifies that, once tracking, a
|
||||
// validator that never connects earns 0% uptime (it is offline from this node's
|
||||
// perspective) — the property that lets the reward gate withhold rewards.
|
||||
func TestUptimeTrackerDisconnectedValidatorGetsZero(t *testing.T) {
|
||||
require := require.New(t)
|
||||
|
||||
state := newFakeUptimeState()
|
||||
netID := ids.GenerateTestID()
|
||||
nodeA := ids.GenerateTestNodeID()
|
||||
nodeB := ids.GenerateTestNodeID()
|
||||
nodeID := ids.GenerateTestNodeID()
|
||||
|
||||
now := time.Now()
|
||||
now := time.Now().Truncate(time.Second)
|
||||
clk := func() time.Time { return now }
|
||||
|
||||
state.addValidator(nodeA, netID, now.Add(-time.Hour))
|
||||
state.addValidator(nodeB, netID, now.Add(-time.Hour))
|
||||
startTime := now
|
||||
state.addValidator(nodeID, netID, startTime)
|
||||
|
||||
tracker := newUptimeTracker(state, netID, clk)
|
||||
require.NoError(tracker.StartTracking([]ids.NodeID{nodeID}))
|
||||
|
||||
tracker.Connect(nodeA)
|
||||
tracker.Connect(nodeB)
|
||||
now = now.Add(time.Hour) // an hour passes, never connected
|
||||
|
||||
now = now.Add(5 * time.Minute)
|
||||
require.NoError(tracker.Shutdown())
|
||||
|
||||
require.Equal(5*time.Minute, state.uptimes[nodeA])
|
||||
require.Equal(5*time.Minute, state.uptimes[nodeB])
|
||||
pct, err := tracker.CalculateUptimePercent(nodeID, netID)
|
||||
require.NoError(err)
|
||||
require.InDelta(0.0, pct, 0.001, "never-connected validator (while tracking) must earn 0%%")
|
||||
require.False(tracker.IsConnected(nodeID))
|
||||
}
|
||||
|
||||
// TestUptimeTrackerStopTrackingFlushesAll verifies that StopTracking persists all
|
||||
// connected validators' sessions and leaves tracking mode.
|
||||
func TestUptimeTrackerStopTrackingFlushesAll(t *testing.T) {
|
||||
require := require.New(t)
|
||||
|
||||
state := newFakeUptimeState()
|
||||
netID := ids.GenerateTestID()
|
||||
|
||||
now := time.Now().Truncate(time.Second)
|
||||
clk := func() time.Time { return now }
|
||||
|
||||
const numValidators = 10
|
||||
nodeIDs := make([]ids.NodeID, numValidators)
|
||||
for i := range nodeIDs {
|
||||
nodeIDs[i] = ids.GenerateTestNodeID()
|
||||
state.addValidator(nodeIDs[i], netID, now)
|
||||
}
|
||||
|
||||
tracker := newUptimeTracker(state, netID, clk)
|
||||
require.NoError(tracker.StartTracking(nodeIDs))
|
||||
for _, nid := range nodeIDs {
|
||||
tracker.Connect(nid)
|
||||
}
|
||||
|
||||
now = now.Add(3 * time.Minute)
|
||||
require.NoError(tracker.StopTracking(nodeIDs))
|
||||
require.False(tracker.StartedTracking())
|
||||
|
||||
for _, nid := range nodeIDs {
|
||||
require.Equal(3*time.Minute, state.uptimes[nid])
|
||||
}
|
||||
}
|
||||
|
||||
// TestUptimeTrackerDoubleStartTracking verifies StartTracking is not re-entrant.
|
||||
func TestUptimeTrackerDoubleStartTracking(t *testing.T) {
|
||||
require := require.New(t)
|
||||
|
||||
state := newFakeUptimeState()
|
||||
netID := ids.GenerateTestID()
|
||||
nodeID := ids.GenerateTestNodeID()
|
||||
now := time.Now().Truncate(time.Second)
|
||||
|
||||
state.addValidator(nodeID, netID, now)
|
||||
tracker := newUptimeTracker(state, netID, func() time.Time { return now })
|
||||
|
||||
require.NoError(tracker.StartTracking([]ids.NodeID{nodeID}))
|
||||
require.ErrorIs(tracker.StartTracking([]ids.NodeID{nodeID}), errAlreadyStartedTracking)
|
||||
}
|
||||
|
||||
// TestUptimeTrackerStopWithoutStart verifies StopTracking errors before start.
|
||||
func TestUptimeTrackerStopWithoutStart(t *testing.T) {
|
||||
require := require.New(t)
|
||||
|
||||
state := newFakeUptimeState()
|
||||
netID := ids.GenerateTestID()
|
||||
now := time.Now().Truncate(time.Second)
|
||||
tracker := newUptimeTracker(state, netID, func() time.Time { return now })
|
||||
|
||||
require.ErrorIs(tracker.StopTracking(nil), errNotStartedTracking)
|
||||
}
|
||||
|
||||
// TestUptimeTrackerUnknownValidator verifies non-validators are handled safely:
|
||||
// StartTracking/Connect/Disconnect skip them without error, and a percent query
|
||||
// surfaces the not-found error.
|
||||
func TestUptimeTrackerUnknownValidator(t *testing.T) {
|
||||
require := require.New(t)
|
||||
|
||||
@@ -164,21 +295,25 @@ func TestUptimeTrackerUnknownValidator(t *testing.T) {
|
||||
netID := ids.GenerateTestID()
|
||||
unknownNode := ids.GenerateTestNodeID()
|
||||
|
||||
now := time.Now()
|
||||
now := time.Now().Truncate(time.Second)
|
||||
clk := func() time.Time { return now }
|
||||
|
||||
tracker := newUptimeTracker(state, netID, clk)
|
||||
|
||||
// Connect an unknown validator. Disconnect should not error.
|
||||
// StartTracking must not fail on a node that has no state record.
|
||||
require.NoError(tracker.StartTracking([]ids.NodeID{unknownNode}))
|
||||
|
||||
// Connecting then disconnecting an unknown node is a no-op, not an error.
|
||||
tracker.Connect(unknownNode)
|
||||
now = now.Add(time.Minute)
|
||||
require.NoError(tracker.Disconnect(unknownNode))
|
||||
|
||||
// CalculateUptimePercent for unknown validator should error.
|
||||
// A percent query for an unknown validator surfaces the error.
|
||||
_, err := tracker.CalculateUptimePercent(unknownNode, netID)
|
||||
require.Error(err)
|
||||
}
|
||||
|
||||
// TestUptimeTrackerWrongNet verifies a query for a different network returns 0.
|
||||
func TestUptimeTrackerWrongNet(t *testing.T) {
|
||||
require := require.New(t)
|
||||
|
||||
@@ -187,41 +322,50 @@ func TestUptimeTrackerWrongNet(t *testing.T) {
|
||||
otherNet := ids.GenerateTestID()
|
||||
nodeID := ids.GenerateTestNodeID()
|
||||
|
||||
now := time.Now()
|
||||
now := time.Now().Truncate(time.Second)
|
||||
clk := func() time.Time { return now }
|
||||
|
||||
state.addValidator(nodeID, netID, now.Add(-time.Hour))
|
||||
tracker := newUptimeTracker(state, netID, clk)
|
||||
|
||||
// Querying the wrong net should return 0.
|
||||
pct, err := tracker.CalculateUptimePercent(nodeID, otherNet)
|
||||
require.NoError(err)
|
||||
require.Equal(0.0, pct)
|
||||
|
||||
up, total, err := tracker.CalculateUptime(nodeID, otherNet)
|
||||
require.NoError(err)
|
||||
require.Equal(time.Duration(0), up)
|
||||
require.Equal(time.Duration(0), total)
|
||||
}
|
||||
|
||||
func TestUptimeTrackerDisconnectWithoutConnect(t *testing.T) {
|
||||
// TestUptimeTrackerDoubleConnect verifies a duplicate Connect keeps the original
|
||||
// connection time (repeated router dispatch cannot inflate uptime).
|
||||
func TestUptimeTrackerDoubleConnect(t *testing.T) {
|
||||
require := require.New(t)
|
||||
|
||||
state := newFakeUptimeState()
|
||||
netID := ids.GenerateTestID()
|
||||
nodeID := ids.GenerateTestNodeID()
|
||||
|
||||
now := time.Now()
|
||||
now := time.Now().Truncate(time.Second)
|
||||
clk := func() time.Time { return now }
|
||||
|
||||
state.addValidator(nodeID, netID, now.Add(-time.Hour))
|
||||
state.addValidator(nodeID, netID, now)
|
||||
tracker := newUptimeTracker(state, netID, clk)
|
||||
require.NoError(tracker.StartTracking([]ids.NodeID{nodeID}))
|
||||
|
||||
// Disconnect without prior connect should be no-op.
|
||||
tracker.Connect(nodeID)
|
||||
now = now.Add(5 * time.Minute)
|
||||
tracker.Connect(nodeID) // duplicate — must NOT reset the 5-minute-old session
|
||||
now = now.Add(5 * time.Minute)
|
||||
require.NoError(tracker.Disconnect(nodeID))
|
||||
require.Equal(time.Duration(0), state.uptimes[nodeID])
|
||||
|
||||
// Ten minutes total, not five.
|
||||
require.Equal(10*time.Minute, state.uptimes[nodeID])
|
||||
}
|
||||
|
||||
// --- Additional edge-case and inversion tests ---
|
||||
|
||||
// TestUptimeTrackerRapidConnectDisconnect verifies that rapid Connect/Disconnect
|
||||
// cycling does not accumulate phantom uptime. Each cycle should only account
|
||||
// for the exact clock delta during that connection.
|
||||
// TestUptimeTrackerRapidConnectDisconnect verifies rapid cycling never fabricates
|
||||
// uptime beyond the exact connected intervals.
|
||||
func TestUptimeTrackerRapidConnectDisconnect(t *testing.T) {
|
||||
require := require.New(t)
|
||||
|
||||
@@ -229,179 +373,49 @@ func TestUptimeTrackerRapidConnectDisconnect(t *testing.T) {
|
||||
netID := ids.GenerateTestID()
|
||||
nodeID := ids.GenerateTestNodeID()
|
||||
|
||||
now := time.Now()
|
||||
now := time.Now().Truncate(time.Second)
|
||||
clk := func() time.Time { return now }
|
||||
|
||||
state.addValidator(nodeID, netID, now.Add(-time.Hour))
|
||||
state.addValidator(nodeID, netID, now)
|
||||
tracker := newUptimeTracker(state, netID, clk)
|
||||
require.NoError(tracker.StartTracking([]ids.NodeID{nodeID}))
|
||||
|
||||
// Rapid connect/disconnect 100 times with no clock advance.
|
||||
// 100 cycles with no clock advance — zero accrual.
|
||||
for i := 0; i < 100; i++ {
|
||||
tracker.Connect(nodeID)
|
||||
require.NoError(tracker.Disconnect(nodeID))
|
||||
}
|
||||
|
||||
// No time passed, so uptime should be 0.
|
||||
require.Equal(time.Duration(0), state.uptimes[nodeID])
|
||||
|
||||
// Now do 50 cycles with 1ms advance each.
|
||||
// 50 cycles, each holding the connection for one second.
|
||||
for i := 0; i < 50; i++ {
|
||||
tracker.Connect(nodeID)
|
||||
now = now.Add(1 * time.Millisecond)
|
||||
now = now.Add(time.Second)
|
||||
require.NoError(tracker.Disconnect(nodeID))
|
||||
}
|
||||
|
||||
// Should be exactly 50ms of uptime.
|
||||
require.Equal(50*time.Millisecond, state.uptimes[nodeID])
|
||||
require.Equal(50*time.Second, state.uptimes[nodeID])
|
||||
}
|
||||
|
||||
// TestUptimeTrackerShutdownFlushesAll verifies that Shutdown flushes all
|
||||
// currently connected validators and leaves the connected map empty.
|
||||
func TestUptimeTrackerShutdownFlushesAll(t *testing.T) {
|
||||
require := require.New(t)
|
||||
|
||||
state := newFakeUptimeState()
|
||||
netID := ids.GenerateTestID()
|
||||
|
||||
now := time.Now()
|
||||
clk := func() time.Time { return now }
|
||||
|
||||
const numValidators = 10
|
||||
nodeIDs := make([]ids.NodeID, numValidators)
|
||||
for i := range nodeIDs {
|
||||
nodeIDs[i] = ids.GenerateTestNodeID()
|
||||
state.addValidator(nodeIDs[i], netID, now.Add(-time.Hour))
|
||||
}
|
||||
|
||||
tracker := newUptimeTracker(state, netID, clk)
|
||||
|
||||
// Connect all
|
||||
for _, nid := range nodeIDs {
|
||||
tracker.Connect(nid)
|
||||
}
|
||||
|
||||
now = now.Add(3 * time.Minute)
|
||||
require.NoError(tracker.Shutdown())
|
||||
|
||||
// All should have 3 minutes of uptime
|
||||
for _, nid := range nodeIDs {
|
||||
require.Equal(3*time.Minute, state.uptimes[nid])
|
||||
}
|
||||
|
||||
// Connected map should be empty after shutdown
|
||||
tracker.mu.RLock()
|
||||
require.Len(tracker.connected, 0)
|
||||
tracker.mu.RUnlock()
|
||||
}
|
||||
|
||||
// TestUptimeTrackerNeverConnected verifies that a validator that was never
|
||||
// connected has 0% uptime.
|
||||
func TestUptimeTrackerNeverConnected(t *testing.T) {
|
||||
// TestUptimeTrackerConcurrent races Connect/Disconnect/reads and StartTracking to
|
||||
// assert there are no data races (run with -race).
|
||||
func TestUptimeTrackerConcurrent(t *testing.T) {
|
||||
require := require.New(t)
|
||||
|
||||
state := newFakeUptimeState()
|
||||
netID := ids.GenerateTestID()
|
||||
nodeID := ids.GenerateTestNodeID()
|
||||
|
||||
now := time.Now()
|
||||
clk := func() time.Time { return now }
|
||||
|
||||
// Validator has been registered for 1 hour but never connected
|
||||
state.addValidator(nodeID, netID, now.Add(-time.Hour))
|
||||
tracker := newUptimeTracker(state, netID, clk)
|
||||
|
||||
pct, err := tracker.CalculateUptimePercent(nodeID, netID)
|
||||
require.NoError(err)
|
||||
require.Equal(0.0, pct, "never-connected validator must have 0%% uptime")
|
||||
}
|
||||
|
||||
// TestUptimeTrackerAlwaysConnected verifies that a validator connected for
|
||||
// the entire tracking period reports 100% uptime.
|
||||
func TestUptimeTrackerAlwaysConnected(t *testing.T) {
|
||||
require := require.New(t)
|
||||
|
||||
state := newFakeUptimeState()
|
||||
netID := ids.GenerateTestID()
|
||||
nodeID := ids.GenerateTestNodeID()
|
||||
|
||||
now := time.Now()
|
||||
clk := func() time.Time { return now }
|
||||
|
||||
startTime := now
|
||||
state.addValidator(nodeID, netID, startTime)
|
||||
tracker := newUptimeTracker(state, netID, clk)
|
||||
|
||||
// Connect immediately at start
|
||||
tracker.Connect(nodeID)
|
||||
|
||||
// Advance clock by 1 hour
|
||||
now = now.Add(time.Hour)
|
||||
|
||||
pct, err := tracker.CalculateUptimePercent(nodeID, netID)
|
||||
require.NoError(err)
|
||||
require.InDelta(1.0, pct, 0.001, "always-connected validator must have ~100%% uptime")
|
||||
}
|
||||
|
||||
// TestUptimeTrackerConcurrentConnect verifies that concurrent Connect calls
|
||||
// from multiple goroutines do not cause data races or phantom uptime.
|
||||
func TestUptimeTrackerConcurrentConnect(t *testing.T) {
|
||||
require := require.New(t)
|
||||
|
||||
state := newFakeUptimeState()
|
||||
netID := ids.GenerateTestID()
|
||||
nodeID := ids.GenerateTestNodeID()
|
||||
|
||||
now := time.Now()
|
||||
clk := func() time.Time { return now }
|
||||
|
||||
state.addValidator(nodeID, netID, now.Add(-time.Hour))
|
||||
tracker := newUptimeTracker(state, netID, clk)
|
||||
|
||||
// Race 50 goroutines calling Connect on the same node.
|
||||
const goroutines = 50
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(goroutines)
|
||||
for i := 0; i < goroutines; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
tracker.Connect(nodeID)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
// Should have exactly one entry in connected map.
|
||||
tracker.mu.RLock()
|
||||
_, exists := tracker.connected[nodeID]
|
||||
require.True(exists)
|
||||
tracker.mu.RUnlock()
|
||||
|
||||
// Advance clock and disconnect
|
||||
now = now.Add(5 * time.Minute)
|
||||
require.NoError(tracker.Disconnect(nodeID))
|
||||
|
||||
// Uptime should be exactly 5 minutes, not 5*50 minutes.
|
||||
require.Equal(5*time.Minute, state.uptimes[nodeID])
|
||||
}
|
||||
|
||||
// TestUptimeTrackerConcurrentConnectDisconnect races Connect and Disconnect
|
||||
// from multiple goroutines on the same node.
|
||||
func TestUptimeTrackerConcurrentConnectDisconnect(t *testing.T) {
|
||||
require := require.New(t)
|
||||
|
||||
state := newFakeUptimeState()
|
||||
netID := ids.GenerateTestID()
|
||||
nodeID := ids.GenerateTestNodeID()
|
||||
|
||||
now := time.Now()
|
||||
mu := sync.Mutex{}
|
||||
var clkMu sync.Mutex
|
||||
now := time.Now().Truncate(time.Second)
|
||||
clk := func() time.Time {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
clkMu.Lock()
|
||||
defer clkMu.Unlock()
|
||||
return now
|
||||
}
|
||||
|
||||
state.addValidator(nodeID, netID, now.Add(-time.Hour))
|
||||
state.addValidator(nodeID, netID, now)
|
||||
tracker := newUptimeTracker(state, netID, clk)
|
||||
require.NoError(tracker.StartTracking([]ids.NodeID{nodeID}))
|
||||
|
||||
const goroutines = 100
|
||||
var wg sync.WaitGroup
|
||||
@@ -409,46 +423,22 @@ func TestUptimeTrackerConcurrentConnectDisconnect(t *testing.T) {
|
||||
for i := 0; i < goroutines; i++ {
|
||||
go func(idx int) {
|
||||
defer wg.Done()
|
||||
if idx%2 == 0 {
|
||||
switch idx % 4 {
|
||||
case 0:
|
||||
tracker.Connect(nodeID)
|
||||
} else {
|
||||
case 1:
|
||||
_ = tracker.Disconnect(nodeID)
|
||||
case 2:
|
||||
_, _ = tracker.CalculateUptimePercent(nodeID, netID)
|
||||
case 3:
|
||||
_ = tracker.IsConnected(nodeID)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
// Should not panic. Final state depends on ordering but must be consistent.
|
||||
// Clean up: ensure we can still shutdown.
|
||||
mu.Lock()
|
||||
clkMu.Lock()
|
||||
now = now.Add(time.Minute)
|
||||
mu.Unlock()
|
||||
require.NoError(tracker.Shutdown())
|
||||
}
|
||||
|
||||
func TestUptimeTrackerCalculateUptimePercentFrom(t *testing.T) {
|
||||
require := require.New(t)
|
||||
|
||||
state := newFakeUptimeState()
|
||||
netID := ids.GenerateTestID()
|
||||
nodeID := ids.GenerateTestNodeID()
|
||||
|
||||
now := time.Now()
|
||||
clk := func() time.Time { return now }
|
||||
|
||||
startTime := now.Add(-2 * time.Hour)
|
||||
state.addValidator(nodeID, netID, startTime)
|
||||
tracker := newUptimeTracker(state, netID, clk)
|
||||
|
||||
// Connect for 1 hour.
|
||||
tracker.Connect(nodeID)
|
||||
now = now.Add(time.Hour)
|
||||
require.NoError(tracker.Disconnect(nodeID))
|
||||
|
||||
// Total time is 3 hours (2h before + 1h connected).
|
||||
// Connected for 1 hour. Rate = 1/3.
|
||||
// From startTime: same rate applies.
|
||||
pct, err := tracker.CalculateUptimePercentFrom(nodeID, netID, startTime)
|
||||
require.NoError(err)
|
||||
require.InDelta(0.333, pct, 0.01)
|
||||
clkMu.Unlock()
|
||||
require.NoError(tracker.StopTracking([]ids.NodeID{nodeID}))
|
||||
}
|
||||
|
||||
+38
-10
@@ -260,9 +260,17 @@ func (vm *VM) Initialize(
|
||||
validatorManager := pvalidators.NewManager(vm.Internal, vm.state, vm.metrics, &vm.nodeClock)
|
||||
vm.State = validatorManager
|
||||
utxoHandler := utxo.NewHandler(context.Background(), &vm.nodeClock, vm.fx)
|
||||
// Create uptime manager - use the configured UptimeLockedCalculator which
|
||||
// delegates to its fallback calculator (NoOp by default, but tests can
|
||||
// configure ZeroUptimeCalculator for "never connected" scenarios)
|
||||
|
||||
// Create the real uptime tracker for the primary network NOW, at Initialize,
|
||||
// so peer Connect events delivered during bootstrap are captured — the
|
||||
// connected map must already be populated by the time StartTracking runs at
|
||||
// normal-operations start. Register it with the thread-safe LockedCalculator,
|
||||
// through which both the API (service.go getCurrentValidators) and the reward
|
||||
// gate (block/executor/options.go prefersCommit) read uptime.
|
||||
vm.tracker = newUptimeTracker(vm.state, constants.PrimaryNetworkID, vm.nodeClock.Time)
|
||||
if err := vm.UptimeLockedCalculator.SetCalculator(constants.PrimaryNetworkID, vm.tracker); err != nil {
|
||||
return fmt.Errorf("failed to register uptime tracker: %w", err)
|
||||
}
|
||||
vm.uptimeManager = vm.UptimeLockedCalculator
|
||||
|
||||
txExecutorBackend := &txexecutor.Backend{
|
||||
@@ -531,6 +539,16 @@ func (vm *VM) createNet(netID ids.ID) error {
|
||||
func (vm *VM) onBootstrapStarted() error {
|
||||
vm.bootstrapped.Set(false)
|
||||
vm.bootstrappedConsensus.Set(false)
|
||||
|
||||
// On a normal-ops → re-bootstrap transition, flush and stop uptime tracking
|
||||
// so connected sessions are persisted before we stop measuring. This is a
|
||||
// no-op on the first bootstrap (tracking hasn't started yet).
|
||||
if vm.tracker != nil && vm.tracker.StartedTracking() {
|
||||
primaryVdrIDs := vm.Validators.GetValidatorIDs(constants.PrimaryNetworkID)
|
||||
if err := vm.tracker.StopTracking(primaryVdrIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return vm.fx.Bootstrapping()
|
||||
}
|
||||
|
||||
@@ -546,12 +564,19 @@ func (vm *VM) onReady() error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create and register the real uptime tracker for the primary network.
|
||||
vm.tracker = newUptimeTracker(vm.state, constants.PrimaryNetworkID, vm.nodeClock.Time)
|
||||
if err := vm.UptimeLockedCalculator.SetCalculator(constants.PrimaryNetworkID, vm.tracker); err != nil {
|
||||
return err
|
||||
// Begin tracking validator uptime for the primary network. The tracker was
|
||||
// created and registered at Initialize (so bootstrap-time peer connections
|
||||
// were already captured); StartTracking baselines every current validator's
|
||||
// uptime record and switches the tracker into live-tracking mode. Mirrors
|
||||
// avalanchego's onNormalOperationsStarted.
|
||||
if vm.tracker != nil && !vm.tracker.StartedTracking() {
|
||||
primaryVdrIDs := vm.Validators.GetValidatorIDs(constants.PrimaryNetworkID)
|
||||
if err := vm.tracker.StartTracking(primaryVdrIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
vm.log.Info("uptime tracking started for primary network",
|
||||
log.Int("validators", len(primaryVdrIDs)))
|
||||
}
|
||||
vm.log.Info("uptime tracker registered for primary network")
|
||||
|
||||
// Commit state BEFORE starting background goroutines to avoid race conditions
|
||||
// between state readers (forwardNotifications) and state writers (Commit)
|
||||
@@ -592,8 +617,11 @@ func (vm *VM) Shutdown(context.Context) error {
|
||||
|
||||
vm.onShutdownCtxCancel()
|
||||
|
||||
if vm.tracker != nil {
|
||||
if err := vm.tracker.Shutdown(); err != nil {
|
||||
// Flush uptime for all primary-network validators before closing state, so
|
||||
// connected sessions are durably persisted across the restart.
|
||||
if vm.tracker != nil && vm.tracker.StartedTracking() {
|
||||
primaryVdrIDs := vm.Validators.GetValidatorIDs(constants.PrimaryNetworkID)
|
||||
if err := vm.tracker.StopTracking(primaryVdrIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := vm.state.Commit(); err != nil {
|
||||
|
||||
Reference in New Issue
Block a user