Files
netrunner/local/snapshot.go
T
2026-01-14 19:56:11 -08:00

661 lines
21 KiB
Go

package local
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"time"
"maps"
"slices"
luxconfig "github.com/luxfi/config"
"github.com/luxfi/constants"
"github.com/luxfi/ids"
log "github.com/luxfi/log"
"github.com/luxfi/netrunner/api"
"github.com/luxfi/netrunner/network"
"github.com/luxfi/netrunner/network/node"
"github.com/luxfi/netrunner/utils"
"github.com/luxfi/config"
"github.com/luxfi/sdk/api/admin"
dircopy "github.com/otiai10/copy"
)
// NetworkState defines dynamic network information not available on chain db
type NetworkState struct {
// Map from chain id to elastic chain tx id
ChainID2ElasticChainID map[string]string `json:"chainID2ElasticChainID"`
}
const (
hotSnapshotManifestName = "hot_snapshot.json"
hotSnapshotVersion = 1
)
type hotSnapshotManifest struct {
Version int `json:"version"`
Network string `json:"network"`
Nodes map[string]hotSnapshotNode `json:"nodes"`
}
type hotSnapshotNode struct {
LastVersion uint64 `json:"last_version"`
Backups []hotSnapshotBackup `json:"backups"`
}
type hotSnapshotBackup struct {
Since uint64 `json:"since"`
Version uint64 `json:"version"`
Path string `json:"path"`
CreatedAt string `json:"created_at"`
}
// NewNetwork returns a new network from the given snapshot
func NewNetworkFromSnapshot(
log log.Logger,
snapshotName string,
rootDir string,
snapshotsDir string,
binaryPath string,
pluginDir string,
chainConfigs map[string]string,
upgradeConfigs map[string]string,
pChainConfigs map[string]string,
flags map[string]interface{},
reassignPortsIfUsed bool,
) (network.Network, error) {
net, err := newNetwork(
log,
api.NewAPIClient,
&nodeProcessCreator{
colorPicker: utils.NewColorPicker(),
logger: log,
stdout: os.Stdout,
stderr: os.Stderr,
},
rootDir,
snapshotsDir,
reassignPortsIfUsed,
)
if err != nil {
return net, err
}
err = net.loadSnapshot(
context.Background(),
snapshotName,
binaryPath,
pluginDir,
chainConfigs,
upgradeConfigs,
pChainConfigs,
flags,
)
return net, err
}
// Save network snapshot
// Network is stopped in order to do a safe preservation
func (ln *localNetwork) SaveSnapshot(ctx context.Context, snapshotName string) (string, error) {
ln.lock.Lock()
defer ln.lock.Unlock()
if ln.stopCalled() {
return "", network.ErrStopped
}
if len(snapshotName) == 0 {
return "", fmt.Errorf("invalid snapshotName %q", snapshotName)
}
// check if snapshot already exists
snapshotDir := filepath.Join(ln.snapshotsDir, snapshotPrefix+snapshotName)
if _, err := os.Stat(snapshotDir); err == nil {
return "", fmt.Errorf("snapshot %q already exists", snapshotName)
}
// keep copy of node info that will be removed by stop
nodesConfig := map[string]node.Config{}
nodesDBDir := map[string]string{}
for nodeName, node := range ln.nodes {
nodeConfig := node.config
// depending on how the user generated the config, different nodes config flags
// may point to the same map, so we made a copy to avoid always modifying the same value
nodeConfig.Flags = maps.Clone(nodeConfig.Flags)
nodesConfig[nodeName] = nodeConfig
nodesDBDir[nodeName] = node.GetDbDir()
}
// we change nodeConfig.Flags so as to preserve in snapshot the current node ports
for nodeName, nodeConfig := range nodesConfig {
nodeConfig.Flags[config.HTTPPortKey] = ln.nodes[nodeName].GetAPIPort()
nodeConfig.Flags[config.StakingPortKey] = ln.nodes[nodeName].GetP2PPort()
}
// make copy of network flags
networkConfigFlags := maps.Clone(ln.flags)
// remove all data dir, log dir references
delete(networkConfigFlags, config.DataDirKey)
delete(networkConfigFlags, config.LogsDirKey)
for nodeName, nodeConfig := range nodesConfig {
if nodeConfig.ConfigFile != "" {
var err error
nodeConfig.ConfigFile, err = utils.SetJSONKey(nodeConfig.ConfigFile, config.LogsDirKey, "")
if err != nil {
return "", err
}
}
delete(nodeConfig.Flags, config.DataDirKey)
delete(nodeConfig.Flags, config.LogsDirKey)
nodesConfig[nodeName] = nodeConfig
}
// stop network to safely save snapshot
if err := ln.stop(ctx); err != nil {
return "", err
}
syncFilesystem()
// create main snapshot dirs
snapshotDBDir := filepath.Join(snapshotDir, defaultDBSubdir)
if err := os.MkdirAll(snapshotDBDir, os.ModePerm); err != nil {
return "", err
}
// save db
for _, nodeConfig := range nodesConfig {
sourceDBDir, ok := nodesDBDir[nodeConfig.Name]
if !ok {
return "", fmt.Errorf("failure obtaining db path for node %q", nodeConfig.Name)
}
sourceDBDir = filepath.Join(sourceDBDir, constants.NetworkName(ln.networkID))
targetDBDir := filepath.Join(filepath.Join(snapshotDBDir, nodeConfig.Name), constants.NetworkName(ln.networkID))
if err := dircopy.Copy(sourceDBDir, targetDBDir); err != nil {
return "", fmt.Errorf("failure saving node %q db dir: %w", nodeConfig.Name, err)
}
}
// save network conf
networkConfig := network.Config{
Genesis: string(ln.genesis),
Flags: networkConfigFlags,
NodeConfigs: []node.Config{},
BinaryPath: ln.binaryPath,
ChainConfigFiles: ln.chainConfigFiles,
GenesisConfigFiles: ln.genesisConfigFiles,
UpgradeConfigFiles: ln.upgradeConfigFiles,
PChainConfigFiles: ln.pChainConfigFiles,
}
// no need to save this, will be generated automatically on snapshot load
networkConfig.NodeConfigs = append(networkConfig.NodeConfigs, slices.Collect(maps.Values(nodesConfig))...)
networkConfigJSON, err := json.MarshalIndent(networkConfig, "", " ")
if err != nil {
return "", err
}
if err := createFileAndWrite(filepath.Join(snapshotDir, "network.json"), networkConfigJSON); err != nil {
return "", err
}
// save dynamic part of network not available on blockchain
chainID2ElasticChainID := map[string]string{}
for chainID, elasticChainID := range ln.chainID2ElasticChainID {
chainID2ElasticChainID[chainID.String()] = elasticChainID.String()
}
networkState := NetworkState{
ChainID2ElasticChainID: chainID2ElasticChainID,
}
networkStateJSON, err := json.MarshalIndent(networkState, "", " ")
if err != nil {
return "", err
}
if err := createFileAndWrite(filepath.Join(snapshotDir, "state.json"), networkStateJSON); err != nil {
return "", err
}
return snapshotDir, nil
}
// start network from snapshot
func (ln *localNetwork) loadSnapshot(
ctx context.Context,
snapshotName string,
binaryPath string,
pluginDir string,
chainConfigs map[string]string,
upgradeConfigs map[string]string,
pChainConfigs map[string]string,
flags map[string]interface{},
) error {
ln.lock.Lock()
defer ln.lock.Unlock()
snapshotDir := filepath.Join(ln.snapshotsDir, snapshotPrefix+snapshotName)
snapshotDBDir := filepath.Join(snapshotDir, defaultDBSubdir)
_, err := os.Stat(snapshotDir)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return ErrSnapshotNotFound
} else {
return fmt.Errorf("failure accessing snapshot %q: %w", snapshotName, err)
}
}
// load network config
networkConfigJSON, err := os.ReadFile(filepath.Join(snapshotDir, "network.json"))
if err != nil {
return fmt.Errorf("failure reading network config file from snapshot: %w", err)
}
networkConfig := network.Config{}
if err := json.Unmarshal(networkConfigJSON, &networkConfig); err != nil {
return fmt.Errorf("failure unmarshaling network config from snapshot: %w", err)
}
// add flags
for i := range networkConfig.NodeConfigs {
for k, v := range flags {
networkConfig.NodeConfigs[i].Flags[k] = v
}
}
hotManifestPath := filepath.Join(snapshotDir, hotSnapshotManifestName)
hotManifest, err := loadHotSnapshotManifest(hotManifestPath)
if err != nil {
return err
}
if hotManifest != nil {
if err := ln.prepareHotSnapshotDirs(networkConfig.NodeConfigs); err != nil {
return err
}
for i := range networkConfig.NodeConfigs {
targetDBDir := filepath.Join(filepath.Join(ln.rootDir, networkConfig.NodeConfigs[i].Name), defaultDBSubdir)
networkConfig.NodeConfigs[i].Flags[config.DBPathKey] = targetDBDir
}
} else {
// load db from full directory copies
for _, nodeConfig := range networkConfig.NodeConfigs {
sourceDBDir := filepath.Join(snapshotDBDir, nodeConfig.Name)
targetDBDir := filepath.Join(filepath.Join(ln.rootDir, nodeConfig.Name), defaultDBSubdir)
if err := dircopy.Copy(sourceDBDir, targetDBDir); err != nil {
return fmt.Errorf("failure loading node %q db dir: %w", nodeConfig.Name, err)
}
nodeConfig.Flags[config.DBPathKey] = targetDBDir
}
}
// replace binary path
if binaryPath != "" {
for i := range networkConfig.NodeConfigs {
networkConfig.NodeConfigs[i].BinaryPath = binaryPath
}
}
// set plugin dir
resolvedPluginDir := pluginDir
if resolvedPluginDir == "" {
resolvedPluginDir = luxconfig.ResolvePluginDir()
}
for i := range networkConfig.NodeConfigs {
networkConfig.NodeConfigs[i].Flags[config.PluginDirKey] = resolvedPluginDir
}
// add chain configs and upgrade configs
for i := range networkConfig.NodeConfigs {
if networkConfig.NodeConfigs[i].ChainConfigFiles == nil {
networkConfig.NodeConfigs[i].ChainConfigFiles = map[string]string{}
}
if networkConfig.NodeConfigs[i].UpgradeConfigFiles == nil {
networkConfig.NodeConfigs[i].UpgradeConfigFiles = map[string]string{}
}
if networkConfig.NodeConfigs[i].ChainConfigFiles == nil {
networkConfig.NodeConfigs[i].ChainConfigFiles = map[string]string{}
}
for k, v := range chainConfigs {
networkConfig.NodeConfigs[i].ChainConfigFiles[k] = v
}
for k, v := range upgradeConfigs {
networkConfig.NodeConfigs[i].UpgradeConfigFiles[k] = v
}
for k, v := range chainConfigs {
networkConfig.NodeConfigs[i].ChainConfigFiles[k] = v
}
}
// load network state not available at blockchain db
networkStateJSON, err := os.ReadFile(filepath.Join(snapshotDir, "state.json"))
if err != nil {
if !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("failure reading network state file from snapshot: %w", err)
}
ln.logger.Warn("network state file not found on snapshot")
} else {
networkState := NetworkState{}
if err := json.Unmarshal(networkStateJSON, &networkState); err != nil {
return fmt.Errorf("failure unmarshaling network state from snapshot: %w", err)
}
ln.chainID2ElasticChainID = map[ids.ID]ids.ID{}
for chainIDStr, elasticChainIDStr := range networkState.ChainID2ElasticChainID {
chainID, err := ids.FromString(chainIDStr)
if err != nil {
return err
}
elasticChainID, err := ids.FromString(elasticChainIDStr)
if err != nil {
return err
}
ln.chainID2ElasticChainID[chainID] = elasticChainID
}
}
if err := ln.loadConfig(ctx, networkConfig); err != nil {
return err
}
if hotManifest != nil {
if err := ln.applyHotSnapshot(ctx, snapshotDir, networkConfig.NodeConfigs, hotManifest); err != nil {
return err
}
}
return nil
}
// SaveHotSnapshot saves a snapshot without stopping the network.
// Uses the admin snapshot API to stream a consistent backup while nodes continue serving.
func (ln *localNetwork) SaveHotSnapshot(ctx context.Context, snapshotName string) (string, error) {
ln.lock.RLock() // Read lock - doesn't block network operations
defer ln.lock.RUnlock()
if ln.stopCalled() {
return "", network.ErrStopped
}
if len(snapshotName) == 0 {
return "", fmt.Errorf("invalid snapshotName %q", snapshotName)
}
snapshotDir := filepath.Join(ln.snapshotsDir, snapshotPrefix+snapshotName)
networkName := constants.NetworkName(ln.networkID)
snapshotExists := true
if _, err := os.Stat(snapshotDir); err != nil {
if errors.Is(err, os.ErrNotExist) {
snapshotExists = false
} else {
return "", fmt.Errorf("failed to access snapshot dir: %w", err)
}
}
// Collect node info (read-only, safe during operation)
nodesConfig := map[string]node.Config{}
for nodeName, node := range ln.nodes {
nodeConfig := node.config
nodeConfig.Flags = maps.Clone(nodeConfig.Flags)
nodesConfig[nodeName] = nodeConfig
}
// Preserve current node ports
for nodeName, nodeConfig := range nodesConfig {
nodeConfig.Flags[config.HTTPPortKey] = ln.nodes[nodeName].GetAPIPort()
nodeConfig.Flags[config.StakingPortKey] = ln.nodes[nodeName].GetP2PPort()
}
// Make copy of network flags
networkConfigFlags := maps.Clone(ln.flags)
delete(networkConfigFlags, config.DataDirKey)
delete(networkConfigFlags, config.LogsDirKey)
for nodeName, nodeConfig := range nodesConfig {
if nodeConfig.ConfigFile != "" {
var err error
nodeConfig.ConfigFile, err = utils.SetJSONKey(nodeConfig.ConfigFile, config.LogsDirKey, "")
if err != nil {
return "", err
}
}
delete(nodeConfig.Flags, config.DataDirKey)
delete(nodeConfig.Flags, config.LogsDirKey)
nodesConfig[nodeName] = nodeConfig
}
// Create main snapshot dirs
snapshotDBDir := filepath.Join(snapshotDir, defaultDBSubdir)
if err := os.MkdirAll(snapshotDBDir, os.ModePerm); err != nil {
return "", err
}
manifestPath := filepath.Join(snapshotDir, hotSnapshotManifestName)
manifest, err := loadHotSnapshotManifest(manifestPath)
if err != nil {
return "", err
}
if manifest == nil {
if snapshotExists {
return "", fmt.Errorf("snapshot %q exists without hot manifest; choose a new name or remove it", snapshotName)
}
manifest = &hotSnapshotManifest{
Version: hotSnapshotVersion,
Network: networkName,
Nodes: map[string]hotSnapshotNode{},
}
} else if manifest.Network != networkName {
return "", fmt.Errorf("hot snapshot network mismatch: got %q want %q", manifest.Network, networkName)
}
// Save network config
networkConfig := network.Config{
Genesis: string(ln.genesis),
Flags: networkConfigFlags,
NodeConfigs: []node.Config{},
BinaryPath: ln.binaryPath,
ChainConfigFiles: ln.chainConfigFiles,
GenesisConfigFiles: ln.genesisConfigFiles,
UpgradeConfigFiles: ln.upgradeConfigFiles,
PChainConfigFiles: ln.pChainConfigFiles,
}
networkConfig.NodeConfigs = append(networkConfig.NodeConfigs, slices.Collect(maps.Values(nodesConfig))...)
networkConfigJSON, err := json.MarshalIndent(networkConfig, "", " ")
if err != nil {
return "", err
}
if err := createFileAndWrite(filepath.Join(snapshotDir, "network.json"), networkConfigJSON); err != nil {
return "", err
}
// Save dynamic network state
chainID2ElasticChainID := map[string]string{}
for chainID, elasticChainID := range ln.chainID2ElasticChainID {
chainID2ElasticChainID[chainID.String()] = elasticChainID.String()
}
networkState := NetworkState{
ChainID2ElasticChainID: chainID2ElasticChainID,
}
networkStateJSON, err := json.MarshalIndent(networkState, "", " ")
if err != nil {
return "", err
}
if err := createFileAndWrite(filepath.Join(snapshotDir, "state.json"), networkStateJSON); err != nil {
return "", err
}
for _, nodeConfig := range nodesConfig {
localNode, ok := ln.nodes[nodeConfig.Name]
if !ok {
return "", fmt.Errorf("node %q not found for hot snapshot", nodeConfig.Name)
}
adminClient := localNode.GetAPIClient().AdminAPI()
if adminClient == nil {
return "", fmt.Errorf("admin API client is nil for node %q", nodeConfig.Name)
}
nodeManifest := manifest.Nodes[nodeConfig.Name]
since := nodeManifest.LastVersion
nodeSnapshotDir := filepath.Join(snapshotDBDir, nodeConfig.Name, networkName, "backups")
if err := os.MkdirAll(nodeSnapshotDir, os.ModePerm); err != nil {
return "", fmt.Errorf("failed to create backup dir for node %q: %w", nodeConfig.Name, err)
}
timestamp := time.Now().UTC().Format("20060102T150405Z")
backupFile := fmt.Sprintf("backup_since_%d_%s.bak", since, timestamp)
backupPath := filepath.Join(nodeSnapshotDir, backupFile)
version, err := requestAdminSnapshot(ctx, adminClient, backupPath, since)
if err != nil {
_ = os.Remove(backupPath)
return "", fmt.Errorf("failed to snapshot node %q: %w", nodeConfig.Name, err)
}
relativePath := filepath.ToSlash(filepath.Join(defaultDBSubdir, nodeConfig.Name, networkName, "backups", backupFile))
nodeManifest.Backups = append(nodeManifest.Backups, hotSnapshotBackup{
Since: since,
Version: version,
Path: relativePath,
CreatedAt: time.Now().UTC().Format(time.RFC3339),
})
nodeManifest.LastVersion = version
manifest.Nodes[nodeConfig.Name] = nodeManifest
}
if err := saveHotSnapshotManifest(manifestPath, manifest); err != nil {
return "", err
}
ln.logger.Info("Hot snapshot saved", log.String("snapshot", snapshotName), log.String("path", snapshotDir))
return snapshotDir, nil
}
// Remove network snapshot
func (ln *localNetwork) RemoveSnapshot(snapshotName string) error {
snapshotDir := filepath.Join(ln.snapshotsDir, snapshotPrefix+snapshotName)
_, err := os.Stat(snapshotDir)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return ErrSnapshotNotFound
} else {
return fmt.Errorf("failure accessing snapshot %q: %w", snapshotName, err)
}
}
if err := os.RemoveAll(snapshotDir); err != nil {
return fmt.Errorf("failure removing snapshot path %q: %w", snapshotDir, err)
}
return nil
}
// Get network snapshots
func (ln *localNetwork) GetSnapshotNames() ([]string, error) {
_, err := os.Stat(ln.snapshotsDir)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil, fmt.Errorf("snapshots dir %q does not exists", ln.snapshotsDir)
} else {
return nil, fmt.Errorf("failure accessing snapshots dir %q: %w", ln.snapshotsDir, err)
}
}
matches, err := filepath.Glob(filepath.Join(ln.snapshotsDir, snapshotPrefix+"*"))
if err != nil {
return nil, err
}
snapshots := []string{}
for _, match := range matches {
snapshots = append(snapshots, strings.TrimPrefix(filepath.Base(match), snapshotPrefix))
}
return snapshots, nil
}
func loadHotSnapshotManifest(path string) (*hotSnapshotManifest, error) {
data, err := os.ReadFile(path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil, nil
}
return nil, fmt.Errorf("failed to read hot snapshot manifest: %w", err)
}
manifest := &hotSnapshotManifest{}
if err := json.Unmarshal(data, manifest); err != nil {
return nil, fmt.Errorf("failed to parse hot snapshot manifest: %w", err)
}
if manifest.Version != hotSnapshotVersion {
return nil, fmt.Errorf("unsupported hot snapshot version %d", manifest.Version)
}
if manifest.Nodes == nil {
manifest.Nodes = map[string]hotSnapshotNode{}
}
return manifest, nil
}
func saveHotSnapshotManifest(path string, manifest *hotSnapshotManifest) error {
data, err := json.MarshalIndent(manifest, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal hot snapshot manifest: %w", err)
}
return createFileAndWrite(path, data)
}
func (ln *localNetwork) prepareHotSnapshotDirs(nodeConfigs []node.Config) error {
for _, nodeConfig := range nodeConfigs {
targetDBDir := filepath.Join(ln.rootDir, nodeConfig.Name, defaultDBSubdir)
if err := os.RemoveAll(targetDBDir); err != nil {
return fmt.Errorf("failed to clear db dir for node %q: %w", nodeConfig.Name, err)
}
if err := os.MkdirAll(targetDBDir, 0o750); err != nil {
return fmt.Errorf("failed to create db dir for node %q: %w", nodeConfig.Name, err)
}
}
return nil
}
func (ln *localNetwork) applyHotSnapshot(
ctx context.Context,
snapshotDir string,
nodeConfigs []node.Config,
manifest *hotSnapshotManifest,
) error {
for _, nodeConfig := range nodeConfigs {
nodeName := nodeConfig.Name
nodeManifest, ok := manifest.Nodes[nodeName]
if !ok || len(nodeManifest.Backups) == 0 {
return fmt.Errorf("missing hot snapshot backups for node %q", nodeName)
}
localNode, ok := ln.nodes[nodeName]
if !ok {
return fmt.Errorf("node %q not found while applying hot snapshot", nodeName)
}
adminClient := localNode.GetAPIClient().AdminAPI()
if adminClient == nil {
return fmt.Errorf("admin API client is nil for node %q", nodeName)
}
backups := append([]hotSnapshotBackup(nil), nodeManifest.Backups...)
sort.Slice(backups, func(i, j int) bool {
return backups[i].Since < backups[j].Since
})
for _, backup := range backups {
backupPath := filepath.Join(snapshotDir, filepath.FromSlash(backup.Path))
if err := requestAdminLoad(ctx, adminClient, backupPath); err != nil {
return fmt.Errorf("failed to load backup for node %q: %w", nodeName, err)
}
}
}
return nil
}
type adminSnapshotArgs struct {
Path string `json:"path"`
Since uint64 `json:"since"`
}
type adminSnapshotReply struct {
Success bool `json:"success"`
Version uint64 `json:"version"`
}
type adminLoadArgs struct {
Path string `json:"path"`
}
func requestAdminSnapshot(ctx context.Context, adminClient *admin.Client, path string, since uint64) (uint64, error) {
reply := &adminSnapshotReply{}
if err := adminClient.Requester.SendRequest(ctx, "admin.snapshot", &adminSnapshotArgs{
Path: path,
Since: since,
}, reply); err != nil {
return 0, err
}
return reply.Version, nil
}
func requestAdminLoad(ctx context.Context, adminClient *admin.Client, path string) error {
return adminClient.Requester.SendRequest(ctx, "admin.load", &adminLoadArgs{
Path: path,
}, &struct{}{})
}