Merge branch 'main' into fastsync-e2e-add-chainconf

.github/workflows/build-test-release.rs.yml

@@ -1,80 +0,0 @@
-name: Test and release (Rust SDK)
-
-# ref. https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
-on:
-  push:
-    branches:
-      - main
-    tags:
-      - "*"
-  pull_request:
-
-permissions:
-  contents: read
-
-jobs:
-  static_analysis:
-    name: Static analysis for Rust SDK
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Install Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly
-          profile: minimal
-          components: rustfmt, clippy
-          override: true
-      - name: Check Rust version
-        run: rustc --version
-      - uses: Swatinem/rust-cache@v1
-        with:
-          cache-on-failure: true
-      - name: Run static analysis tests
-        shell: bash
-        run: pushd ./avalanche-network-runner-sdk && scripts/static-analysis.sh && popd
-
-  check_cargo_unused:
-    name: Check Cargo unused for Rust SDK
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Install Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly
-          profile: minimal
-          components: rustfmt, clippy
-          override: true
-      - name: Check Rust version
-        run: rustc --version
-      - uses: Swatinem/rust-cache@v1
-        with:
-          cache-on-failure: true
-      - name: Check unused Cargo dependencies
-        shell: bash
-        run: pushd ./avalanche-network-runner-sdk && scripts/cargo.unused.sh && popd
-
-  unit_tests:
-    name: Unit tests for Rust SDK
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Install Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal
-          override: true
-      - name: Check Rust version
-        run: rustc --version
-      - uses: Swatinem/rust-cache@v1
-        with:
-          cache-on-failure: true
-      - name: Run unit tests
-        run: pushd ./avalanche-network-runner-sdk && scripts/tests.unit.sh && popd
-
-  # TODO: automate crate publish/release

.github/workflows/build-test-release.yml

@@ -54,7 +54,7 @@ jobs:
           go-version: 1.17
       - name: Run e2e tests
         shell: bash
-        run: scripts/tests.e2e.sh 1.7.8 1.7.9
+        run: scripts/tests.e2e.sh 1.7.12 1.7.13
   release:
     needs: [lint_test, unit_test, e2e_test]
     runs-on: ubuntu-latest

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "avalanche-network-runner-sdk/googleapis"]
-	path = avalanche-network-runner-sdk/googleapis
-	url = https://github.com/googleapis/googleapis

.golangci.yml

@@ -4,7 +4,6 @@ run:
   tests: false
   skip-dirs:
     - api/mocks
-    - k8s/mocks
     - local/mocks
     - rpcpb
   go: "1.17"

README.md

@@ -81,6 +81,8 @@ avalanche-network-runner server \
 --log-level debug \
 --port=":8080" \
 --grpc-gateway-port=":8081"
+
+# set "--disable-grpc-gateway" to disable gRPC gateway
 ```
 
 Note that the above command will run until you stop it with `CTRL + C`. You should run further commands in a separate terminal.
@@ -109,9 +111,50 @@ curl -X POST -k http://localhost:8081/v1/control/start -d '{"execPath":"'${AVALA
 avalanche-network-runner control start \
 --log-level debug \
 --endpoint="0.0.0.0:8080" \
+--number-of-nodes=5 \
 --avalanchego-path ${AVALANCHEGO_EXEC_PATH}
 ```
 
+Additional optional parameters which can be passed to the start command:
+
+```bash
+  --plugin-dir ${AVALANCHEGO_PLUGIN_PATH} \
+	--custom-vms '{"subnetevm":"/tmp/subnet-evm.genesis.json"}'
+	--global-node-config '{"index-enabled":false, "api-admin-enabled":true,"network-peer-list-gossip-frequency":"300ms"}'
+	--custom-node-configs" '{"node1":{"log-level":"debug","api-admin-enabled":false},"node2":{...},...}'
+```
+
+`--plugin-dir` and `--custom-vms` are parameters relevant to subnet operation.
+See the [subnet](#network-runner-rpc-server-subnet-evm-example) section for details about how to run subnets.
+
+The network-runner supports avalanchego node configuration at different levels.
+1. If neither `--global-node-config` nor `--custom-node-configs` is supplied, all nodes get a standard set of config options. Currently this set contains:
+    ```json
+        {
+        "network-peer-list-gossip-frequency":"250ms",
+        "network-max-reconnect-delay":"1s",
+        "public-ip":"127.0.0.1",
+        "health-check-frequency":"2s",
+        "api-admin-enabled":true,
+        "api-ipcs-enabled":true,
+        "index-enabled":true
+        }
+    ```
+2. `--global-node-config` is a JSON string representing a *single* avalanchego config, which will be applied to **all nodes**. This makes it easy to define common properties to all nodes. Whatever is set here will be *combined* with the standard set above.
+3. `--custom-node-configs` is a map of JSON strings representing the *complete* network with individual configs. This allows to configure each node independently. If set, `--number-of-nodes` will be **ignored** to avoid conflicts.
+4. The configs can be combined and will be merged, i.e. one could set global `--global-node-config` entries applied to each node, and also set `--custom-node-configs` for additional entries.
+5. Common `--custom-node-configs` entries override `--global-node-config` entries which override the standard set.
+6. The following entries will be **ignored in all cases** because the network-runner needs to set them internally to function properly:
+    ```
+      --log-dir
+      --db-dir
+      --http-port
+      --staking-port
+      --public-ip
+    ```
+
+**NAMING CONVENTION**: Currently, node names should be called `node` + a number, i.e. `node1,node2,node3,...node 101` 
+
 To wait for all the nodes in the cluster to become healthy:
 
 ```bash
@@ -156,6 +199,79 @@ stream-status \
 --endpoint="0.0.0.0:8080"
 ```
 
+To save the network to a snapshot:
+
+```bash
+curl -X POST -k http://localhost:8081/v1/control/savesnapshot -d '{"snapshot_name":"node5"}'
+
+# or
+avalanche-network-runner control save-snapshot snapshotName
+```
+
+To load a network from a snapshot:
+
+```bash
+curl -X POST -k http://localhost:8081/v1/control/loadsnapshot -d '{"snapshot_name":"node5"}'
+
+# or
+avalanche-network-runner control load-snapshot snapshotName
+```
+
+An avalanchego binary path and/or plugin dir can be specified when loading the snapshot. This is
+optional. If not specified, will use the paths saved with the snapshot:
+
+```bash
+curl -X POST -k http://localhost:8081/v1/control/loadsnapshot -d '{"snapshot_name":"node5","execPath":"'${AVALANCHEGO_EXEC_PATH}'","pluginDir":"'${AVALANCHEGO_PLUGIN_PATH}'"}'
+
+# or
+avalanche-network-runner control load-snapshot snapshotName --avalanchego-path ${AVALANCHEGO_EXEC_PATH} --plugin-dir ${AVALANCHEGO_PLUGIN_PATH}
+```
+
+To get the list of snapshots:
+
+```bash
+curl -X POST -k http://localhost:8081/v1/control/getsnapshotnames
+
+# or
+avalanche-network-runner control get-snapshot-names
+```
+
+To remove a snapshot:
+
+```bash
+curl -X POST -k http://localhost:8081/v1/control/removesnapshot -d '{"snapshot_name":"node5"}'
+
+# or
+avalanche-network-runner control remove-snapshot snapshotName
+```
+
+To create N validated subnets (requires network restart):
+
+```bash
+curl -X POST -k http://localhost:8081/v1/control/createsubnets -d '{"num_subnets":5}'
+
+# or
+avalanche-network-runner control create-subnets 5
+```
+
+To create a blockchain without a subnet id (requires network restart):
+
+```bash
+curl -X POST -k http://localhost:8081/v1/control/createblockchains -d '{"pluginDir":"'$PLUGIN_DIR'","customVms":[{"vm_name":"'$VM_NAME'","genesis":"'$GENESIS_PATH'"}]}'
+
+# or
+avalanche-network-runner control create-blockchains --custom-vms '[{"vm_name":"'$VM_NAME'","genesis":"'$GENESIS_PATH'"}]' --plugin-dir $PLUGIN_DIR
+```
+
+To create a blockchain with a subnet id (does not require restart):
+
+```bash
+curl -X POST -k http://localhost:8081/v1/control/createblockchains -d '{"pluginDir":"'$PLUGIN_DIR'","customVms":[{"vm_name":"'$VM_NAME'","genesis":"'$GENESIS_PATH'", "subnet_id": "'$SUBNET_ID'"}]}'
+
+# or
+avalanche-network-runner control create-blockchains --custom-vms '[{"vm_name":"'$VM_NAME'","genesis":"'$GENESIS_PATH'", "subnet_id": "'$SUBNET_ID'"}]' --plugin-dir $PLUGIN_DIR
+```
+
 To remove (stop) a node:
 
 ```bash
@@ -188,6 +304,46 @@ avalanche-network-runner control restart-node \
 --avalanchego-path ${AVALANCHEGO_EXEC_PATH}
 ```
 
+To add a node (in this case, a new node named `node99`):
+
+```bash
+# e.g., ${HOME}/go/src/github.com/ava-labs/avalanchego/build/avalanchego
+AVALANCHEGO_EXEC_PATH="avalanchego"
+
+# Note that you can add the new node with a different binary by providing
+# a different execPath
+curl -X POST -k http://localhost:8081/v1/control/addnode -d '{"name":"node99","execPath":"'${AVALANCHEGO_EXEC_PATH}'","logLevel":"INFO"}'
+
+# or
+avalanche-network-runner control add-node \
+--request-timeout=3m \
+--log-level debug \
+--endpoint="0.0.0.0:8080" \
+--node-name node99 \
+--avalanchego-path ${AVALANCHEGO_EXEC_PATH}
+```
+
+You can also provide additional flags that specify the node's config, and what custom VMs it supports:
+
+```
+	--node-config '{"index-enabled":false, "api-admin-enabled":true,"network-peer-list-gossip-frequency":"300ms"}'
+	--custom-vms '{"subnetevm":"/tmp/subnet-evm.genesis.json"}'
+```
+
+`--node-config` allows to specify specific avalanchego config parameters to the new node.
+See [here](https://docs.avax.network/build/references/avalanchego-config-flags) for the reference of supported flags.
+
+**Note**: The following parameters will be *ignored* if set in `--node-config`, because the network runner needs to set its own in order to function properly:
+`--log-dir`
+`--db-dir`
+
+`--custom-vms` allows to configure custom VMs supported by this node.
+See the [subnet](#network-runner-rpc-server-subnet-evm-example) section for details about how to run subnets.
+
+**Note**: The following subnet parameters will be set from the global network configuration to this node:
+`--whitelisted-subnets`
+`--plugin-dir`
+
 AvalancheGo exposes a "test peer", which you can attach to a node.
 (See [here](https://github.com/ava-labs/avalanchego/blob/master/network/peer/test_peer.go) for more information.)
 You can send messages through the test peer to the node it is attached to.
@@ -443,33 +599,40 @@ A node config is defined by this struct:
 
 ```go
 type Config struct {
-  // Configuration specific to a particular implementation of a node.
-  ImplSpecificConfig interface{}
   // A node's name must be unique from all other nodes
   // in a network. If Name is the empty string, a
   // unique name is assigned on node creation.
-  Name string
+  Name string `json:"name"`
   // True if other nodes should use this node
   // as a bootstrap beacon.
-  IsBeacon bool
-  // Must not be nil
-  StakingKey []byte
-  // Must not be nil
-  StakingCert []byte
+  IsBeacon bool `json:"isBeacon"`
+  // Must not be nil.
+  StakingKey string `json:"stakingKey"`
+  // Must not be nil.
+  StakingCert string `json:"stakingCert"`
   // May be nil.
-  ConfigFile []byte
+  ConfigFile string `json:"configFile"`
   // May be nil.
-  CChainConfigFile []byte
+  ChainConfigFiles map[string]string `json:"chainConfigFiles"`
+  // Flags can hold additional flags for the node.
+  // It can be empty.
+  // The precedence of flags handling is:
+  // 1. Flags defined in node.Config (this struct) override
+  // 2. Flags defined in network.Config override
+  // 3. Flags defined in the json config file
+  Flags map[string]interface{} `json:"flags"`
+  // What type of node this is
+  BinaryPath string `json:"binaryPath"`
+  // If non-nil, direct this node's Stdout to os.Stdout
+  RedirectStdout bool `json:"redirectStdout"`
+  // If non-nil, direct this node's Stderr to os.Stderr
+  RedirectStderr bool `json:"redirectStderr"`
 }
 ```
 
 As you can see, some fields of the config must be set, while others will be auto-generated if not provided.
 Bootstrap IPs/ IDs will be overwritten even if provided.
 
-A node's configuration may include fields that are specific to the type of network runner being used (see `ImplSpecificConfig` in the struct above.)
-For example, a node running in a Kubernetes cluster has a config field that specifies the Docker image that the node runs,
-whereas a node running locally has a config field that specifies the path of the binary that the node runs.
-
 ## Genesis Generation
 
 You can create a custom AvalancheGo genesis with function `network.NewAvalancheGoGenesis`:
@@ -494,25 +657,28 @@ Later on the genesis contents can be used in network creation.
 
 ## Network Creation
 
-Each network runner implementation (local/Kubernetes) has a function `NewNetwork` that returns a new network,
-parameterized on `network.Config`:
+Th function `NewNetwork` returns a new network, parameterized on `network.Config`:
 
 ```go
 type Config struct {
-  // Configuration specific to a particular implementation of a network.
-  ImplSpecificConfig interface{}
-  // Must not be nil
-  Genesis []byte
+  // Must not be empty
+  Genesis string `json:"genesis"`
   // May have length 0
   // (i.e. network may have no nodes on creation.)
-  NodeConfigs []node.Config
-  // Log level for the whole network
-  LogLevel string
-  // Name for the network
-  Name string
-  // How many nodes in the network.
-  // TODO move to k8s package?
-  NodeCount int
+  NodeConfigs []node.Config `json:"nodeConfigs"`
+  // Flags that will be passed to each node in this network.
+  // It can be empty.
+  // Config flags may also be passed in a node's config struct
+  // or config file.
+  // The precedence of flags handling is, from highest to lowest:
+  // 1. Flags defined in a node's node.Config
+  // 2. Flags defined in a network's network.Config
+  // 3. Flags defined in a node's config file
+  // For example, if a network.Config has flag W set to X,
+  // and a node within that network has flag W set to Y,
+  // and the node's config file has flag W set to Z,
+  // then the node will be started with flag W set to Y.
+  Flags map[string]interface{} `json:"flags"`
 }
 ```
 
@@ -520,7 +686,7 @@ The function that returns a new network may have additional configuration fields
 
 ## Default Network Creation
 
-The local network runner implementation includes a helper function `NewDefaultNetwork`, which returns a network using a pre-defined configuration.
+The helper function `NewDefaultNetwork` returns a network using a pre-defined configuration.
 This allows users to create a new network without needing to define any configurations.
 
 ```go
@@ -551,6 +717,24 @@ func NewDefaultNetwork(
 
 The associated pre-defined configuration is also available to users by calling `NewDefaultConfig` function.
 
+## Network Snapshots
+
+A given network state, including the node ports and the full blockchain state, can be saved to a named snapshot. 
+The network can then be restarted from such a snapshot any time later.
+
+```
+// Save network snapshot
+// Network is stopped in order to do a safe persistence
+// Returns the full local path to the snapshot dir
+SaveSnapshot(context.Context, string) (string, error)
+// Remove network snapshot
+RemoveSnapshot(string) error
+// Get names of all available snapshots
+GetSnapshotNames() ([]string, error)
+```
+
+To create a new network from a snapshot, the function `NewNetworkFromSnapshot` is provided.
+
 ## Network Interaction
 
 The network runner allows users to interact with an AvalancheGo network using the `network.Network` interface:
@@ -558,46 +742,71 @@ The network runner allows users to interact with an AvalancheGo network using th
 ```go
 // Network is an abstraction of an Avalanche network
 type Network interface {
-  // Returns a chan that is closed when
-  // all the nodes in the network are healthy.
-  // If an error is sent on this channel, at least 1
-  // node didn't become healthy before the timeout.
-  // If an error isn't sent on the channel before it
-  // closes, all the nodes are healthy.
-  // A stopped network is considered unhealthy.
-  // Timeout is given by the context parameter.
-  // [ctx] must eventually be cancelled -- if it isn't, a goroutine is leaked.
-  Healthy(context.Context) chan error
-  // Stop all the nodes.
-  // Returns ErrStopped if Stop() was previously called.
-  Stop(context.Context) error
-  // Start a new node with the given config.
-  // Returns ErrStopped if Stop() was previously called.
-  AddNode(node.Config) (node.Node, error)
-  // Stop the node with this name.
-  // Returns ErrStopped if Stop() was previously called.
-  RemoveNode(name string) error
-  // Return the node with this name.
-  // Returns ErrStopped if Stop() was previously called.
-  GetNode(name string) (node.Node, error)
-  // Returns the names of all nodes in this network.
-  // Returns ErrStopped if Stop() was previously called.
-  GetNodeNames() ([]string, error)
-  // TODO add methods
+	// Returns nil if all the nodes in the network are healthy.
+	// A stopped network is considered unhealthy.
+	// Timeout is given by the context parameter.
+	Healthy(context.Context) error
+	// Stop all the nodes.
+	// Returns ErrStopped if Stop() was previously called.
+	Stop(context.Context) error
+	// Start a new node with the given config.
+	// Returns ErrStopped if Stop() was previously called.
+	AddNode(node.Config) (node.Node, error)
+	// Stop the node with this name.
+	// Returns ErrStopped if Stop() was previously called.
+	RemoveNode(name string) error
+	// Return the node with this name.
+	// Returns ErrStopped if Stop() was previously called.
+	GetNode(name string) (node.Node, error)
+	// Return all the nodes in this network.
+	// Node name --> Node.
+	// Returns ErrStopped if Stop() was previously called.
+	GetAllNodes() (map[string]node.Node, error)
+	// Returns the names of all nodes in this network.
+	// Returns ErrStopped if Stop() was previously called.
+	GetNodeNames() ([]string, error)
+	// Save network snapshot
+	// Network is stopped in order to do a safe preservation
+    // Returns the full local path to the snapshot dir
+	SaveSnapshot(context.Context, string) (string, error)
+	// Remove network snapshot
+	RemoveSnapshot(string) error
+	// Get name of available snapshots
+	GetSnapshotNames() ([]string, error)
 }
 ```
 
 and allows users to interact with a node using the `node.Node` interface:
 
 ```go
-// An AvalancheGo node
+// Node represents an AvalancheGo node
 type Node interface {
-    // Return this node's name, which is unique
-    // across all the nodes in its network.
-    GetName() string
-    // Return this node's Avalanche node ID.
-    GetNodeID() ids.ShortID
-    // Return a client that can be used to make API calls.
-    GetAPIClient() api.Client
+	// Return this node's name, which is unique
+	// across all the nodes in its network.
+	GetName() string
+	// Return this node's Avalanche node ID.
+	GetNodeID() ids.ShortID
+	// Return a client that can be used to make API calls.
+	GetAPIClient() api.Client
+	// Return this node's IP (e.g. 127.0.0.1).
+	GetURL() string
+	// Return this node's P2P (staking) port.
+	GetP2PPort() uint16
+	// Return this node's HTTP API port.
+	GetAPIPort() uint16
+	// Starts a new test peer, connects it to the given node, and returns the peer.
+	// [handler] defines how the test peer handles messages it receives.
+	// The test peer can be used to send messages to the node it's attached to.
+	// It's left to the caller to maintain a reference to the returned peer.
+	// The caller should call StartClose() on the peer when they're done with it.
+	AttachPeer(ctx context.Context, handler router.InboundHandler) (peer.Peer, error)
+	// Return this node's avalanchego binary path
+	GetBinaryPath() string
+	// Return this node's db dir
+	GetDbDir() string
+	// Return this node's logs dir
+	GetLogsDir() string
+	// Return this node's config file contents
+	GetConfigFile() string
 }
 ```

api/api_client.go

@@ -53,8 +53,8 @@ func NewAPIClient(ipAddr string, port uint16) Client {
 		ipcs:         ipcs.NewClient(uri),
 		keystore:     keystore.NewClient(uri),
 		admin:        admin.NewClient(uri),
-		pindex:       indexer.NewClient(uri, "/ext/index/P/block"),
-		cindex:       indexer.NewClient(uri, "/ext/index/C/block"),
+		pindex:       indexer.NewClient(uri + "/ext/index/P/block"),
+		cindex:       indexer.NewClient(uri + "/ext/index/C/block"),
 	}
 }
 

avalanche-network-runner-sdk/.gitignore

@@ -1,10 +0,0 @@
-# Generated by Cargo
-# will have compiled files and executables
-/target/
-
-# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
-# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
-Cargo.lock
-
-# These are backup files generated by rustfmt
-**/*.rs.bk

avalanche-network-runner-sdk/Cargo.toml

@@ -1,26 +0,0 @@
-[package]
-name = "avalanche-network-runner-sdk"
-version = "0.0.0"
-edition = "2021"
-rust-version = "1.60"
-publish = false
-description = "avalanche-network-runner SDK in Rust"
-license = "BSD-3-Clause"
-homepage = "https://avax.network"
-repository = "https://github.com/ava-labs/avalanche-network-runner"
-readme = "README.md"
-
-[dependencies]
-log = "0.4.16"
-prost = "0.10.1"
-prost-types = "0.10.1"
-tokio = { version = "1.17.0", features = ["fs", "rt-multi-thread"] }
-tokio-stream = { version = "0.1.8", features = ["net"] }
-tonic = "0.7.1"
-
-[build-dependencies]
-# ref. https://github.com/hyperium/tonic/tree/master/tonic-build
-tonic-build = "0.7.0"
-
-[dev-dependencies]
-env_logger = "0.9.0"

avalanche-network-runner-sdk/README.md

@@ -1,5 +0,0 @@
-
-## Avalanche network runner Rust SDK
-
-Hello!
-

avalanche-network-runner-sdk/build.rs

@@ -1,14 +0,0 @@
-/// ref. https://github.com/hyperium/tonic/tree/master/tonic-build
-fn main() {
-    tonic_build::configure()
-        .build_server(false)
-        .build_client(true)
-        .compile(
-            &[
-                "googleapis/google/pubsub/v1/pubsub.proto",
-                "../rpcpb/rpc.proto",
-            ],
-            &["googleapis", "../rpcpb"],
-        )
-        .unwrap();
-}

avalanche-network-runner-sdk/examples/ping.rs

@@ -1,24 +0,0 @@
-use std::env::args;
-
-use log::info;
-use tokio::runtime::Runtime;
-
-use avalanche_network_runner_sdk::Client;
-
-/// cargo run --example ping -- [HTTP RPC ENDPOINT]
-/// cargo run --example ping -- http://127.0.0.1:8080
-fn main() {
-    // ref. https://github.com/env-logger-rs/env_logger/issues/47
-    env_logger::init_from_env(
-        env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
-    );
-
-    let url = args().nth(1).expect("no url given");
-    let rt = Runtime::new().unwrap();
-
-    info!("creating client");
-    let cli = rt.block_on(Client::new(&url));
-
-    let resp = rt.block_on(cli.ping()).expect("failed ping");
-    info!("ping response: {:?}", resp);
-}

avalanche-network-runner-sdk/examples/start.rs

@@ -1,32 +0,0 @@
-use std::env::args;
-
-use log::info;
-use tokio::runtime::Runtime;
-
-use avalanche_network_runner_sdk::{rpcpb::StartRequest, Client};
-
-/// cargo run --example start -- [HTTP RPC ENDPOINT] [EXEC PATH]
-/// cargo run --example start -- http://127.0.0.1:8080 /Users/gyuho.lee/go/src/github.com/ava-labs/avalanchego/build/avalanchego
-fn main() {
-    // ref. https://github.com/env-logger-rs/env_logger/issues/47
-    env_logger::init_from_env(
-        env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
-    );
-
-    let url = args().nth(1).expect("no url given");
-    let exec_path = args().nth(2).expect("no exec path given");
-    let rt = Runtime::new().unwrap();
-
-    info!("creating client");
-    let cli = rt.block_on(Client::new(&url));
-
-    let resp = rt
-        .block_on(cli.start(StartRequest {
-            exec_path,
-            num_nodes: Some(5),
-            log_level: Some(String::from("INFO")),
-            ..Default::default()
-        }))
-        .expect("failed start");
-    info!("start response: {:?}", resp);
-}

avalanche-network-runner-sdk/examples/status.rs

@@ -1,24 +0,0 @@
-use std::env::args;
-
-use log::info;
-use tokio::runtime::Runtime;
-
-use avalanche_network_runner_sdk::Client;
-
-/// cargo run --example status -- [HTTP RPC ENDPOINT]
-/// cargo run --example status -- http://127.0.0.1:8080
-fn main() {
-    // ref. https://github.com/env-logger-rs/env_logger/issues/47
-    env_logger::init_from_env(
-        env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
-    );
-
-    let url = args().nth(1).expect("no url given");
-    let rt = Runtime::new().unwrap();
-
-    info!("creating client");
-    let cli = rt.block_on(Client::new(&url));
-
-    let resp = rt.block_on(cli.status()).expect("failed status");
-    info!("status response: {:?}", resp);
-}

avalanche-network-runner-sdk/examples/stop.rs

@@ -1,24 +0,0 @@
-use std::env::args;
-
-use log::info;
-use tokio::runtime::Runtime;
-
-use avalanche_network_runner_sdk::Client;
-
-/// cargo run --example stop -- [HTTP RPC ENDPOINT]
-/// cargo run --example stop -- http://127.0.0.1:8080
-fn main() {
-    // ref. https://github.com/env-logger-rs/env_logger/issues/47
-    env_logger::init_from_env(
-        env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
-    );
-
-    let url = args().nth(1).expect("no url given");
-    let rt = Runtime::new().unwrap();
-
-    info!("creating client");
-    let cli = rt.block_on(Client::new(&url));
-
-    let resp = rt.block_on(cli.stop()).expect("failed stop");
-    info!("stop response: {:?}", resp);
-}

avalanche-network-runner-sdk/examples/uris.rs

@@ -1,24 +0,0 @@
-use std::env::args;
-
-use log::info;
-use tokio::runtime::Runtime;
-
-use avalanche_network_runner_sdk::Client;
-
-/// cargo run --example uris -- [HTTP RPC ENDPOINT]
-/// cargo run --example uris -- http://127.0.0.1:8080
-fn main() {
-    // ref. https://github.com/env-logger-rs/env_logger/issues/47
-    env_logger::init_from_env(
-        env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
-    );
-
-    let url = args().nth(1).expect("no url given");
-    let rt = Runtime::new().unwrap();
-
-    info!("creating client");
-    let cli = rt.block_on(Client::new(&url));
-
-    let resp = rt.block_on(cli.uris()).expect("failed uris");
-    info!("uris response: {:?}", resp);
-}

avalanche-network-runner-sdk/scripts/build.release.sh

@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-set -xue
-
-if ! [[ "$0" =~ scripts/build.release.sh ]]; then
-  echo "must be run from repository root"
-  exit 255
-fi
-
-# git submodule add https://github.com/googleapis/googleapis
-git submodule update --init --remote
-
-cargo build --release

avalanche-network-runner-sdk/scripts/cargo.unused.sh

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-set -xue
-
-if ! [[ "$0" =~ scripts/cargo.unused.sh ]]; then
-  echo "must be run from repository root"
-  exit 255
-fi
-
-# git submodule add https://github.com/googleapis/googleapis
-git submodule update --init --remote
-
-# https://github.com/est31/cargo-udeps
-cargo install cargo-udeps --locked
-cargo +nightly udeps
-
-echo "ALL SUCCESS!"

avalanche-network-runner-sdk/scripts/static-analysis.sh

@@ -1,38 +0,0 @@
-#!/usr/bin/env bash
-set -xue
-
-if ! [[ "$0" =~ scripts/static-analysis.sh ]]; then
-  echo "must be run from repository root"
-  exit 255
-fi
-
-# git submodule add https://github.com/googleapis/googleapis
-git submodule update --init --remote
-
-# check https://www.rust-lang.org/tools/install for Rust compiler installation
-# e.g.,
-# curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
-#
-# install Rust nightly builds https://rust-lang.github.io/rustup/installation/index.html
-# e.g.,
-# rustup toolchain install nightly --allow-downgrade --profile minimal --component clippy
-#
-# https://github.com/rust-lang/rustfmt
-# rustup component add rustfmt
-# rustup component add rustfmt --toolchain nightly
-# rustup component add clippy
-# rustup component add clippy --toolchain nightly
-
-rustup default stable
-cargo fmt --all --verbose -- --check
-
-# TODO: enable nightly fmt
-rustup default nightly
-cargo +nightly fmt --all -- --config-path .rustfmt.nightly.toml --verbose --check || true
-
-# TODO: remove "|| true"
-cargo +nightly clippy --all --all-features -- -D warnings || true
-
-rustup default stable
-
-echo "ALL SUCCESS!"

avalanche-network-runner-sdk/scripts/tests.unit.sh

@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-set -xue
-
-if ! [[ "$0" =~ scripts/tests.unit.sh ]]; then
-  echo "must be run from repository root"
-  exit 255
-fi
-
-# git submodule add https://github.com/googleapis/googleapis
-git submodule update --init --remote
-
-RUST_LOG=debug cargo test --all --all-features -- --show-output
-
-echo "ALL SUCCESS!"

avalanche-network-runner-sdk/src/lib.rs

@@ -1,127 +0,0 @@
-use std::{
-    io::{self, Error, ErrorKind},
-    sync::{Arc, Mutex},
-};
-
-use log::info;
-use tonic::transport::Channel;
-
-pub mod rpcpb {
-    tonic::include_proto!("rpcpb");
-}
-use rpcpb::{
-    control_service_client::ControlServiceClient, ping_service_client::PingServiceClient,
-    HealthRequest, HealthResponse, PingRequest, PingResponse, StartRequest, StartResponse,
-    StatusRequest, StatusResponse, StopRequest, StopResponse, UrIsRequest,
-};
-
-pub struct Client<T> {
-    pub rpc_endpoint: String,
-    /// Shared gRPC client connections.
-    pub grpc_client: Arc<GrpcClient<T>>,
-}
-
-pub struct GrpcClient<T> {
-    pub ping_client: Mutex<PingServiceClient<T>>,
-    pub control_client: Mutex<ControlServiceClient<T>>,
-}
-
-impl Client<Channel> {
-    /// Creates a new network-runner client.
-    ///
-    /// # Arguments
-    ///
-    /// * `rpc_endpoint` - HTTP RPC endpoint to the network runner server.
-    pub async fn new(rpc_endpoint: &str) -> Self {
-        info!("creating a new client with {}", rpc_endpoint);
-        let ep = String::from(rpc_endpoint);
-        let ping_client = PingServiceClient::connect(ep.clone()).await.unwrap();
-        let control_client = ControlServiceClient::connect(ep).await.unwrap();
-        let grpc_client = GrpcClient {
-            ping_client: Mutex::new(ping_client),
-            control_client: Mutex::new(control_client),
-        };
-        Self {
-            rpc_endpoint: String::from(rpc_endpoint),
-            grpc_client: Arc::new(grpc_client),
-        }
-    }
-
-    /// Pings the network-runner server.
-    pub async fn ping(&self) -> io::Result<PingResponse> {
-        let mut ping_client = self.grpc_client.ping_client.lock().unwrap();
-        let req = tonic::Request::new(PingRequest {});
-        let resp = ping_client
-            .ping(req)
-            .await
-            .map_err(|e| Error::new(ErrorKind::Other, format!("failed ping '{}'", e)))?;
-
-        let ping_resp = resp.into_inner();
-        Ok(ping_resp)
-    }
-
-    /// Starts a cluster.
-    pub async fn start(&self, req: StartRequest) -> io::Result<StartResponse> {
-        let mut control_client = self.grpc_client.control_client.lock().unwrap();
-        let req = tonic::Request::new(req);
-        let resp = control_client
-            .start(req)
-            .await
-            .map_err(|e| Error::new(ErrorKind::Other, format!("failed stop '{}'", e)))?;
-
-        let start_resp = resp.into_inner();
-        Ok(start_resp)
-    }
-
-    /// Fetches the current cluster health information via network-runner.
-    pub async fn health(&self) -> io::Result<HealthResponse> {
-        let mut control_client = self.grpc_client.control_client.lock().unwrap();
-        let req = tonic::Request::new(HealthRequest {});
-        let resp = control_client
-            .health(req)
-            .await
-            .map_err(|e| Error::new(ErrorKind::Other, format!("failed status '{}'", e)))?;
-
-        let health_resp = resp.into_inner();
-        Ok(health_resp)
-    }
-
-    /// Fetches the URIs for the current cluster.
-    pub async fn uris(&self) -> io::Result<Vec<String>> {
-        let mut control_client = self.grpc_client.control_client.lock().unwrap();
-        let req = tonic::Request::new(UrIsRequest {});
-        let resp = control_client
-            .ur_is(req)
-            .await
-            .map_err(|e| Error::new(ErrorKind::Other, format!("failed status '{}'", e)))?;
-
-        let uris_resp = resp.into_inner();
-        Ok(uris_resp.uris)
-    }
-
-    /// Fetches the current cluster status via network-runner.
-    pub async fn status(&self) -> io::Result<StatusResponse> {
-        let mut control_client = self.grpc_client.control_client.lock().unwrap();
-        let req = tonic::Request::new(StatusRequest {});
-        let resp = control_client
-            .status(req)
-            .await
-            .map_err(|e| Error::new(ErrorKind::Other, format!("failed status '{}'", e)))?;
-
-        let status_resp = resp.into_inner();
-        Ok(status_resp)
-    }
-
-    /// Stop the currently running cluster.
-    pub async fn stop(&self) -> io::Result<StopResponse> {
-        let mut control_client = self.grpc_client.control_client.lock().unwrap();
-        let req = tonic::Request::new(StopRequest {});
-        let resp = control_client
-            .stop(req)
-            .await
-            .map_err(|e| Error::new(ErrorKind::Other, format!("failed stop '{}'", e)))?;
-
-        let stop_resp = resp.into_inner();
-        Ok(stop_resp)
-    }
-}

client/client.go

@@ -13,7 +13,6 @@ import (
 	"time"
 
 	"github.com/ava-labs/avalanche-network-runner/local"
-	"github.com/ava-labs/avalanche-network-runner/pkg/color"
 	"github.com/ava-labs/avalanche-network-runner/pkg/logutil"
 	"github.com/ava-labs/avalanche-network-runner/rpcpb"
 	"go.uber.org/zap"
@@ -32,6 +31,8 @@ type Config struct {
 type Client interface {
 	Ping(ctx context.Context) (*rpcpb.PingResponse, error)
 	Start(ctx context.Context, execPath string, opts ...OpOption) (*rpcpb.StartResponse, error)
+	CreateBlockchains(ctx context.Context, blockchainSpecs []*rpcpb.BlockchainSpec) (*rpcpb.CreateBlockchainsResponse, error)
+	CreateSubnets(ctx context.Context, opts ...OpOption) (*rpcpb.CreateSubnetsResponse, error)
 	Health(ctx context.Context) (*rpcpb.HealthResponse, error)
 	URIs(ctx context.Context) ([]string, error)
 	Status(ctx context.Context) (*rpcpb.StatusResponse, error)
@@ -43,6 +44,10 @@ type Client interface {
 	AttachPeer(ctx context.Context, nodeName string) (*rpcpb.AttachPeerResponse, error)
 	SendOutboundMessage(ctx context.Context, nodeName string, peerID string, op uint32, msgBody []byte) (*rpcpb.SendOutboundMessageResponse, error)
 	Close() error
+	SaveSnapshot(ctx context.Context, snapshotName string) (*rpcpb.SaveSnapshotResponse, error)
+	LoadSnapshot(ctx context.Context, snapshotName string, opts ...OpOption) (*rpcpb.LoadSnapshotResponse, error)
+	RemoveSnapshot(ctx context.Context, snapshotName string) (*rpcpb.RemoveSnapshotResponse, error)
+	GetSnapshotNames(ctx context.Context) ([]string, error)
 }
 
 type client struct {
@@ -66,7 +71,8 @@ func New(cfg Config) (Client, error) {
 	}
 	_ = zap.ReplaceGlobals(logger)
 
-	color.Outf("{{blue}}dialing endpoint %q{{/}}\n", cfg.Endpoint)
+	zap.L().Debug("dialing server", zap.String("endpoint", cfg.Endpoint))
+
 	ctx, cancel := context.WithTimeout(context.Background(), cfg.DialTimeout)
 	conn, err := grpc.DialContext(
 		ctx,
@@ -108,9 +114,6 @@ func (c *client) Start(ctx context.Context, execPath string, opts ...OpOption) (
 	if ret.whitelistedSubnets != "" {
 		req.WhitelistedSubnets = &ret.whitelistedSubnets
 	}
-	if ret.logLevel != "" {
-		req.LogLevel = &ret.logLevel
-	}
 	if ret.rootDataDir != "" {
 		req.RootDataDir = &ret.rootDataDir
 	}
@@ -120,11 +123,40 @@ func (c *client) Start(ctx context.Context, execPath string, opts ...OpOption) (
 	if len(ret.customVMs) > 0 {
 		req.CustomVms = ret.customVMs
 	}
+	if ret.globalNodeConfig != "" {
+		req.GlobalNodeConfig = &ret.globalNodeConfig
+	}
+	if ret.customNodeConfigs != nil {
+		req.CustomNodeConfigs = ret.customNodeConfigs
+	}
 
 	zap.L().Info("start")
 	return c.controlc.Start(ctx, req)
 }
 
+func (c *client) CreateBlockchains(ctx context.Context, blockchainSpecs []*rpcpb.BlockchainSpec) (*rpcpb.CreateBlockchainsResponse, error) {
+	req := &rpcpb.CreateBlockchainsRequest{
+		BlockchainSpecs: blockchainSpecs,
+	}
+
+	zap.L().Info("create blockchains")
+	return c.controlc.CreateBlockchains(ctx, req)
+}
+
+func (c *client) CreateSubnets(ctx context.Context, opts ...OpOption) (*rpcpb.CreateSubnetsResponse, error) {
+	ret := &Op{}
+	ret.applyOpts(opts)
+
+	req := &rpcpb.CreateSubnetsRequest{}
+
+	if ret.numSubnets != 0 {
+		req.NumSubnets = &ret.numSubnets
+	}
+
+	zap.L().Info("create subnets")
+	return c.controlc.CreateSubnets(ctx, req)
+}
+
 func (c *client) Health(ctx context.Context) (*rpcpb.HealthResponse, error) {
 	zap.L().Info("health")
 	return c.controlc.Health(ctx, &rpcpb.HealthRequest{})
@@ -200,13 +232,13 @@ func (c *client) AddNode(ctx context.Context, name string, execPath string, opts
 	ret := &Op{}
 	ret.applyOpts(opts)
 
-	req := &rpcpb.AddNodeRequest{Name: name}
+	req := &rpcpb.AddNodeRequest{
+		Name:         name,
+		StartRequest: &rpcpb.StartRequest{},
+	}
 	if ret.whitelistedSubnets != "" {
 		req.StartRequest.WhitelistedSubnets = &ret.whitelistedSubnets
 	}
-	if ret.logLevel != "" {
-		req.StartRequest.LogLevel = &ret.logLevel
-	}
 	if ret.execPath != "" {
 		req.StartRequest.ExecPath = ret.execPath
 	}
@@ -235,9 +267,6 @@ func (c *client) RestartNode(ctx context.Context, name string, opts ...OpOption)
 	if ret.whitelistedSubnets != "" {
 		req.WhitelistedSubnets = &ret.whitelistedSubnets
 	}
-	if ret.logLevel != "" {
-		req.LogLevel = &ret.logLevel
-	}
 	if ret.rootDataDir != "" {
 		req.RootDataDir = &ret.rootDataDir
 	}
@@ -261,6 +290,44 @@ func (c *client) SendOutboundMessage(ctx context.Context, nodeName string, peerI
 	})
 }
 
+func (c *client) SaveSnapshot(ctx context.Context, snapshotName string) (*rpcpb.SaveSnapshotResponse, error) {
+	zap.L().Info("save snapshot", zap.String("snapshot-name", snapshotName))
+	return c.controlc.SaveSnapshot(ctx, &rpcpb.SaveSnapshotRequest{SnapshotName: snapshotName})
+}
+
+func (c *client) LoadSnapshot(ctx context.Context, snapshotName string, opts ...OpOption) (*rpcpb.LoadSnapshotResponse, error) {
+	zap.L().Info("load snapshot", zap.String("snapshot-name", snapshotName))
+	ret := &Op{}
+	ret.applyOpts(opts)
+	req := rpcpb.LoadSnapshotRequest{
+		SnapshotName: snapshotName,
+	}
+	if ret.execPath != "" {
+		req.ExecPath = &ret.execPath
+	}
+	if ret.pluginDir != "" {
+		req.PluginDir = &ret.pluginDir
+	}
+	if ret.rootDataDir != "" {
+		req.RootDataDir = &ret.rootDataDir
+	}
+	return c.controlc.LoadSnapshot(ctx, &req)
+}
+
+func (c *client) RemoveSnapshot(ctx context.Context, snapshotName string) (*rpcpb.RemoveSnapshotResponse, error) {
+	zap.L().Info("remove snapshot", zap.String("snapshot-name", snapshotName))
+	return c.controlc.RemoveSnapshot(ctx, &rpcpb.RemoveSnapshotRequest{SnapshotName: snapshotName})
+}
+
+func (c *client) GetSnapshotNames(ctx context.Context) ([]string, error) {
+	zap.L().Info("get snapshot names")
+	resp, err := c.controlc.GetSnapshotNames(ctx, &rpcpb.GetSnapshotNamesRequest{})
+	if err != nil {
+		return nil, err
+	}
+	return resp.SnapshotNames, nil
+}
+
 func (c *client) Close() error {
 	c.closeOnce.Do(func() {
 		close(c.closed)
@@ -272,11 +339,12 @@ type Op struct {
 	numNodes           uint32
 	execPath           string
 	whitelistedSubnets string
-	logLevel           string
+	globalNodeConfig   string
 	rootDataDir        string
-
-	pluginDir    string
-	customVMs    map[string]string
+	pluginDir          string
+	customVMs          map[string]string
+	customNodeConfigs  map[string]string
+	numSubnets         uint32
 	chainConfigs map[string]string
 }
 
@@ -288,6 +356,12 @@ func (op *Op) applyOpts(opts []OpOption) {
 	}
 }
 
+func WithGlobalNodeConfig(nodeConfig string) OpOption {
+	return func(op *Op) {
+		op.globalNodeConfig = nodeConfig
+	}
+}
+
 func WithNumNodes(numNodes uint32) OpOption {
 	return func(op *Op) {
 		op.numNodes = numNodes
@@ -306,12 +380,6 @@ func WithWhitelistedSubnets(whitelistedSubnets string) OpOption {
 	}
 }
 
-func WithLogLevel(logLevel string) OpOption {
-	return func(op *Op) {
-		op.logLevel = logLevel
-	}
-}
-
 func WithRootDataDir(rootDataDir string) OpOption {
 	return func(op *Op) {
 		op.rootDataDir = rootDataDir
@@ -335,6 +403,17 @@ func WithCustomVMs(customVMs map[string]string) OpOption {
 func WithChainConfigs(chainConfigs map[string]string) OpOption {
 	return func(op *Op) {
 		op.chainConfigs = chainConfigs
+
+// Map from node name to its custom node config
+func WithCustomNodeConfigs(customNodeConfigs map[string]string) OpOption {
+	return func(op *Op) {
+		op.customNodeConfigs = customNodeConfigs
+	}
+}
+
+func WithNumSubnets(numSubnets uint32) OpOption {
+	return func(op *Op) {
+		op.numSubnets = numSubnets
 	}
 }
 

cmd/avalanche-network-runner/control/control.go

@@ -7,6 +7,8 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/json"
+	"errors"
+	"fmt"
 	"os"
 	"os/signal"
 	"syscall"
@@ -16,6 +18,7 @@ import (
 	"github.com/ava-labs/avalanche-network-runner/local"
 	"github.com/ava-labs/avalanche-network-runner/pkg/color"
 	"github.com/ava-labs/avalanche-network-runner/pkg/logutil"
+	"github.com/ava-labs/avalanche-network-runner/rpcpb"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
 )
@@ -25,12 +28,15 @@ func init() {
 }
 
 var (
-	logLevel       string
-	endpoint       string
-	dialTimeout    time.Duration
-	requestTimeout time.Duration
+	logLevel           string
+	whitelistedSubnets string
+	endpoint           string
+	dialTimeout        time.Duration
+	requestTimeout     time.Duration
 )
 
+// NOTE: Naming convention for node names is currently `node` + number, i.e. `node1,node2,node3,...node101`
+
 func NewCommand() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "control [options]",
@@ -44,6 +50,8 @@ func NewCommand() *cobra.Command {
 
 	cmd.AddCommand(
 		newStartCommand(),
+		newCreateBlockchainsCommand(),
+		newCreateSubnetsCommand(),
 		newHealthCommand(),
 		newURIsCommand(),
 		newStatusCommand(),
@@ -54,6 +62,10 @@ func NewCommand() *cobra.Command {
 		newAttachPeerCommand(),
 		newSendOutboundMessageCommand(),
 		newStopCommand(),
+		newSaveSnapshotCommand(),
+		newLoadSnapshotCommand(),
+		newRemoveSnapshotCommand(),
+		newGetSnapshotNamesCommand(),
 	)
 
 	return cmd
@@ -63,7 +75,12 @@ var (
 	avalancheGoBinPath        string
 	numNodes                  uint32
 	pluginDir                 string
+	globalNodeConfig          string
+	addNodeConfig             string
 	customVMNameToGenesisPath string
+	customNodeConfigs         string
+	rootDataDir               string
+	numSubnets                uint32
 )
 
 func newStartCommand() *cobra.Command {
@@ -71,6 +88,7 @@ func newStartCommand() *cobra.Command {
 		Use:   "start [options]",
 		Short: "Starts the server.",
 		RunE:  startFunc,
+		Args:  cobra.ExactArgs(0),
 	}
 	cmd.PersistentFlags().StringVar(
 		&avalancheGoBinPath,
@@ -90,21 +108,41 @@ func newStartCommand() *cobra.Command {
 		"",
 		"[optional] plugin directory",
 	)
+	cmd.PersistentFlags().StringVar(
+		&rootDataDir,
+		"root-data-dir",
+		"",
+		"[optional] root data directory to store logs and configurations",
+	)
 	cmd.PersistentFlags().StringVar(
 		&customVMNameToGenesisPath,
 		"custom-vms",
 		"",
 		"[optional] JSON string of map that maps from VM to its genesis file path",
 	)
+	cmd.PersistentFlags().StringVar(
+		&globalNodeConfig,
+		"global-node-config",
+		"",
+		"[optional] global node config as JSON string, applied to all nodes",
+	)
+	cmd.PersistentFlags().StringVar(
+		&customNodeConfigs,
+		"custom-node-configs",
+		"",
+		"[optional] custom node configs as JSON string of map, for each node individually. Common entries override `global-node-config`, but can be combined. Invalidates `number-of-nodes` (provide all node configs if used).",
+	)
+	cmd.PersistentFlags().StringVar(
+		&whitelistedSubnets,
+		"whitelisted-subnets",
+		"",
+		"whitelisted subnets (comma-separated)",
+	)
 	return cmd
 }
 
 func startFunc(cmd *cobra.Command, args []string) error {
-	cli, err := client.New(client.Config{
-		LogLevel:    logLevel,
-		Endpoint:    endpoint,
-		DialTimeout: dialTimeout,
-	})
+	cli, err := newClient()
 	if err != nil {
 		return err
 	}
@@ -113,23 +151,38 @@ func startFunc(cmd *cobra.Command, args []string) error {
 	opts := []client.OpOption{
 		client.WithNumNodes(numNodes),
 		client.WithPluginDir(pluginDir),
+		client.WithWhitelistedSubnets(whitelistedSubnets),
+		client.WithRootDataDir(rootDataDir),
 	}
+
+	if globalNodeConfig != "" {
+		color.Outf("{{yellow}} global node config provided, will be applied to all nodes{{/}} %+v\n", globalNodeConfig)
+
+		// validate it's valid JSON
+		var js json.RawMessage
+		if err := json.Unmarshal([]byte(globalNodeConfig), &js); err != nil {
+			return fmt.Errorf("failed to validate JSON for provided config file: %s", err)
+		}
+		opts = append(opts, client.WithGlobalNodeConfig(globalNodeConfig))
+	}
+
+	if customNodeConfigs != "" {
+		nodeConfigs := make(map[string]string)
+		if err := json.Unmarshal([]byte(customNodeConfigs), &nodeConfigs); err != nil {
+			return err
+		}
+		opts = append(opts, client.WithCustomNodeConfigs(nodeConfigs))
+	}
+
 	if customVMNameToGenesisPath != "" {
 		customVMs := make(map[string]string)
-		err = json.Unmarshal([]byte(customVMNameToGenesisPath), &customVMs)
-		if err != nil {
+		if err := json.Unmarshal([]byte(customVMNameToGenesisPath), &customVMs); err != nil {
 			return err
 		}
 		opts = append(opts, client.WithCustomVMs(customVMs))
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
-	// don't call since "start" is async
-	// and the top-level context here "ctx" is passed
-	// to all underlying function calls
-	// just set the timeout to halt "Start" async ops
-	// when the deadline is reached
-	_ = cancel
+	ctx := getAsyncContext()
 
 	info, err := cli.Start(
 		ctx,
@@ -144,21 +197,108 @@ func startFunc(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
+func newCreateBlockchainsCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "create-blockchains [options]",
+		Short: "Create blockchains.",
+		RunE:  createBlockchainsFunc,
+		Args:  cobra.ExactArgs(0),
+	}
+	cmd.PersistentFlags().StringVar(
+		&customVMNameToGenesisPath,
+		"custom-vms",
+		"",
+		"JSON string of list of [(VM name, its genesis file path, optional subnet id to use)]",
+	)
+	if err := cmd.MarkPersistentFlagRequired("custom-vms"); err != nil {
+		panic(err)
+	}
+	return cmd
+}
+
+func createBlockchainsFunc(cmd *cobra.Command, args []string) error {
+	cli, err := newClient()
+	if err != nil {
+		return err
+	}
+	defer cli.Close()
+
+	if customVMNameToGenesisPath == "" {
+		return errors.New("empty custom-vms argument")
+	}
+
+	blockchainSpecs := []*rpcpb.BlockchainSpec{}
+	if err := json.Unmarshal([]byte(customVMNameToGenesisPath), &blockchainSpecs); err != nil {
+		return err
+	}
+
+	ctx := getAsyncContext()
+
+	info, err := cli.CreateBlockchains(
+		ctx,
+		blockchainSpecs,
+	)
+	if err != nil {
+		return err
+	}
+
+	color.Outf("{{green}}deploy-blockchains response:{{/}} %+v\n", info)
+	return nil
+}
+
+func newCreateSubnetsCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "create-subnets [options]",
+		Short: "Create subnets.",
+		RunE:  createSubnetsFunc,
+		Args:  cobra.ExactArgs(0),
+	}
+	cmd.PersistentFlags().Uint32Var(
+		&numSubnets,
+		"num-subnets",
+		0,
+		"number of subnets",
+	)
+	return cmd
+}
+
+func createSubnetsFunc(cmd *cobra.Command, args []string) error {
+	cli, err := newClient()
+	if err != nil {
+		return err
+	}
+	defer cli.Close()
+
+	opts := []client.OpOption{
+		client.WithNumSubnets(numSubnets),
+	}
+
+	ctx := getAsyncContext()
+
+	info, err := cli.CreateSubnets(
+		ctx,
+		opts...,
+	)
+	if err != nil {
+		return err
+	}
+
+	color.Outf("{{green}}add-subnets response:{{/}} %+v\n", info)
+	return nil
+}
+
 func newHealthCommand() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "health [options]",
 		Short: "Requests server health.",
 		RunE:  healthFunc,
+		Args:  cobra.ExactArgs(0),
 	}
 	return cmd
 }
 
 func healthFunc(cmd *cobra.Command, args []string) error {
-	cli, err := client.New(client.Config{
-		LogLevel:    logLevel,
-		Endpoint:    endpoint,
-		DialTimeout: dialTimeout,
-	})
+	cli, err := newClient()
 	if err != nil {
 		return err
 	}
@@ -180,16 +320,13 @@ func newURIsCommand() *cobra.Command {
 		Use:   "uris [options]",
 		Short: "Requests server uris.",
 		RunE:  urisFunc,
+		Args:  cobra.ExactArgs(0),
 	}
 	return cmd
 }
 
 func urisFunc(cmd *cobra.Command, args []string) error {
-	cli, err := client.New(client.Config{
-		LogLevel:    logLevel,
-		Endpoint:    endpoint,
-		DialTimeout: dialTimeout,
-	})
+	cli, err := newClient()
 	if err != nil {
 		return err
 	}
@@ -211,16 +348,13 @@ func newStatusCommand() *cobra.Command {
 		Use:   "status [options]",
 		Short: "Requests server status.",
 		RunE:  statusFunc,
+		Args:  cobra.ExactArgs(0),
 	}
 	return cmd
 }
 
 func statusFunc(cmd *cobra.Command, args []string) error {
-	cli, err := client.New(client.Config{
-		LogLevel:    logLevel,
-		Endpoint:    endpoint,
-		DialTimeout: dialTimeout,
-	})
+	cli, err := newClient()
 	if err != nil {
 		return err
 	}
@@ -244,6 +378,7 @@ func newStreamStatusCommand() *cobra.Command {
 		Use:   "stream-status [options]",
 		Short: "Requests server bootstrap status.",
 		RunE:  streamStatusFunc,
+		Args:  cobra.ExactArgs(0),
 	}
 	cmd.PersistentFlags().DurationVar(
 		&pushInterval,
@@ -255,11 +390,7 @@ func newStreamStatusCommand() *cobra.Command {
 }
 
 func streamStatusFunc(cmd *cobra.Command, args []string) error {
-	cli, err := client.New(client.Config{
-		LogLevel:    logLevel,
-		Endpoint:    endpoint,
-		DialTimeout: dialTimeout,
-	})
+	cli, err := newClient()
 	if err != nil {
 		return err
 	}
@@ -300,17 +431,14 @@ func newRemoveNodeCommand() *cobra.Command {
 		Use:   "remove-node [options]",
 		Short: "Removes a node.",
 		RunE:  removeNodeFunc,
+		Args:  cobra.ExactArgs(0),
 	}
 	cmd.PersistentFlags().StringVar(&nodeName, "node-name", "", "node name to remove")
 	return cmd
 }
 
 func removeNodeFunc(cmd *cobra.Command, args []string) error {
-	cli, err := client.New(client.Config{
-		LogLevel:    logLevel,
-		Endpoint:    endpoint,
-		DialTimeout: dialTimeout,
-	})
+	cli, err := newClient()
 	if err != nil {
 		return err
 	}
@@ -327,7 +455,83 @@ func removeNodeFunc(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
-var whitelistedSubnets string
+func newAddNodeCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "add-node [options]",
+		Short: "Add a new node to the network",
+		RunE:  addNodeFunc,
+		Args:  cobra.ExactArgs(0),
+	}
+	cmd.PersistentFlags().StringVar(
+		&nodeName,
+		"node-name",
+		"",
+		"node name to add",
+	)
+	cmd.PersistentFlags().StringVar(
+		&avalancheGoBinPath,
+		"avalanchego-path",
+		"",
+		"avalanchego binary path",
+	)
+	cmd.PersistentFlags().StringVar(
+		&customVMNameToGenesisPath,
+		"custom-vms",
+		"",
+		"[optional] JSON string of map that maps from VM to its genesis file path",
+	)
+	cmd.PersistentFlags().StringVar(
+		&addNodeConfig,
+		"node-config",
+		"",
+		"node config as string",
+	)
+	return cmd
+}
+
+func addNodeFunc(cmd *cobra.Command, args []string) error {
+	cli, err := newClient()
+	if err != nil {
+		return err
+	}
+	defer cli.Close()
+
+	opts := []client.OpOption{}
+
+	if addNodeConfig != "" {
+		color.Outf("{{yellow}}WARNING: overriding node configs with custom provided config {{/}} %+v\n", addNodeConfig)
+		// validate it's valid JSON
+		var js json.RawMessage
+		if err := json.Unmarshal([]byte(addNodeConfig), &js); err != nil {
+			return fmt.Errorf("failed to validate JSON for provided config file: %s", err)
+		}
+		opts = append(opts, client.WithGlobalNodeConfig(addNodeConfig))
+	}
+
+	if customVMNameToGenesisPath != "" {
+		customVMs := make(map[string]string)
+		err = json.Unmarshal([]byte(customVMNameToGenesisPath), &customVMs)
+		if err != nil {
+			return err
+		}
+		opts = append(opts, client.WithCustomVMs(customVMs))
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
+	info, err := cli.AddNode(
+		ctx,
+		nodeName,
+		avalancheGoBinPath,
+		opts...,
+	)
+	cancel()
+	if err != nil {
+		return err
+	}
+
+	color.Outf("{{green}}add node response:{{/}} %+v\n", info)
+	return nil
+}
 
 func newAddNodeCommand() *cobra.Command {
 	cmd := &cobra.Command{
@@ -400,6 +604,7 @@ func newRestartNodeCommand() *cobra.Command {
 		Use:   "restart-node [options]",
 		Short: "Restarts the server.",
 		RunE:  restartNodeFunc,
+		Args:  cobra.ExactArgs(0),
 	}
 	cmd.PersistentFlags().StringVar(
 		&nodeName,
@@ -423,18 +628,19 @@ func newRestartNodeCommand() *cobra.Command {
 }
 
 func restartNodeFunc(cmd *cobra.Command, args []string) error {
-	cli, err := client.New(client.Config{
-		LogLevel:    logLevel,
-		Endpoint:    endpoint,
-		DialTimeout: dialTimeout,
-	})
+	cli, err := newClient()
 	if err != nil {
 		return err
 	}
 	defer cli.Close()
 
 	ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
-	info, err := cli.RestartNode(ctx, nodeName, client.WithExecPath(avalancheGoBinPath), client.WithWhitelistedSubnets(whitelistedSubnets))
+	info, err := cli.RestartNode(
+		ctx,
+		nodeName,
+		client.WithExecPath(avalancheGoBinPath),
+		client.WithWhitelistedSubnets(whitelistedSubnets),
+	)
 	cancel()
 	if err != nil {
 		return err
@@ -449,6 +655,7 @@ func newAttachPeerCommand() *cobra.Command {
 		Use:   "attach-peer [options]",
 		Short: "Attaches a peer to the node.",
 		RunE:  attachPeerFunc,
+		Args:  cobra.ExactArgs(0),
 	}
 	cmd.PersistentFlags().StringVar(
 		&nodeName,
@@ -460,11 +667,7 @@ func newAttachPeerCommand() *cobra.Command {
 }
 
 func attachPeerFunc(cmd *cobra.Command, args []string) error {
-	cli, err := client.New(client.Config{
-		LogLevel:    logLevel,
-		Endpoint:    endpoint,
-		DialTimeout: dialTimeout,
-	})
+	cli, err := newClient()
 	if err != nil {
 		return err
 	}
@@ -492,6 +695,7 @@ func newSendOutboundMessageCommand() *cobra.Command {
 		Use:   "send-outbound-message [options]",
 		Short: "Sends an outbound message to an attached peer.",
 		RunE:  sendOutboundMessageFunc,
+		Args:  cobra.ExactArgs(0),
 	}
 	cmd.PersistentFlags().StringVar(
 		&nodeName,
@@ -521,11 +725,7 @@ func newSendOutboundMessageCommand() *cobra.Command {
 }
 
 func sendOutboundMessageFunc(cmd *cobra.Command, args []string) error {
-	cli, err := client.New(client.Config{
-		LogLevel:    logLevel,
-		Endpoint:    endpoint,
-		DialTimeout: dialTimeout,
-	})
+	cli, err := newClient()
 	if err != nil {
 		return err
 	}
@@ -552,16 +752,13 @@ func newStopCommand() *cobra.Command {
 		Use:   "stop [options]",
 		Short: "Requests server stop.",
 		RunE:  stopFunc,
+		Args:  cobra.ExactArgs(0),
 	}
 	return cmd
 }
 
 func stopFunc(cmd *cobra.Command, args []string) error {
-	cli, err := client.New(client.Config{
-		LogLevel:    logLevel,
-		Endpoint:    endpoint,
-		DialTimeout: dialTimeout,
-	})
+	cli, err := newClient()
 	if err != nil {
 		return err
 	}
@@ -577,3 +774,158 @@ func stopFunc(cmd *cobra.Command, args []string) error {
 	color.Outf("{{green}}stop response:{{/}} %+v\n", info)
 	return nil
 }
+
+func newSaveSnapshotCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "save-snapshot snapshot-name",
+		Short: "Requests server to save network snapshot.",
+		RunE:  saveSnapshotFunc,
+		Args:  cobra.ExactArgs(1),
+	}
+	return cmd
+}
+
+func saveSnapshotFunc(cmd *cobra.Command, args []string) error {
+	cli, err := newClient()
+	if err != nil {
+		return err
+	}
+	defer cli.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
+	resp, err := cli.SaveSnapshot(ctx, args[0])
+	cancel()
+	if err != nil {
+		return err
+	}
+
+	color.Outf("{{green}}save-snapshot response:{{/}} %+v\n", resp)
+	return nil
+}
+
+func newLoadSnapshotCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "load-snapshot snapshot-name",
+		Short: "Requests server to load network snapshot.",
+		RunE:  loadSnapshotFunc,
+		Args:  cobra.ExactArgs(1),
+	}
+	cmd.PersistentFlags().StringVar(
+		&avalancheGoBinPath,
+		"avalanchego-path",
+		"",
+		"avalanchego binary path",
+	)
+	cmd.PersistentFlags().StringVar(
+		&pluginDir,
+		"plugin-dir",
+		"",
+		"plugin directory",
+	)
+	cmd.PersistentFlags().StringVar(
+		&rootDataDir,
+		"root-data-dir",
+		"",
+		"root data directory to store logs and configurations",
+	)
+	return cmd
+}
+
+func loadSnapshotFunc(cmd *cobra.Command, args []string) error {
+	cli, err := newClient()
+	if err != nil {
+		return err
+	}
+	defer cli.Close()
+
+	opts := []client.OpOption{
+		client.WithExecPath(avalancheGoBinPath),
+		client.WithPluginDir(pluginDir),
+		client.WithRootDataDir(rootDataDir),
+	}
+
+	ctx := getAsyncContext()
+
+	resp, err := cli.LoadSnapshot(ctx, args[0], opts...)
+
+	if err != nil {
+		return err
+	}
+
+	color.Outf("{{green}}load-snapshot response:{{/}} %+v\n", resp)
+	return nil
+}
+
+func newRemoveSnapshotCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "remove-snapshot snapshot-name",
+		Short: "Requests server to remove network snapshot.",
+		RunE:  removeSnapshotFunc,
+		Args:  cobra.ExactArgs(1),
+	}
+	return cmd
+}
+
+func removeSnapshotFunc(cmd *cobra.Command, args []string) error {
+	cli, err := newClient()
+	if err != nil {
+		return err
+	}
+	defer cli.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
+	resp, err := cli.RemoveSnapshot(ctx, args[0])
+	cancel()
+	if err != nil {
+		return err
+	}
+
+	color.Outf("{{green}}remove-snapshot response:{{/}} %+v\n", resp)
+	return nil
+}
+
+func newGetSnapshotNamesCommand() *cobra.Command {
+	return &cobra.Command{
+		Use:   "get-snapshot-names [options]",
+		Short: "Requests server to get list of snapshot.",
+		RunE:  getSnapshotNamesFunc,
+		Args:  cobra.ExactArgs(0),
+	}
+}
+
+func getSnapshotNamesFunc(cmd *cobra.Command, args []string) error {
+	cli, err := newClient()
+	if err != nil {
+		return err
+	}
+	defer cli.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
+	snapshotNames, err := cli.GetSnapshotNames(ctx)
+	cancel()
+	if err != nil {
+		return err
+	}
+
+	color.Outf("{{green}}Snapshots:{{/}} %q\n", snapshotNames)
+	return nil
+}
+
+func newClient() (client.Client, error) {
+	return client.New(client.Config{
+		LogLevel:    logLevel,
+		Endpoint:    endpoint,
+		DialTimeout: dialTimeout,
+	})
+}
+
+func getAsyncContext() context.Context {
+	ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
+	// don't call since function using it is async
+	// and the top-level context here "ctx" is passed
+	// to all underlying function calls
+	// just set the timeout to halt "Start" async ops
+	// when the deadline is reached
+	_ = cancel
+	return ctx
+}

cmd/avalanche-network-runner/server/server.go

@@ -22,10 +22,13 @@ func init() {
 }
 
 var (
-	logLevel    string
-	port        string
-	gwPort      string
-	dialTimeout time.Duration
+	logLevel           string
+	port               string
+	gwPort             string
+	gwDisabled         bool
+	dialTimeout        time.Duration
+	disableNodesOutput bool
+	snapshotsDir       string
 )
 
 func NewCommand() *cobra.Command {
@@ -33,12 +36,16 @@ func NewCommand() *cobra.Command {
 		Use:   "server [options]",
 		Short: "Start a network runner server.",
 		RunE:  serverFunc,
+		Args:  cobra.ExactArgs(0),
 	}
 
 	cmd.PersistentFlags().StringVar(&logLevel, "log-level", logutil.DefaultLogLevel, "log level")
 	cmd.PersistentFlags().StringVar(&port, "port", ":8080", "server port")
 	cmd.PersistentFlags().StringVar(&gwPort, "grpc-gateway-port", ":8081", "grpc-gateway server port")
+	cmd.PersistentFlags().BoolVar(&gwDisabled, "disable-grpc-gateway", false, "true to disable grpc-gateway server (overrides --grpc-gateway-port)")
 	cmd.PersistentFlags().DurationVar(&dialTimeout, "dial-timeout", 10*time.Second, "server dial timeout")
+	cmd.PersistentFlags().BoolVar(&disableNodesOutput, "disable-nodes-output", false, "true to disable nodes stdout/stderr")
+	cmd.PersistentFlags().StringVar(&snapshotsDir, "snapshots-dir", "", "directory for snapshots")
 
 	return cmd
 }
@@ -53,9 +60,12 @@ func serverFunc(cmd *cobra.Command, args []string) (err error) {
 	_ = zap.ReplaceGlobals(logger)
 
 	s, err := server.New(server.Config{
-		Port:        port,
-		GwPort:      gwPort,
-		DialTimeout: dialTimeout,
+		Port:                port,
+		GwPort:              gwPort,
+		GwDisabled:          gwDisabled,
+		DialTimeout:         dialTimeout,
+		RedirectNodesOutput: !disableNodesOutput,
+		SnapshotsDir:        snapshotsDir,
 	})
 	if err != nil {
 		return err
@@ -73,10 +83,13 @@ func serverFunc(cmd *cobra.Command, args []string) (err error) {
 	case sig := <-sigc:
 		zap.L().Warn("signal received; closing server", zap.String("signal", sig.String()))
 		rootCancel()
-		zap.L().Warn("closed server", zap.Error(<-errc))
-	case err = <-errc:
-		zap.L().Warn("server closed", zap.Error(err))
-		rootCancel()
+		// wait for server stop
+		waitForServerStop := <-errc
+		zap.L().Warn("closed server", zap.Error(waitForServerStop))
+	case serverClosed := <-errc:
+		// server already stopped here
+		_ = rootCancel
+		zap.L().Warn("server closed", zap.Error(serverClosed))
 	}
 	return err
 }

examples/k8s/Dockerfile

@@ -1,36 +0,0 @@
-# Build the manager binary
-#FROM golang:1.16 as builder
-FROM golang:1.17-bullseye as builder
-
-RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates git linux-headers-amd64 musl-dev gcc bash
-
-# Copy the predefined netrc file into the location that git depends on
-COPY ./.netrc /root/.netrc
-RUN chmod 600 /root/.netrc
-
-WORKDIR /workspace
-# Copy the Go Modules manifests
-COPY go.mod go.mod
-COPY go.sum go.sum
-# cache deps before building and copying source so that we don't need to re-download as much
-# and so that source changes don't invalidate our downloaded layer
-RUN go mod download
-
-# Copy the go source
-copy network/ network/
-COPY api/ api/
-COPY k8s/ k8s/
-#COPY constants/ constants/
-COPY utils/ utils/
-COPY examples/ examples/
-
-# Build
-RUN GOOS=linux GOARCH=amd64 go build -o simplenet ./examples/k8s/main.go
-
-# Use distroless as minimal base image to package the manager binary
-# Refer to https://github.com/GoogleContainerTools/distroless for more details
-FROM gcr.io/distroless/base-debian11
-COPY --from=builder /workspace/simplenet / 
-COPY --from=builder /workspace/examples/ /examples/
-
-ENTRYPOINT ["/simplenet"]

examples/k8s/README.md

@@ -1,71 +0,0 @@
-# Kubernetes deployment of an avalanche network runner 
-
-## Scope 
-The avalanche network runner can operate in 
-* local
-* kubernetes
-
-environments.
-
-This README is *solely* about the kubernetes environment.
-
-## Overview
-Avalanchego is a *stateful* application: It requires a DB and runs with an identity (NodeID).
-Standard Kubernetes is best suited for *stateless* applications.
-
-To allow for stateful applications, the **Operator** pattern has been introduced into the kubernetes world [Operator pattern](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/).
-
-This means that in order to successfully run an avalanchego network with this network runner tool on kubernetes, a "domain-specific" implementation of the pattern is required.
-This implementation can be found at [avalanchego-operator](https://github.com/ava-labs/avalanchego-operator).
-
-There only needs to be running **1 operator instance** per kubernetes namespace.
-Therefore it is very important to distinguish between:
-
-* A local kubernetes cluster environment
-* A cloud-based cluster environment
-
-
-## Cloud-based environment
-
-### Github action
-If running in a cloud provided (for Ava Labs, typically AWS), then the **devops** team is responsible for setting up the required infrastructure.
-
-**Devops would be deploying the `avalanchego-operator`**.
-
-
-The normal operation is to create a github action which triggers the deployment to the kubernetes cluster in the cloud. 
-An example of such a github action can be found at `examples/github` **TODO**.
-The github action triggers the deployment of a self-contained pod or a permissioned `main` with access to the cluster.
-
-In this scenario, devops needs to be informed about a project running such a github action using up kubernetes resources.
-They will then provide the necessary environment information (namespaces and other potentially required configurations) to be configured on the github action.
-They will also install the appropriate kubernetes self-hosted runner into the repository so that this integration will work.
-
-Usually there would be nothing else to be done for the deployment other than implementing the network logic.
-
-
-### Without github action
-If a deployment without a github action should be required, then devops involvement is likewise required.
-They also would deploy the `avalanchego-operator` and provide necessary deployment information (namespace etc.).
-
-The network runner can then analogously be deployed either via self-contained pod or a `main` binary. Both need to have the required permissions.
-
-
-## Local kubernetes environment
-Kubernetes can also be run locally. This is usually only needed for development. In fact, for "normal" avalanchego testing and work, it shouldn't be necessary to run a local kubernetes cluster. However, for developing the integration in this repository, it is highly recommended as it speeds up development time.
-
-To run a local kubernetes environment:
-
-
-1. Install a kubernetes environment, e.g. `minikube` or `k3s`. We use `k3s` for this README: [k3s](https://k3s.io/). Follow instructions. This will also install `kubectl`.
-2. Make sure to run `sudo k3s server --docker --write-kubeconfig-mode 644`. This will allow to use **locally built images** (otherwise you'll need to use a public repo and push there). It may be required to `sudo systemctl stop k3s` service for `k3s` if the installation configured that mode, and disable it: `sudo systemctl disable k3s` (or edit the systemd script). Switch to other terminal.
-3. Install `kubectx` to allow to set an easy default environment: [kubectx](https://github.com/ahmetb/kubectx)
-4. Set the `$KUBECONFIG` environment variable to read the `k3s` environment: `export KUBECONFIG=/etc/rancher/k3s/k3s.yaml` (put this into your `.profile` or preferred environment persistance method). Make sure this variable is present in all terminals used for this tool.
-5. Run `kubectx` to check that the environment is fine (it should print `default` if there are no other kubernetes configs, otherwise, check the `kubectx` docs).
-6. **Run the avalanchego-operator locally**
-   6a. Clone the repository: `github.com/ava-labs/avalanchego-operator`
-   6b. Run `make install` followed by `make run`. This should be everything needed.
-7. Install the service account, role and roleconfigs for the network runner pod: `kubectl apply -f examples/k8s/svc-rbac.yaml`.
-8. Build this app as a pod: `docker build -f ./examples/k8s/Dockerfile -t k8s-netrunner:alpha .`. If you use a different app name than `k8s-netrunner` and/or tag `alpha`, the `examples/k8s/simple-network-pod.yaml` file needs to be updated accordingly.
-9. Deploy the pod to your local cluster via `kubectl apply -f examples/k8s/simple-netrunner-pod.yaml`. This picks the image built in 8. to deploy as a pod. Then from within this pod it starts all other nodes.
-10. Check that all is ok with `kubectl get pods`. You will first see the `k8s-netrunner` pod, and after some time, the other configured nodes should appear. To check logs, execute `kubectl logs k8s-netrunner`.

examples/k8s/conf.json

@@ -1,48 +0,0 @@
-{
-  "networkConfig": {
-    "name": "custom-test-net",
-    "logLevel": "DEBUG"
-  },
-  "k8sConfig": [
-    {
-      "namespace": "ci-avalanchego",
-      "identifier": "avalanchego-test-validator-0",
-      "kind": "Avalanchego",
-      "apiVersion": "chain.avax.network/v1alpha1",
-      "image": "avaplatform/avalanchego",
-      "tag": "v1.6.4"
-    },
-    {
-      "namespace": "ci-avalanchego",
-      "identifier": "avalanchego-test-validator-1",
-      "kind": "Avalanchego",
-      "apiVersion": "chain.avax.network/v1alpha1",
-      "image": "avaplatform/avalanchego",
-      "tag": "v1.6.4"
-    },
-    {
-      "namespace": "ci-avalanchego",
-      "identifier": "avalanchego-test-validator-2",
-      "kind": "Avalanchego",
-      "apiVersion": "chain.avax.network/v1alpha1",
-      "image": "avaplatform/avalanchego",
-      "tag": "v1.6.4"
-    },
-    {
-      "namespace": "ci-avalanchego",
-      "identifier": "avalanchego-test-validator-3",
-      "kind": "Avalanchego",
-      "apiVersion": "chain.avax.network/v1alpha1",
-      "image": "avaplatform/avalanchego",
-      "tag": "v1.6.4"
-    },
-    {
-      "namespace": "ci-avalanchego",
-      "identifier": "avalanchego-test-validator-4",
-      "kind": "Avalanchego",
-      "apiVersion": "chain.avax.network/v1alpha1",
-      "image": "avaplatform/avalanchego",
-      "tag": "v1.6.4"
-    }
-  ]
-}

examples/k8s/configs/genesis.json

@@ -1,76 +0,0 @@
-{
-  "networkID": 1337,
-  "allocations": [
-    {
-      "ethAddr": "0xb3d82b1367d362de99ab59a658165aff520cbd4d",
-      "avaxAddr": "X-custom1g65uqn6t77p656w64023nh8nd9updzmxwd59gh",
-      "initialAmount": 0,
-      "unlockSchedule": [
-        {
-          "amount": 10000000000000000,
-          "locktime": 1633824000
-        }
-      ]
-    },
-    {
-      "ethAddr": "0xb3d82b1367d362de99ab59a658165aff520cbd4d",
-      "avaxAddr": "X-custom18jma8ppw3nhx5r4ap8clazz0dps7rv5u9xde7p",
-      "initialAmount": 300000000000000000,
-      "unlockSchedule": [
-        {
-          "amount": 20000000000000000
-        },
-        {
-          "amount": 10000000000000000,
-          "locktime": 1633824000
-        }
-      ]
-    },
-    {
-      "ethAddr": "0xb3d82b1367d362de99ab59a658165aff520cbd4d",
-      "avaxAddr": "X-custom1ur873jhz9qnaqv5qthk5sn3e8nj3e0kmzpjrhp",
-      "initialAmount": 10000000000000000,
-      "unlockSchedule": [
-        {
-          "amount": 10000000000000000,
-          "locktime": 1633824000
-        }
-      ]
-    }
-  ],
-  "startTime": 1630987200,
-  "initialStakeDuration": 31536000,
-  "initialStakeDurationOffset": 5400,
-  "initialStakedFunds": [
-    "X-custom1g65uqn6t77p656w64023nh8nd9updzmxwd59gh"
-  ],
-  "initialStakers": [
-    {
-      "nodeID": "NodeID-7Xhw2mDxuDS44j42TCB6U5579esbSt3Lg",
-      "rewardAddress": "X-custom18jma8ppw3nhx5r4ap8clazz0dps7rv5u9xde7p",
-      "delegationFee": 1000000
-    },
-    {
-      "nodeID": "NodeID-MFrZFVCXPv5iCn6M9K6XduxGTYp891xXZ",
-      "rewardAddress": "X-custom18jma8ppw3nhx5r4ap8clazz0dps7rv5u9xde7p",
-      "delegationFee": 500000
-    },
-    {
-      "nodeID": "NodeID-NFBbbJ4qCmNaCzeW7sxErhvWqvEQMnYcN",
-      "rewardAddress": "X-custom18jma8ppw3nhx5r4ap8clazz0dps7rv5u9xde7p",
-      "delegationFee": 250000
-    },
-    {
-      "nodeID": "NodeID-GWPcbFJZFfZreETSoWjPimr846mXEKCtu",
-      "rewardAddress": "X-custom18jma8ppw3nhx5r4ap8clazz0dps7rv5u9xde7p",
-      "delegationFee": 125000
-    },
-    {
-      "nodeID": "NodeID-P7oB2McjBGgW2NXXWVYjV8JEDFoW9xDE5",
-      "rewardAddress": "X-custom18jma8ppw3nhx5r4ap8clazz0dps7rv5u9xde7p",
-      "delegationFee": 62500
-    }
-  ],
-  "cChainGenesis": "{\"config\":{\"chainId\":43112,\"homesteadBlock\":0,\"daoForkBlock\":0,\"daoForkSupport\":true,\"eip150Block\":0,\"eip150Hash\":\"0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0\",\"eip155Block\":0,\"eip158Block\":0,\"byzantiumBlock\":0,\"constantinopleBlock\":0,\"petersburgBlock\":0,\"istanbulBlock\":0,\"muirGlacierBlock\":0,\"apricotPhase1BlockTimestamp\":0,\"apricotPhase2BlockTimestamp\":0},\"nonce\":\"0x0\",\"timestamp\":\"0x0\",\"extraData\":\"0x00\",\"gasLimit\":\"0x5f5e100\",\"difficulty\":\"0x0\",\"mixHash\":\"0x0000000000000000000000000000000000000000000000000000000000000000\",\"coinbase\":\"0x0000000000000000000000000000000000000000\",\"alloc\":{\"8db97C7cEcE249c2b98bDC0226Cc4C2A57BF52FC\":{\"balance\":\"0x295BE96E64066972000000\"}},\"number\":\"0x0\",\"gasUsed\":\"0x0\",\"parentHash\":\"0x0000000000000000000000000000000000000000000000000000000000000000\"}",
-  "message": "{{ fun_quote }}"
-}

examples/k8s/configs/node0/config.json

@@ -1,6 +0,0 @@
-{
-  "network-peer-list-gossip-frequency": "250ms",
-  "network-max-reconnect-delay": "1s",
-  "health-check-frequency": "2s",
-  "log-level":"debug"
-}

examples/k8s/configs/node0/staking.crt

@@ -1,30 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIFNzCCAx8CCQC687XFxtDRSjANBgkqhkiG9w0BAQsFADB/MQswCQYDVQQGEwJV
-UzELMAkGA1UECAwCTlkxDzANBgNVBAcMBkl0aGFjYTEQMA4GA1UECgwHQXZhbGFi
-czEOMAwGA1UECwwFR2Vja28xDDAKBgNVBAMMA2F2YTEiMCAGCSqGSIb3DQEJARYT
-c3RlcGhlbkBhdmFsYWJzLm9yZzAgFw0xOTA3MDIxNjEyMTVaGA8zMDE5MDcxMDE2
-MTIxNVowOjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAk5ZMRAwDgYDVQQKDAdBdmFs
-YWJzMQwwCgYDVQQDDANhdmEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoIC
-AQDKYSRw/W0YpYH/MTQhiFrR0m89l6yTuzLpDtjudr/5RnhIPvtqk7YIGm/m9l29
-xwR4J5r7SZGs+70yBetkbS+h7PwJ2rmWDwbrdyJKvVBhqf8kSn+VU2LePSIcJj19
-3LDyWhV1H4lqNkUkcAR76Fh9qjMvA2p0vJ66+eDLXlph/RYapQx9HgOj/0BmAKMr
-YCyo5BhRih+Ougg8aK4G9PQTIA5G2wTWW2QkHxM/QppFjZd/XwQeJ2H6ubWMFc5f
-ttf6AzpJvFIDBu/JDCKWiCu5m8t4GL8w2OrIx8Js19lF4YYE2eojCreqgPi64S3o
-cqwKsDoySTw6/5iKQ5BUYwUXX3z7EXOqD8SMHefUKeczj4WvAaZLzR27qXm55EgR
-YQAIX4fhmY7NfSop3Wh0Eo62+JHoM/1g+UgOXlbnWpY95Mgd7/fwDSWLu4IxE0/u
-q8VufIbfC4yrY8qlTVfAffI1ldRdvJjPJBPiQ0CNrOl60LVptpkGc9shH7wZ2bP0
-bEnYKTgLAfOzD8Ut71O2AOIa80A1GNFl4Yle/MSNJOcQOSpgtWdREzIUoenAjfuz
-M4OeTr4cRg4+VYTAo9KHKriN1DuewNzGd8WjKAVHmcIMjqISLTlzMhdsdm+OmfQ6
-OvyX7v0GTOBbhP09NGcww5A0gCzXN18FS5oxnxe6OG9D0wIDAQABMA0GCSqGSIb3
-DQEBCwUAA4ICAQAqL1TWI1PTMm3JaXkhdTBe8tsk7+FsHAFzTcBVBsB8dkJNGhxb
-dlu7XIm+AyGUn0j8siz8qojKbO+rEPV/ImTH5W7Q36rXSdgvNUWpKrKIC5S8PUF5
-T4pH+lpYIlQHnTaKMuqH3nO3I40IhEhPaa2wAwy2kDlz46fJcr6aMzj6Zg43J5UK
-Zid+BQsiWAUau5V7CpC7GMCx4YdOZWWsT3dAsug9hvwTe81kK1JoTH0juwPTBH0t
-xUgUVIWyuweM1UwYF3n8Hmwq6B46YmujhMDKT+3lgqZt7eZ1XvieLdBRlVQWzOa/
-6QYTkrqwPZioKIStrxVGYjk40qECNodCSCIwRDgbnQubRWrdslxiIyc5blJNuOV+
-jgv5d2EeUpwUjvpZuEV7FqPKGRgiG0jfl6Psms9gYUXd+y3ytG9HeoDNmLTSTBE4
-nCQXX935P2/xOuok6CpiGpP89DX7t8yiwk8LFNnY3rvv50nVy8kerVdnfHTmoMZ9
-/IBgojSIKov4lmPKdgzFfimzhbssVCa4DO/LIhTF7bQbH1ut/Oq7npdOpMjLYIBE
-9lagvRVTVFwT/uwrCcXHCb21b/puwV94SNXVwt7BheFTFBdtxJrR4jjr2T5odLkX
-6nQcY8V2OT7KOxn0KVc6pl3saJTLmL+H/3CtAao9NtmuUDapKINRSVNyvg==
------END CERTIFICATE-----

examples/k8s/configs/node0/staking.key

@@ -1,51 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIIJKAIBAAKCAgEAymEkcP1tGKWB/zE0IYha0dJvPZesk7sy6Q7Y7na/+UZ4SD77
-apO2CBpv5vZdvccEeCea+0mRrPu9MgXrZG0voez8Cdq5lg8G63ciSr1QYan/JEp/
-lVNi3j0iHCY9fdyw8loVdR+JajZFJHAEe+hYfaozLwNqdLyeuvngy15aYf0WGqUM
-fR4Do/9AZgCjK2AsqOQYUYofjroIPGiuBvT0EyAORtsE1ltkJB8TP0KaRY2Xf18E
-Hidh+rm1jBXOX7bX+gM6SbxSAwbvyQwilogruZvLeBi/MNjqyMfCbNfZReGGBNnq
-Iwq3qoD4uuEt6HKsCrA6Mkk8Ov+YikOQVGMFF198+xFzqg/EjB3n1CnnM4+FrwGm
-S80du6l5ueRIEWEACF+H4ZmOzX0qKd1odBKOtviR6DP9YPlIDl5W51qWPeTIHe/3
-8A0li7uCMRNP7qvFbnyG3wuMq2PKpU1XwH3yNZXUXbyYzyQT4kNAjazpetC1abaZ
-BnPbIR+8Gdmz9GxJ2Ck4CwHzsw/FLe9TtgDiGvNANRjRZeGJXvzEjSTnEDkqYLVn
-URMyFKHpwI37szODnk6+HEYOPlWEwKPShyq4jdQ7nsDcxnfFoygFR5nCDI6iEi05
-czIXbHZvjpn0Ojr8l+79BkzgW4T9PTRnMMOQNIAs1zdfBUuaMZ8XujhvQ9MCAwEA
-AQKCAgEAuUM4Mt8r8bYBTPVj/ZZvXUjAYKfqacqijkrzN0kp8C4cijZtvWC+8KgS
-7GF36vS3GK9Y5tSwMKS6y4IzvFlfk2H4T6UU41OaSA9lKvonDWCrmjNAnBgbl8pq
-4U34WLGgohrpLbDTAJHxtat9z1ghOdiGxnDgEUFiJVP9/u2+25jtlTKmPhstxgEy
-mK3YsSp3d5xmzq4cuXF/fJ1vQhsXHDLqHt78jKZZA+AWpIB57VXy67y1bk0rGnTK
-xxRnOaOODubJgxqMEQ1WkLs1Jow9Sspd9vDghPzt4SNMzorB8YDESMib17xF6iXq
-jFj6x6HB8H7mp4X3RyMYJuo2w6lpzBsEncUYpKhqMabF0I/giI5VdpSDvkCCOFen
-nWZLV9Ai/x7tTq/0F+cVM69Mgfe8iYymqlfd6WRZITKfViNHALlG/Pq9yHJsz7Ng
-S8BKODt/sj4Q0xLtFDT/DmpP50iq7SiS14obcKcQr8FAjM/sOY/Ulg4M8MA7EugS
-pDJwLl6XDoIMMCNwZ1HGsDstzmx5Mf50bS4tbK4iZzcpPX5RBTlVdo9MTSgnFizp
-Ii1NjHLuVVCSLb1OjoTgu0cQFiWEBCkC1XuoR8RCY6iWVrUH4Gezni7ckt2mJaNA
-pd6/87dFKE3jh5T6jZeJMJg5skTZHSozJDuaj9pMK/JONSD06sECggEBAPq2lEmd
-g1hpMIqa7ey1uoLd1zFFzlWrxTJLlu38N69mYDOHrV/zqRGOpZB+1nH7tQJIT/L1
-xLN33mFVqCrN8yUmZ+iUWioaI5JZ1jzCgemVGeBgodwP9MOZfxxrDp17oTdabaEq
-7ZaBYnY8xK/4bCxu/B4mFiF3Za8ZTd/+2yev7JM+E3MorWc7rrKm1ApflfxytdhO
-JLBiqOcqobI3dgHyzesVb8cT4XCpoRhdrFwort0JI7ryfddd49vMJ3ElRbnN/h4F
-f24cWY/sQPq/nfDmec28Z7nVza1D4rszNylYDvzdjF0Q1mL5dFVntWbZA1CNurVw
-nTfwuyQ8RF9YnYMCggEBAM6lpNeqaiG9ixKSr65pYOKtByUI3/eTT4vBnrDtYF+8
-ohiKgIymG/vJsSdrynKfwJObEy2dBYhCGF3h9z2nc9KJQD/su7wxCsdmBs7YoDiM
-uzNPlRAmI0QAFILPCk48z/lUQk3r/Mzu0YzRv7fI4WSpIGAefVPDqy1uXsATDoDJ
-arcEkND5Lib89Lx7r02EevJJTdhTJM8mBdRl6wpNV3xBdwis67uSyunFZYpSiMw7
-WWjIRhzhLIvpgD78UvNvuJi0UGVEjTqnxvuW3Y6sLfIk80KSR24USinT27t//x7z
-yzNko75avF2hm1f8Y/EpcHHAax8NAQF5uuV9xBNvv3ECggEAdS/sRjCK2UNpvg/G
-0FLtWAgrcsuHM4IzjVvJs3ml6aV3p/5uKqBw0VUUzGKNCAA4TlXQkOcRxzVrS6HH
-FiLn2OCHxy24q19Gazz0p7ffE3hu/PMOFRecN+VChd0AmtnTtFTfU2sGXMgjZtLm
-uL3siiRiUhFJXOE7NUolnWK5u2Y+tWBZpQVJcCx0busNx7+AEtznZLC583xaKJtD
-s1K7JRQB7jU55xrC0G9pbkMysm0NtyFzgwmfipBHVlCpyvg6DCxd8FhvhN9Zea1b
-fhkc0SJZorHC5hkqpydJDmlVCk0vzEAeQM4C94ZUOytbnjQnmXp14CNASYqLXteQ
-ueRo0wKCAQAG0F10IxFm1WotjZqvZJgmQVBX/0frUPcxg4vpB5rC7WRm7MI6YQvR
-LKBjzWEakHv4Igfq3B+fk5ZcGiRd6xSdn5r3wKWcGf3h/1JAJdJ6quFNWtVud+N3
-zYzfl1YeqFCvRwD8ssheNY3BV/U7aStNd2oy4S5+wZf2YopLSRWUV4/mQwdHbMAB
-1xt2z5lDNBgdvx8LAArZrcZJb6blaxF0bnAvYAxR3hBEzxZ/DiOmoFpdYyU0tJQU
-dPmemhFeJ5PtrRxtimohwgCEsT/TAYhuUJuY2VvznEWpxWucbicKbT2JD0t67mEB
-sV9+8jqVbCliBtdBadtbohjwkkoR3gBxAoIBAG3cZuNkIWpELEbeICKouSOKN06r
-Fs/UXU8roNThPR7vPtjeD1NDMmUHJr1FG4SJrSigdD8qNBg8w/G3nI0Iw7eFskk5
-8mNm21CpDzON36ZO7IDMj5uyBlj2t+Ixl/uJYhYSpuNXyUTMm+rkFJ0vdSV4fjLd
-J2m30juYnMiBBJf7dz5M95+T0xicGWyV24zVYYBbSo0NHEGxqeRhikNqZNPkod6f
-kfOJZGalh2KaK5RMpZpFFhZ/kW9xRWNJZyCWgkIoYkdilMuISBu3lCrk8rdMpAL0
-wHEcq8xwcgYCS2qk8HwjtmVd3gpB1y9UshMr3qnuH1wMpU5C+nM2oy3vSko=
------END RSA PRIVATE KEY-----

examples/k8s/configs/node1/config.json

@@ -1,6 +0,0 @@
-{
-  "network-peer-list-gossip-frequency": "250ms",
-  "network-max-reconnect-delay": "1s",
-  "health-check-frequency": "2s",
-  "log-level":"debug"
-}

examples/k8s/configs/node1/staking.crt

@@ -1,30 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIFNzCCAx8CCQC687XFxtDRSjANBgkqhkiG9w0BAQsFADB/MQswCQYDVQQGEwJV
-UzELMAkGA1UECAwCTlkxDzANBgNVBAcMBkl0aGFjYTEQMA4GA1UECgwHQXZhbGFi
-czEOMAwGA1UECwwFR2Vja28xDDAKBgNVBAMMA2F2YTEiMCAGCSqGSIb3DQEJARYT
-c3RlcGhlbkBhdmFsYWJzLm9yZzAgFw0xOTA3MDIxNjEyMTlaGA8zMDE5MDcxMDE2
-MTIxOVowOjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAk5ZMRAwDgYDVQQKDAdBdmFs
-YWJzMQwwCgYDVQQDDANhdmEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoIC
-AQDdToR60na6NuR9iSAUMyzPXJNMWVQbLyT5/iZCiJ3BB4YWMBhfxpJWJiWXcM+z
-nDgpJuyCEeh5Dp6ZY3Fe7k6Hht6FmFpDjwnjpQmdkEKUg00G+ElPTp/UsmsPL+JA
-swPqBZWpMBS3dsXQNunMMtMGlrf5S0l6XX4y7kc/GTxYgveWZ9JtR/m2KNer+wjg
-BHqJ4rPqnHB30sDYPZg91Cz1Ak8Bb2w2I108zQVgKK6eIqNKXJJ/4pizSZdU4920
-wMxYBpnfDAchnxei9U/v3QbT7eKUI2fGr+hOWTIWU80+VeOBt8a6P4sS9AQh5/6G
-8qwmAqO3YQ9dxN82iu/H3+N+GGa/M0r5rEWrzwIuFhwKvyQcpPRBm2yQnBnhL9G5
-kN6n4OBM0KsgZ3CYlHZSg4eWcNgBt1WCFsQc7vfUFaJnr8QP3pF4V/4Bok7wTO5H
-N0A1EYEVYuX53NGnrKVe+Fg9+xMOgXPWkUNqdvpI9ZbV3Z0S5866qF3/vBZrhgCr
-Kc5E/vMexBRe8Ki4wKqONVhi9WGUcRHvFEikc+7VrPj0YaG6zVLd+uOAJN81fKOP
-Yo4X4sZrMyPYl3OjGtMhfV4KvCaLEr1duOklqO6cCvGQ8iAlLVy3VJyW5GJ0D0Ky
-iAir4VNdAJKo1ZgiGivJLWulTfjUifCN9o115AiqJxiqwwIDAQABMA0GCSqGSIb3
-DQEBCwUAA4ICAQCQOdwD7eRIxBvbQHUc+m0TRzEa17BCfck1Y2WwN3TZXDGSkPVE
-0uujA8SL3qi8/CTLGRqI9U3gRZJf+tJPBF/P021PEmyaFTS4htxcDxTxuZv2jCo9
-+XhUEyvRWitTmoy1esq3mkotVQHeTmQvwCsQJAhctVA/hRdJwmMPs1B8QxOUI6Bq
-SOBHa9CsXIzVOFv8FqE91PZA2ns30sKQYrrnbH99apfF5WglLUoyPwxf2e3AACh7
-beEdk45ivvKwi5Jk8nr85KDHYPlqkr0bd9Ehl8xplaNBdMPeRufqBDlztjcLJ3wo
-mnrt95gQMeSoLHY3UNsIRjbj43zImu7q9v/DD9ppQpu26aRDRmBNgLZA9GM5XnbZ
-RFi3VxLyqasGcSzaHwz5c7vOBOkOdlqcQzISRvWDxiN1HkAL+hkiQCuMchgORAgM
-wzPooa8rfWtLIpOXMpwuVGb/8rGNLEPovoCK9z6c+WZ+zkRo4+3TQkOMY66Xht7r
-Ahly3ler+Tyg6a5jXT92WKC/MXBYAy2ZQNoy204kNKevcH7R2cSkxITd3n5EacNy
-5MAtCNIk7JweLCh9rLrLUBt+i4n44sP+LVhfWHemngA8CoF4n6eQ0pp0ixZTen0j
-4uN0G2Nf+JeGMlqoObLWdIOdH/pbDppXGoZaKKDd7+bA74Fle5Uh7+1e3A==
------END CERTIFICATE-----

examples/k8s/configs/node1/staking.key

@@ -1,51 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIIJKgIBAAKCAgEA3U6EetJ2ujbkfYkgFDMsz1yTTFlUGy8k+f4mQoidwQeGFjAY
-X8aSViYll3DPs5w4KSbsghHoeQ6emWNxXu5Oh4behZhaQ48J46UJnZBClINNBvhJ
-T06f1LJrDy/iQLMD6gWVqTAUt3bF0DbpzDLTBpa3+UtJel1+Mu5HPxk8WIL3lmfS
-bUf5tijXq/sI4AR6ieKz6pxwd9LA2D2YPdQs9QJPAW9sNiNdPM0FYCiuniKjSlyS
-f+KYs0mXVOPdtMDMWAaZ3wwHIZ8XovVP790G0+3ilCNnxq/oTlkyFlPNPlXjgbfG
-uj+LEvQEIef+hvKsJgKjt2EPXcTfNorvx9/jfhhmvzNK+axFq88CLhYcCr8kHKT0
-QZtskJwZ4S/RuZDep+DgTNCrIGdwmJR2UoOHlnDYAbdVghbEHO731BWiZ6/ED96R
-eFf+AaJO8EzuRzdANRGBFWLl+dzRp6ylXvhYPfsTDoFz1pFDanb6SPWW1d2dEufO
-uqhd/7wWa4YAqynORP7zHsQUXvCouMCqjjVYYvVhlHER7xRIpHPu1az49GGhus1S
-3frjgCTfNXyjj2KOF+LGazMj2JdzoxrTIX1eCrwmixK9XbjpJajunArxkPIgJS1c
-t1ScluRidA9CsogIq+FTXQCSqNWYIhoryS1rpU341InwjfaNdeQIqicYqsMCAwEA
-AQKCAgANGUOgHWrnlK4re/1JFMpXL6yMPVFMFptCrLdJAtsLfM2D7K7UpGUu8i0R
-bJzujZWJYgNno3W2DJZ4j7k7HDHLtcDf+WeGTiYQskkCaXJ3ZdoeSn3UUtwE89aA
-XJ4wpCfcJx53mB/xx/bnXwixjGSPJEaZW8pqkrQQgaf35R98Qawz28tJqpPuIza4
-uDALSliSZretcDr77J57bhHfvvo2Oj/A3v5xqeAv5BaoXWAQfg5aLWaCaUAOhJGP
-dbk+pJazsxhSalzVsZvtikWD9focex0JFZtj2C+Qy5i6V5VzVhQULnN1vKMXqRfB
-hgC7rgtgaJGWHgmRzEBF8y1EEE1fohbo2sqkG4oMz3jBZ4o4MADQcpfK2qchgrnk
-OxIS/uU8szdu84iH8s6F/Hl1+87jnq6O9Re0iMSuvyUbjAEe8Cm9P/a5M1X9eyzw
-WSXSPZBwKSRoP3wuycbEonTWQnQHdwySY+IvdtgliEDhKrVbZGnks5zmaaIydW/y
-LS2S9JRM5Y+Xp0vV3nGlEehCUdrXoQ1Dz/AiHnWHjbxoCFGt0qL6COJziAGfUXKa
-cQ5iDd7zc2J3m2Z6c8W8xkPJe+1dmNWfGHrja8DSHtTcDY6Aqd98Vu0niu8PC7bx
-Avw++6J2wG7LN89rgR0uP7as9Cx4kHHsOFwp+lKODVe2dw0vAQKCAQEA7moNCjP6
-5PkSkSNPi/jw1Y/FCwBoJEzl3Q5ftwGrkYZFRLBTvCCli2jhexaC0D9+yjsVaL/2
-Vap43/xi55ipniqv8c1WAc5xFh+6hCydS6K9owDxlHN75MGLrmrYjY+3aMdo15Dm
-x5bznOLLyMUW4Ak+77MTw12fad/7L0ANXumFFj6ydcS8PHmhJlmz5VegWz5b1KGQ
-K//phcuOm349xekt7J5kKRbDEqLOlZv/EIAdCBQM4U3d6P/2vUUy5nKYG0F1xeaC
-leVpr1EPoEI+XkTy+jjoaBs7iUHpcD359XQCWLniwf1Yfttk9zJp7m6tR/Geablk
-unnH5zyFkwzlQwKCAQEA7aFtNsjL0UEXlyBYjCROoPu6ka/o4QyEaWvMHciXu+Gv
-M7TQCF2i9oeQXABIyTNoQr+pNjARboY8p0+9ZfV8QGlvH6awW2MNzD07lg9hwsjY
-JOCI64XxZj183GhHgN9/cE4PXBrQCqPLPCKdV66yAR9WNm9Va3Y9Xf/RvcoLiNB1
-FAg5bhbNQMnR38nPJs9+suSqYB8xADKvwmKEdony+WIM/GQyYZiDlXEj8EfWQouM
-wAok6Vuhs6cuLiHHzXFR4Y6RCWRb2nf2VrzWopz2Bp02IeHY0UZsZeKnqha9dtUu
-ZCIt2MZUELxih9JS+wzCX8BJk3xedi89zOZKRx4MgQKCAQEAxqnUJ9ZckIQDtrEn
-zckoVayxUpOKNAVn3SXnGAXqQx8RhUUw4SiLCXnhucFuS709F6LYGisrRwMAKhST
-Dc0mOcf0SJcDvgmaLgdOUmkiwS3gu31D0KHScTHeBP6/aGaDPGo9sLLruxDL+sT5
-bljc0N6jdPVR2I+hEIY1NpA3FAmefoTMDFpdSD9Jyz0gLFEyLBXwS2Q9UIy0uGqA
-cI1nSA0f2XW6nIp9DoBfiEcu6T738g1TFkLeURNJNTn+SgzfNob7bmbAFcvOnun7
-DV1lvwPRPDRDZMycdalYrdDXAnMiqXBrxZ4oKb0DiwCVSLss5TAvAoYbq09jBgpm
-e7xZJQKCAQEA3f7l0b1qs5WU3UmJj3rHvhsNY9crvzr7ZKUhLl3cathe3fY4NuiL
-OrbQxTI6zURqTZlSEl57mn5roX6cGOlqZ55YAwCtVuLF3B0EUp8SHG+XhXQCVc1v
-BK3CvQHqctnY62jxboFaA+abEhXgWi7I+sV0vCvsaBUxJWS9ZAmiFvFvvwQj6tYA
-cFta5y9YiBBmc+etx1i8ZUv06Ksyxq7/P707Fnrgmk5p9y2YfnwODWLjXfDcJOnG
-udggC1bhmusXrJmMo3KPYRybFNMbzRTHvswV6zdbX77ju5cwPXU7EQ39ZeyMWiyG
-EpB7mBmEDicQW3V/Bvq0IMLngElP8PqAgQKCAQEAq4BE1PFN6hQOqe0mcO8g9mqu
-zxl2MM0Kb2ABE8fxQ2w4Fy7g42NozDUW13/MN7q1I+AwMhbl4Ib2QImEMTuFaHPY
-A3OZlnE9L0oi4FI+kG2eJOB/+5pHSuf/jrZ/4gARK+uc/CDeaIljP/nxw0cX+sF+
-HjX4Ob4/CyEIeIUGdOGs7g9kf+oirXryuDcZxl/2fQOxqva9dhhBLhPXG3otSp0T
-D90xC1lSPLIHf+VUiF9bLMtUp4meGcgwpXPVjRV5cblLrP9PxbevlhG2D3vnOK9A
-8jWI9P1uNBEAUTSmXV8reMYOyNXJH8YbbT4yiarWnaQM0J0ipWwXGEeWagv/aA==
------END RSA PRIVATE KEY-----

examples/k8s/configs/node2/config.json

@@ -1,6 +0,0 @@
-{
-  "network-peer-list-gossip-frequency": "250ms",
-  "network-max-reconnect-delay": "1s",
-  "health-check-frequency": "2s",
-  "log-level":"debug"
-}

examples/k8s/configs/node2/staking.crt

@@ -1,30 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIFNzCCAx8CCQC687XFxtDRSjANBgkqhkiG9w0BAQsFADB/MQswCQYDVQQGEwJV
-UzELMAkGA1UECAwCTlkxDzANBgNVBAcMBkl0aGFjYTEQMA4GA1UECgwHQXZhbGFi
-czEOMAwGA1UECwwFR2Vja28xDDAKBgNVBAMMA2F2YTEiMCAGCSqGSIb3DQEJARYT
-c3RlcGhlbkBhdmFsYWJzLm9yZzAgFw0xOTA3MDIxNjEyMjJaGA8zMDE5MDcxMDE2
-MTIyMlowOjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAk5ZMRAwDgYDVQQKDAdBdmFs
-YWJzMQwwCgYDVQQDDANhdmEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoIC
-AQC8mVDToHbkUF2gRdVfpydZLNKeQ38d6HZFkUM3U1dWLZFSZNvagN8hlQvY/tQu
-3A40p19WgKbzWZre3tg1Akw8Jztdz9gl4RMn142IIO3CiwIptkE0JopbZhmG5fAC
-2n/MXQtfieI3hzeR04LW4JgLKzf3Nn8xZdlBgJfBmL5qUUnE7O7IbJGGma6gSD3e
-wetE6KQZtNtf0xRIv08doZKYwTl6ItkdGK76ufqq098GVwWvA1wSune4+MFgs9N4
-eFJj6Jyt85fiK/cwPx7KRdgYgBzrZQ4EPshRnwWrBTieOOaJvAA2RMxMEYzKRrJA
-AsYI1zxtNyqIUaBTcxmaz+NXUGW+wHwITic0Gp/XQm2Lwr/lxIV6OnAlL3CgbSXi
-rSnoG+eHQ+vDzBAcRDkTAgv/GUIzlfqT2StTK02uIBgJYzvFTG4plHitccRfy8wx
-sh5Z8xG99lmPQQtLsnlQAV+Li06Cb8CH4hUVoiWiVs5QAahqWmv5fpoX0Es26RyU
-HXGbjE202pyMMA7jUerUVKMijOoGZtcH6zB4p/dJ0TtToRwOgrA7NCI9AYVtqVXr
-XG/udj8ur2r1bTVwIbHsOeTEP3gY0mHRWm2E/bLjt9vbYIRUxR8xWnLkbeBziNTw
-g+36jdDF+6gu3cUz/nbSn8YY+Y1jjXuM3lqF8iMaAobhuwIDAQABMA0GCSqGSIb3
-DQEBCwUAA4ICAQAe2kC0HjKZU+dlnU2RlfBpB4QgzzrFE5N9A8F1MlE4vV3AzCg1
-RVdHPvniXzdNhDiiflK0l/cnrFv2X1TzYMrrA677/usHf2Bw0xjm/ipHOt5V+4TN
-mZAIA4IPl09gP28IZLc9xSuq4FoHeM8OTxhttOlINhqpG9P5d6bPezW6ZzI3CdPP
-CF69xK4GFlj/NQnAoFogid4ojYYNTj/cM4PYQU2KbrlzLyPuUk/CgwefXLMH87/H
-e3kPDev80Tjv2Pm5nD937fZfgrEoyolKxiRVcfZVMxR7qhPhizjueD0DAkfQIs7L
-YVSyx/qjEv2bBYaim5RQakUeHR1Xu5Xj/k5zr33t979ede50byQrcWm4H5JxnEpD
-JxJnFfDOU6o14SKGHSrao5Z4C3dI55DM84WLASnlMI5BK4XtS3notLNzG8dfWWhT
-9m0Hcry+wPNDcGr8Mtj1los/0bMDqMHC4jcFW1hrXCUUs9RYzE+N/xoqwCQSgN1P
-E73uXTySWj5ovMR5TPF6PhcftLB/OziqO7FverEBpvGGHUAnUT61JtjodjXPbEdj
-0VgyMOBY2y53HTXnx3dxeFZkUdRX/VZYy8tMK3MTY+7UIU5cWYnCZAo5LNcc0ukR
-S6WS9+6eaQ6XRjhfNUjx9a7FzqapWdtTedpipmBP1Njap3g29iUuVnLQeg==
------END CERTIFICATE-----

examples/k8s/configs/node2/staking.key

@@ -1,51 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIIJJwIBAAKCAgEAvJlQ06B25FBdoEXVX6cnWSzSnkN/Heh2RZFDN1NXVi2RUmTb
-2oDfIZUL2P7ULtwONKdfVoCm81ma3t7YNQJMPCc7Xc/YJeETJ9eNiCDtwosCKbZB
-NCaKW2YZhuXwAtp/zF0LX4niN4c3kdOC1uCYCys39zZ/MWXZQYCXwZi+alFJxOzu
-yGyRhpmuoEg93sHrROikGbTbX9MUSL9PHaGSmME5eiLZHRiu+rn6qtPfBlcFrwNc
-Erp3uPjBYLPTeHhSY+icrfOX4iv3MD8eykXYGIAc62UOBD7IUZ8FqwU4njjmibwA
-NkTMTBGMykayQALGCNc8bTcqiFGgU3MZms/jV1BlvsB8CE4nNBqf10Jti8K/5cSF
-ejpwJS9woG0l4q0p6Bvnh0Prw8wQHEQ5EwIL/xlCM5X6k9krUytNriAYCWM7xUxu
-KZR4rXHEX8vMMbIeWfMRvfZZj0ELS7J5UAFfi4tOgm/Ah+IVFaIlolbOUAGoalpr
-+X6aF9BLNukclB1xm4xNtNqcjDAO41Hq1FSjIozqBmbXB+sweKf3SdE7U6EcDoKw
-OzQiPQGFbalV61xv7nY/Lq9q9W01cCGx7DnkxD94GNJh0VpthP2y47fb22CEVMUf
-MVpy5G3gc4jU8IPt+o3QxfuoLt3FM/520p/GGPmNY417jN5ahfIjGgKG4bsCAwEA
-AQKCAgA+uHIT3yKK7VslqPO7+tfwJSLqNSI6LQvgON30sUezRjY1A4vGD+OkxG+L
-O7wO1Wn4As2G9AQRm/QQOGYIwvnda2Kn4S5N8psvPdU4t1K6xwXyH0Vx9Xs/yCWn
-IiL+n/GuYicdH7rWoqZNXdz+XvTRig7zrPEB2ZA143EUlhqFOwFgdzc1+j0vWT6k
-2UGSKkV2xjOExQvLw2PUiaLjBM++80uNHbe8oG/YvC7rzsg10Iz4VhKxu8eDAV82
-LLegMcucpEgu5XrWYa60Idm4hR/HjhuQASx3JvXxhwQYiwT4QY4Rsi8T3S9gANok
-jvxKo2F+oS3cWGNRsGu0NOwH+yjsVyMYazcLOUesAAe85ttXgYr02+Z/uMnxqtOF
-gjIHY3X5QZbD4l4gbwx+PLbjsj4KC6r3yZrr51PdLUrBvoqBhqwuCksdaMntWGME
-u0V/ooJi4+uzCYzN06jFfAFXa2pWzVB5yKw1d6yYi9U/bPd4xn1phLUMHrC2bvdM
-H8P18gAS6rkWn+ageiWRHmkf4uoKgv3PrMjijkBaGpf6xjv6+0Q393jdVIC7wgJV
-8W0i1f1Awv68089mHBEarPTv3gz39251WFCPNQhEuSy79Li5zjwOprZXS0MnJXbm
-B00IPTIw51KuaoueWzY1pA2IFQ/0sH3fo2JhD0pp1gI0Dde7EQKCAQEA7RVgNelk
-3H3ZJsoOfOTFa03lnXuEfTAyxhEECRz64k54pSbEWV73PIeYByZvnsrKRydpYWUP
-Cp/mKhAJH4UCf2hzY0GyO7/D6+HEKZdCg6a0DNKckAoFkBfeOlLJLjLVAW2eEVxz
-tlFt+/WBE90GCvE5ovXuEhXGaPxCPp5giIN5phSzSD557bwwOyPwNKFZ7Ao77UNK
-kz6EzcvQgqb205SRRKGpS8/T/9LcLsUYVkBfYQ/BayjffO+cQF4vH5rB4x/8/T7t
-uUa79uY+LeGHgTSFIAui9LEK5ry//2hDJINsItYMks1Qo4Suu23pOuGerjiFTKWl
-mOIoFmPmbebAcwKCAQEAy6WaJczPcKQQ/hqglQtxU6VfP49ZjUKkoi+OCmtvKDts
-7pJfIkuluhnYGeqFfjnopw5bgZHNFm6Gccc5CwBtN6Wk1KnnOgDIg3kYCrfjtKy/
-BSSV3kLEBvhe9EJA56mFgP7RufMbHTXhXPGLkgE7JBZj2EKxp1qGYYVZesTMFwDM
-KEHwzIGcFkyZsd2jptyLYqcfDKzTHmFGcw1mdtLWAUdpv3xrS3GvrCbUMqIodjRd
-qkrg/d/kQpK7A3oLOWfa6eBQ2BXqaWB1x13bzJ2WlshxJAZ1p1ozKii5BQ9rvwWo
-muI5vd7o6A9Xsl8QzluSSSPi+NhjZ64gMBrXciRvmQKCAQB/dB5k3TP71SwITle7
-jMEVDquCHgT7yA2DrWIeBBZb0xPItS6ZXRRM1hhEv8UB+MMFvYpJcarEa3Gw6y38
-Y+UT2XMuyQKoXE9XX+e09DwtylDBE/hW9wxGio5NjHPbAjjAq81uR+Vs/hnCehkK
-NKgq+cOid9OkpVAk4Hg8cagzu3qKblZzYCLsS18ibA+WO6e73USaKLLOta1vdUKC
-+n92/0eZPc9lkjTGMvVrr0mGFNUxuOaiVTbQU4AMmpV6yBezol6/RjVGhWBHOz/y
-KmxOaY2nzJmuMf9KS+5rwAFYf86Ca9AWm4neXlYRLOVVYjWMM5Z1vhdoOSyT3ODj
-9ElBAoIBAGCRPaBxF2j9k8U7ESy8CVg10g3MxxVSJcl2rW9JdKNqUoRqykvz/Tlb
-afsYF4c8pJMbHs85OTxK2tv3MZiC8kdx99CUZL4/gtW9RWZHvuV9CPPCXoLPvC7l
-9fjztd1kqJb7vq3jltbqJtyw+ZMZnFbHez8gmSeXqKNz3XN3AKRjz2vDoREI4OA+
-IJ+UTzcf28TDJNkY1t/QFt0V3KG55psipwWTVTmoRjpnCzabaH5s5IGNElWwpoff
-FmlWpR3qnodKxGtDMS4Y/KC2ZDUKAU+s6uG/YmkiP6LdPqckod4qK8KORf1AR8dL
-BzXhGJISIDMonkeMLM8MZd0JzWIl3vkCggEAPBkExd2j4VY5s+wQJdiMto5DDoci
-kAEIvIkJY9I+Pt2lpinQKAcAAXbvueaJkJpq31f6Y66uok8QnD09bIQCABjjlIve
-o7qQ+H8/iqHQX1nbHDzInaDdad3jYtkWUHjHPaKg2/ktyNkFtlSHskvvCEVw5aju
-80Q3tRpQG9Pe4ZRjKEzNIpMXfQksFH0KwjwAVKwYJLqZxtNEYok4dpefSIsnH/rX
-pwK/pyBrFqxU6PURULUJuLqRlaIRXAU31RmJsVs2JbmI7Cbtj2TmqAOxsLsi5UeJ
-cZxcTAuYCNYMu88ktHul8YJdBF3rQKUOnsgW1cx7H6LGbuPZTpg8Sbyltw==
------END RSA PRIVATE KEY-----

examples/k8s/configs/node3/config.json

@@ -1,6 +0,0 @@
-{
-    "network-peer-list-gossip-frequency":"250ms",
-    "network-max-reconnect-delay":"1s",
-    "health-check-frequency":"2s",
-    "log-level":"debug"
-}

examples/k8s/configs/node3/staking.crt

@@ -1,30 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIFNzCCAx8CCQC687XFxtDRSjANBgkqhkiG9w0BAQsFADB/MQswCQYDVQQGEwJV
-UzELMAkGA1UECAwCTlkxDzANBgNVBAcMBkl0aGFjYTEQMA4GA1UECgwHQXZhbGFi
-czEOMAwGA1UECwwFR2Vja28xDDAKBgNVBAMMA2F2YTEiMCAGCSqGSIb3DQEJARYT
-c3RlcGhlbkBhdmFsYWJzLm9yZzAgFw0xOTA3MDIxNjEyMjVaGA8zMDE5MDcxMDE2
-MTIyNVowOjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAk5ZMRAwDgYDVQQKDAdBdmFs
-YWJzMQwwCgYDVQQDDANhdmEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoIC
-AQDZnDoDHE2nj82xDjH0Tb7OXMqQDHz+zbLidt6MSI1XB3vOAIEiPqrtenGnqRbV
-Fcm5GZxvxh4YQD8CjKSk1qgZJczs0DPSiGQ8Efl4PGO4xnEbllgL3PURPWp7mEV3
-oh6fxICgQKTBlT671EnFzB5lyJWpumRzvA1vyhBMsY8aO+xdq5LUFltYzBdvpgLX
-VaDwHZQ2PQEWtF0d0JO2N0WFFDGNmx6n8pKSeIAVDsTwZCZK+FCeeEyoGfXsINsc
-0yCMQslawkfOMqA9yBV3Ji6QmFYKyGYt65MWGNqPA4XrIyliKwCCXwz9mjaWyN7r
-Ayw9cWlLMODNmDORWzGRZ5290MEAEIZsqjYHVitRTM/RnNIadToZGO0y5uAkM14c
-mTvnsK1CP92qtfSisq75W/I91drThoEtTK78UGOl/5Q1YBR08F+tSUWZWyHeI6UO
-BUCGC2bCtmzKMl7vU25lG6mbCR1JuQi6RYpnfMjXH36lV4S7fTvSwwuR03h2F3H1
-eFkWNG2lbFrW0dzDCPg3lXwmFQ65hUcQhctznoBz5C1lF2eW03wuVgxinnuVlJHj
-y/GrqmWsASn1PDuVs4k7k6DJfwyHAiA0uxXrGfxYvp7H8j4+2YOmWiWl5xYgrEDj
-ur5n8Zx46PHQer2Avq3sbEGEe1MCtXJlj3drd5Him3m+NQIDAQABMA0GCSqGSIb3
-DQEBCwUAA4ICAQA40ax0dAMrbWikaJ5s6kjaGkPkYuxHNJbw047Do0hjw+ncXsxc
-QDHmWcoHHpgMQCx0+vp8y+oKZ4pnqNfGSuOTo7/l05oQW/NbWw9mHwTiLMeI18/x
-Ay+5LpOasw+omqWLbdbbWqL0o/RvtBdK2rkcHzTVzECgGSoxUFfZD+ck2odpH+aR
-sQVu86AZVfclN2mjMyFSqMItqRcVw7rqr3Xy6FcgRQPykUnpguCEgcc9c54c1lQ9
-Zpddt4ezY7cTdk86oh7yA8QFchvtE9Zb5dJ5Vu9bdy9ig1kyscPTm+SeyhXRchUo
-ql4H/czGBVMHUY41wY2VFz7HitECcTAIpS6QvcxxgYevGNjZZxyZvEA8SYpLMZyb
-omk4enDTLd/xK1yF7VFodTDEyq63IAm0NTQZUVvIDfJeuzuNz55uxgdUq2RLpaJe
-0bvrt9Obz+f5j2jonb2e0BuucwSdTyFXkUCxMW+piIUGkyrguAhlcHohDLEo2uB/
-iQ4fosGqqsl47b+TezT5pSSblkgUjiwz6eDpM4lQpx22MxsHVlxFHrcBNm0Td92v
-FixrmllamAZbEz1tB//0bipKaOOZuhANJfrgN8BC6v2ahl4/SBuut09a0Azyxqpp
-uCsyTnfNEd1W6c6noaq24s+7W7KKLIekuNn1NunnHqKqriEuH1xlxxPjYA==
------END CERTIFICATE-----

examples/k8s/configs/node3/staking.key

@@ -1,51 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIIJKQIBAAKCAgEA2Zw6AxxNp4/NsQ4x9E2+zlzKkAx8/s2y4nbejEiNVwd7zgCB
-Ij6q7Xpxp6kW1RXJuRmcb8YeGEA/AoykpNaoGSXM7NAz0ohkPBH5eDxjuMZxG5ZY
-C9z1ET1qe5hFd6Ien8SAoECkwZU+u9RJxcweZciVqbpkc7wNb8oQTLGPGjvsXauS
-1BZbWMwXb6YC11Wg8B2UNj0BFrRdHdCTtjdFhRQxjZsep/KSkniAFQ7E8GQmSvhQ
-nnhMqBn17CDbHNMgjELJWsJHzjKgPcgVdyYukJhWCshmLeuTFhjajwOF6yMpYisA
-gl8M/Zo2lsje6wMsPXFpSzDgzZgzkVsxkWedvdDBABCGbKo2B1YrUUzP0ZzSGnU6
-GRjtMubgJDNeHJk757CtQj/dqrX0orKu+VvyPdXa04aBLUyu/FBjpf+UNWAUdPBf
-rUlFmVsh3iOlDgVAhgtmwrZsyjJe71NuZRupmwkdSbkIukWKZ3zI1x9+pVeEu307
-0sMLkdN4dhdx9XhZFjRtpWxa1tHcwwj4N5V8JhUOuYVHEIXLc56Ac+QtZRdnltN8
-LlYMYp57lZSR48vxq6plrAEp9Tw7lbOJO5OgyX8MhwIgNLsV6xn8WL6ex/I+PtmD
-plolpecWIKxA47q+Z/GceOjx0Hq9gL6t7GxBhHtTArVyZY93a3eR4pt5vjUCAwEA
-AQKCAgBMoBNZZwz9FMkEMJBsizfF6Ky3Pn6BJqN31Q2WbjG+1HbG2iyeh1ye1L/S
-ntrYW5y1ngwU27lbJrxJRIbxOFjmygW32bR1zOsmr9mdef5PYSkQ4sbMHpj44hxt
-uvezIZYRAhuc0kZxmAEIGL+Fc9O8WX5Bzs1yZ2R/2bIVn2xZe4JGlZTVM64kvXD/
-MoDLnG5YPsIiuyZ3/TjQt9JblmjXbH3qdBW+Y88y3lWTlKjKUSmeuoOA2bF8e++5
-nvQo2TsbyKSoXcL1G6SLPLo6Q2qgJdQeZeR9BPe9DzFerInqe24mEChUv+2OG1Bf
-lgnQzUQ1uoquHF78Zjy6UVdJ8Sd8ufvKC9rz8JYsIynfw0gQC3F8/emm1QSabFvY
-tG4+x0K8FgrijjE08RvqgIndx9ftCNoN4u3lXxPrJhKpr2xuXSa4VZbumgN7fqWx
-UBC8lmPQi5VZmj3nJfj4datmBTvs1dOLRMdfdtTFz+cAdWNZxX3HOLZUSqMVWgXY
-kX0s7IV9GnyUntBktX+IEbWlAttzldyqF9md4avjKXQ+Y4PK/sR1yWsuvtiZdYUL
-/QrQHX0CsVv1hRcX0yekA0a8qwaGmxEcndEKv7wF1i626jc2fDR6qI1yp20Xl3Si
-kYBSNh7VK210XIhddSuVxW5/gyNnFABDfp1bSdTh5ZJRfNvtQQKCAQEA9Zipnyu8
-JKlLtWrh2iP9pmFBaeUgoF68IVfd6IXdV7nAHSWysiC43SCUmI63xluMYEJFdAZ+
-m/iRc/n6VFKEBW4+Ujk9VW1E1iqHgSABg7ntEsB2MDcYY0qKsN4CYjC2fNYO97zJ
-5oju84U3Qn8TWNkMsrUU7crm2oAQd08AizVFqLo1d8aIzRq+tl952S/lhfXKc/P9
-kfhl+RKjiYC2zbWnGinxc2Nbf5pWwnmtSrceng+ZkgVfSB3HvSckqzENye9YkpVM
-GE+KjEdss+QnGQRWM2JPlyoYDmhT6rrasRT6TKsecwo1rRXBi4C1eTZQSnZf24Og
-QurS//XzHzbnkQKCAQEA4tQSmaJPZNWYxOHnlivzselfJYyg5JyJVMQWw/7CvTcV
-GOpoD4hC25puAniT1U/RyaYaUFZ+Ip2FhK2Qce+uskNgtqFN9phh/DeO1g8CSaIe
-6Ebtg8N8GLc0YhdiJtd2XGrktj2XthML7OJPYIidd48tGuQizfijo4Fe1S0rSW56
-B4RHTh/O6a0taNeFbnZQJD52ha9wlnc/PZSCUMb9C0d08dSxdBQV+SVdGrl/IRfC
-qHHoC86GYDcmnviD5CFOxpx7AJ/hQAwPFQRCnWGHwDjpcoMOtktyo7pj9MDuzBUb
-kr4r1ei8f7PC9dmSYmYzJMQxLfz+Ti2SyyOmdM1CZQKCAQEAsVr4izCLIrJ7MNyp
-kt1QzDkJgw5q/ETNeQq5/rPE/xfty16w5//HYDCp/m15+y2bdtwEyd/yyHG9oFIS
-W5hnLIDLUpdxWmKZRkvaJP5W+ahnspX4A6OV4gYvl8ALWpsw/X+buX3FE80pOgSm
-vkeEUjIUAG3SWlKfWYUH3xDXJLBoyIsIF6HwoqVAufTCynvTNWUlOY0mPaZzBWZX
-YPHpkS4wKS3G5nwG1GRBaRlzcjRBUQWU8iUdBLg0yL0ett2qxnwoq1pTZG70b48Y
-yePl9CP0mBDTxycnzie7ChS73wt2Ia2lRJBH6OGALlzZMFpvqwZG/P/V2N05WIxl
-cNI2cQKCAQEAoys7VhlUU4zzoG2BUp27aDggobpP4yRYBgoo9kTFgaemHY5B3SqA
-LckhadWjQsdwekZql3AgvHXkHlVcmxl36fReFgJjOwjTM8QjlAin9KAS67RaF3cA
-RidEH2wCxz4nfsPGUvJruCZrZbRGtYKRA/iS0c1a3CAIVw4xUdh0UxaN4epeAO0Q
-wzg4ejrPWW7yp5/nUrOpohOWAo5aUBFU5lA4593A6WephthB6X+W3A9jkBigfB3M
-vFnwBltvRSRQrr7SHNjmCFSkZNHzuZL3PGe0RxPP+YK8rNrgHKjNHzHv69exYOdS
-8eo2TPR+QRqTn9ciKZrctRBDkK3MiCk/oQKCAQAZIZdkOClUPHfSk4x5nBXashKY
-gDzeyYHYLwNaBdEKgHNuf6jCltKWoDzZsqrv1Nya/148sTgSTg931bbch+lnHKJd
-cXrCQZWBnu2UquisFMeNOvpp0cPt4tIYDZVCRMRrwIlZqIJxb2nAwFvb0fEfLk+4
-gmu+3cCaN/vS3oJA9EFkzjxG0XiLOynyAZb5fY04NmFOIsq3rgT4DeCurHTKtOJ2
-t14oTNq06LD566OnT6plL7vaLtTR/9/qJc007Wjw8QdbTuQALqCjWWg2b7BVkOyR
-o9GrhPzSeT6nBHI8EoJv0nxeQWNDX9pZiW/1nsyuAAFJ9ISbDWjz/TwB17UL
------END RSA PRIVATE KEY-----

examples/k8s/configs/node4/config.json

@@ -1,6 +0,0 @@
-{
-    "network-peer-list-gossip-frequency":"250ms",
-    "network-max-reconnect-delay":"1s",
-    "health-check-frequency":"2s",
-    "log-level":"debug"
-}

examples/k8s/configs/node4/staking.crt

@@ -1,30 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIFNzCCAx8CCQC687XFxtDRSjANBgkqhkiG9w0BAQsFADB/MQswCQYDVQQGEwJV
-UzELMAkGA1UECAwCTlkxDzANBgNVBAcMBkl0aGFjYTEQMA4GA1UECgwHQXZhbGFi
-czEOMAwGA1UECwwFR2Vja28xDDAKBgNVBAMMA2F2YTEiMCAGCSqGSIb3DQEJARYT
-c3RlcGhlbkBhdmFsYWJzLm9yZzAgFw0xOTA3MDIxNjEyMjlaGA8zMDE5MDcxMDE2
-MTIyOVowOjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAk5ZMRAwDgYDVQQKDAdBdmFs
-YWJzMQwwCgYDVQQDDANhdmEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoIC
-AQDgK5r5vdHtJFEgw7hGE/lzKaHcvwzr32armq0k9tYchJXfT3k1j1lXtBAdcUN3
-gSRKjgzH/vjbn0ea3AiDCUd2Mck/n0KcJZ43S5I7ZjP7rbav296bKCZ1Hr7r5gXY
-Fhk+3aUsVfDUqAPBwyP8KeV31ARVA/s+WPeWqs69QXTdyJuBYE5pr40v1Sf+ebUI
-nZ37uGY3kiO0Ex/JgcoQsGJzrWD/ztbRCFIvrdNJZd0pGvMlmTKp7XsMR3cpvqk7
-70//MLCdyGW/1IArTSuD1vd7mBX1JyVXKycYN0vIOtbgxPOFutUyqDOeP7o51q4i
-PS3dCRgfmn/hWLwy+CtJe0BGKsb4tk0tKxo0se8v9JA8mUtnmzmMt4Y9jijOrCOB
-7XwWKmJYEm8N5Ubcy6cp2oL8vQVtzz3PXrkFt+3cFt1jrjdpQYgH4jykkWDeOjEf
-y1FCwzsNRudLTvLhfLn86/ZT4cLZ9JI7/WW0IPC8Fc7lhznJ+bIQUeEndaGdgVkx
-uEg0MxdrMr0jU0IFoXySRXNRzcDWZShEjBTv7tnFxLmoNU+uJb/KpMH6sRYi3zs8
-5ecaMKNyG+LDmBahUlHx5hKAH49O8855+AMhsg91ONZJldjQX0oZrIKzK5BpsqeT
-l4c2Yt/fALiZaeFk1pBEsvVeMOBCIuWE+b4UIEaLAOhxfwIDAQABMA0GCSqGSIb3
-DQEBCwUAA4ICAQB+2VXnqRqfG7H2/K0lgzxT+X9r1u+YDn0EaUGAG71s70Qnqbpn
-X7tBmCKLN6XgPL0HrN933nwiYrmfb8S33zZ7kw8GJDvaTamLNyem4/8qTBQmnRwe
-6rQ7SY2l73Ig87mR0WTi+rTnTTtc66+/jLtFeaj0Ycl9hBZXHKiULSGhsbUbwtkz
-iuNlANhoNKXNIABRImUq6OwYhEQN0DwHXj79wkpyDYjKZwHuEZUknc8Pl2oQPBke
-mil3tsrvGRkwhisnXX7tqh6rWKVZNJkO68hy7XO9aTXjbcB/7Y1K83ISNEyGPsH/
-pwFyd/j8O4modwh7Ulww1/hwcqnqiEFE3KzxX2pMh7VxeAmX2t5eXFZOlRx1lecM
-XRkVu19lYDKQHGSrGxng+BFlSOB96e5kXIbuIXKpPAACoBQ/JZYbtHks9H8OtNYO
-P2joqmnQ9wGkE5co1Ii//j2tuoCRCpK86mmbTlyNYvK+1/kkKcsaiiWXNrQsrIDZ
-BFs0FwX5g24OP5+brxTlRZE01R6St8lQj4IUwAcIzG8fFmMCWaYavrCZTeYaEiyF
-A0X2VA/vZ7x9D5P9Z5OakMhrMW+hJTYrpH1rm6KR7B26iU2kJRxTX7xQ9lrksqfB
-7lX+q0iheeYA4cHbGJNWwWgd+FQsK/PTeiyr4rfqututdWA0IxoLRc3XFw==
------END CERTIFICATE-----

examples/k8s/configs/node4/staking.key

@@ -1,51 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIIJKQIBAAKCAgEA4Cua+b3R7SRRIMO4RhP5cymh3L8M699mq5qtJPbWHISV3095
-NY9ZV7QQHXFDd4EkSo4Mx/74259HmtwIgwlHdjHJP59CnCWeN0uSO2Yz+622r9ve
-mygmdR6+6+YF2BYZPt2lLFXw1KgDwcMj/Cnld9QEVQP7Plj3lqrOvUF03cibgWBO
-aa+NL9Un/nm1CJ2d+7hmN5IjtBMfyYHKELBic61g/87W0QhSL63TSWXdKRrzJZky
-qe17DEd3Kb6pO+9P/zCwnchlv9SAK00rg9b3e5gV9SclVysnGDdLyDrW4MTzhbrV
-Mqgznj+6OdauIj0t3QkYH5p/4Vi8MvgrSXtARirG+LZNLSsaNLHvL/SQPJlLZ5s5
-jLeGPY4ozqwjge18FipiWBJvDeVG3MunKdqC/L0Fbc89z165Bbft3BbdY643aUGI
-B+I8pJFg3joxH8tRQsM7DUbnS07y4Xy5/Ov2U+HC2fSSO/1ltCDwvBXO5Yc5yfmy
-EFHhJ3WhnYFZMbhINDMXazK9I1NCBaF8kkVzUc3A1mUoRIwU7+7ZxcS5qDVPriW/
-yqTB+rEWIt87POXnGjCjchviw5gWoVJR8eYSgB+PTvPOefgDIbIPdTjWSZXY0F9K
-GayCsyuQabKnk5eHNmLf3wC4mWnhZNaQRLL1XjDgQiLlhPm+FCBGiwDocX8CAwEA
-AQKCAgEApuMPrxmH7Xn6A+BxkYpRTVETNZnt7rQUZXDzse8pm3WBdgxeemdL5iUh
-Uin+RjuYXwC9ty606hv8XOeuVo9T6kRKRNk157WBwjy6kwoVbSr4NJgFc5FCgDLx
-hAFtHF/nT4wG6ajZcBfdJCU45wPx13G5/+jE5LerKzniS7ctX+d3Daw69CdDfva7
-nZHSGqXs9Xdkcb6UYf1SztuXKTGHOgM7kXXVKy18sg5AnAX/zhhIKBeTRjqMPqn9
-ptBQgVQ6RAtlkTGdvmBfQt1ipfYlrJee0THhdLGlmzufaWOUkSVO/qIHEn1yYD+l
-TmXqoYbWXBXnJbAJwCQlh/SFlWDyiWWOxszxdwwT2ybw7OR3a0DEV0MbKJkUexyF
-92Lr3qoBSZRFQnXVvBgjQOwnzEFph1ANuGY3odL8JSM1tHniIsCs4WhDPOsbAj+h
-kwS51colMk3bNCZ3xeArjMLBVLgT7xLX/7ZYc7/oTEFWik+20TvSEWzdE1N/4gfJ
-jEU/VqrnNjyev2w9Ak6bEkwZFLS6VZ9rTWTF9jk8C1aXj/RhfaaC33xXBbhn9HuX
-lTu/JaLMp0Qc4aClqUYM6LlxIejH5b8fIxCNHJislXJDa6a6aQl85BiQODPFxVT5
-WCpQD4858EuLdX4BRW2fIGRY6DivR6uJRAmxLf+EwAg/rgTzUsECggEBAPSkHX5F
-BhRgudF0MnwN+enj4SoXHhRG+DTorxO1Zh2qN9lnXO9nMKMCXVJLIVvGFuiMRSJ0
-VKf1u0UqaBF02MbIvbei7mzkkW0/74m04X37iyMmtnmooQ0GEV84oONwAt3DeeTg
-vIpOtq9V26XHGaQDxcRFMFBuD02a2yf3JYkXj74i2scMP4xxMHMkJxGK9FSBOhnp
-k/p0hMl3FVGfo5Ns5T1Rl3pMueEF3B5+BvrV1z14IN/0lwuhujrUUYS4Ew+Pk5zC
-FSubfIQMqST1jvXXTaGgX0GPffa4lxgaDEATLewvL3Fjy27Xzl57i9ZvTNC4yFad
-4okjr/eItHtKVHECggEBAOqUKww/6uiJMNvc2NxLUMxuzB07rqOZKT2VMBkG5Gzk
-v81fDtlndD8cwHSqOLKscH/QKXD7WK3FCuvZSvMwCjEB4Pp1zgwJoBexuXvFDDbs
-0T77Qiwe+2WmRIiYev5aRG3lnBMM8RDS/QPzEdoxHdzrFURYVl0rv5l/7rwB2Zd6
-xAYHcUpZc4ZaysEgqQCuZQqC7Mrq7qfByUthH28Yicz1978fpE3dx15ceqjU9jBQ
-xUUwbeKT/UkQQvmYHdtgwEjhzVQL1OAAWkT6RssMqx2RAdi0SqWPFEhxNPHBpG9B
-lKUDBBIM6du916On0Bjghh3WhxQKpTIzveNAiexbXO8CggEBANvJohGyc37VU7wg
-18ZqTA/cwostD8IJ7K6kKb7cJy0Zo2l3mqAfJiwdULhBdWvdMPGmK+qDdxcbBy9h
-pPOh9avJ5+BWyjwcsabkXRFr53ZnCp7/BcuRO3fW7r6Mwsby+DBCkX2Whuz/QNOP
-oHF0yc138jKeMoTgDHGdYa2rNhbPiz24VLOlhmZnvq6DWXJCU7akDw3+swq9qhrS
-GN4nPS+TEvUfG6ctzYWj3RmsAhtTCThZd7edKCK0HvsBi2dgdQdy55xbJefynlCI
-i2IAF3s4/q7pxQrCntmNB3oI1N6wHH7n+Yi2rqsbyXVLK9vwTKPsj1h6Km8pF8ud
-DwEBS5ECggEAMnq2FMnAbE/xgq6wwB85APUq2XOZbj0sYcMz+X7BMym6mKBHGsOn
-gVlXlQN4dgKjpu2NrXF5MNPBOOWmulRxLQChgGRPdcmweMjXCGpr6XnmwW3iXIpC
-QSqZfueJOCkGpruNbZAQZDVzGyF4iwKc0YiJKA72btBWR9r+7dhcEbvqaP27BGvh
-b10kWpEDrVDaD3wDJtuNhe4uuhjpYcffB4s6yBcwDU2XdJfkEWban6UR/oSgcOy1
-yb5FG17/tdDJMCXfQKHXKmkJA+TzzQgp3o/w3MhXc+8pRzmNUiUAlKyBJ01R1+yN
-eqsMt3wKTQAr/EnJAagUyovV5gxiYcl7YwKCAQAdOYcZx/l//O0Ftm6wpXGRjakN
-IHFcP2i7mUW76Ewnj1iFa9Yv7pgbbBD9S1SMuetfIWcqSjDiUaymnDdA18NVYUYv
-lhlUJ6kwdVusejqfcn+75Jf87BvWdIVGrNxPdB7Z/lmbWxFqyZi00R90UGBntaMu
-zg/ibrLgatzA9SKgoWXm2bLt6bbXefmOgnZXyw8Qko70Xxtx5eBR1BDAQjDis81n
-Lg96sJ3LOn7SXHfxJ3BtXshTJAoBFx6EpmulgNoPWIkJtd7XWYP6Yy22D+kK7OhH
-Rq3CiYMtDmZoub/kVBL0MVdSm7hn1TSVTHjFoW6cwQ37iKHjkZVRwX1Kzt0B
------END RSA PRIVATE KEY-----

examples/k8s/main.go

@@ -1,153 +0,0 @@
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"os"
-	"time"
-
-	"github.com/ava-labs/avalanche-network-runner/k8s"
-	"github.com/ava-labs/avalanche-network-runner/network"
-	"github.com/ava-labs/avalanche-network-runner/network/node"
-	"github.com/ava-labs/avalanche-network-runner/utils"
-	"github.com/ava-labs/avalanchego/utils/logging"
-)
-
-const (
-	// defaultNetworkTimeout to wait network to come up until deemed failed
-	defaultNetworkTimeout = 120 * time.Second
-	configFileName        = "/conf.json"
-)
-
-// TODO: shouldn't we think of something like Viper for loading config file?
-var goPath = os.ExpandEnv("$GOPATH")
-
-// Network and node configs
-type allConfig struct {
-	NetworkConfig network.Config   `json:"networkConfig"`
-	K8sConfig     []k8s.ObjectSpec `json:"k8sConfig"`
-}
-
-func main() {
-	if err := run(); err != nil {
-		os.Exit(1)
-	}
-}
-
-func run() error {
-	// Create the logger
-	logFactory := logging.NewFactory(logging.DefaultConfig)
-	log, err := logFactory.Make("main")
-	if err != nil {
-		fmt.Println(err)
-		return err
-	}
-
-	configDir := fmt.Sprintf("%s/src/github.com/ava-labs/avalanche-network-runner/examples/k8s", goPath)
-	if goPath == "" {
-		configDir = "./examples/k8s"
-	}
-	log.Info("reading config file...")
-	configFile, err := os.ReadFile(configDir + configFileName)
-	if err != nil {
-		log.Fatal("%s", err)
-		return err
-	}
-
-	// Network and node configs
-	var allConfig allConfig
-	if err := json.Unmarshal(configFile, &allConfig); err != nil {
-		log.Fatal("%s", err)
-		return err
-	}
-
-	// TODO maybe do networkConfig validation
-	log.Info("parsing config...")
-	networkConfig, err := readConfig(allConfig)
-	if err != nil {
-		log.Fatal("error reading configs: %s", err)
-		return err
-	}
-
-	level, err := logging.ToLevel(networkConfig.LogLevel)
-	if err != nil {
-		log.Fatal("couldn't parse log: %s", err)
-		return err
-	}
-	log.SetLogLevel(level)
-	log.SetDisplayLevel(level)
-	ctx, cancel := context.WithTimeout(context.Background(), defaultNetworkTimeout)
-	defer cancel()
-
-	network, err := k8s.NewNetwork(log, networkConfig)
-	if err != nil {
-		log.Fatal("Error creating network: %s", err)
-		return err
-	}
-	defer func() {
-		if err := network.Stop(ctx); err != nil {
-			log.Error("Error stopping network (ignored): %s", err)
-		}
-	}()
-
-	log.Info("Network created. Booting...")
-
-	errCh := network.Healthy(ctx)
-
-	select {
-	case <-ctx.Done():
-		log.Fatal("Timed out waiting for network to boot. Exiting.")
-		return ctx.Err()
-	case err := <-errCh:
-		if err != nil {
-			log.Fatal("Error booting network: %s", err)
-			return err
-		}
-	}
-	log.Info("Network created!!!")
-	return nil
-}
-
-func readConfig(rconfig allConfig) (network.Config, error) {
-	configDir := "./examples/k8s/configs"
-	genesisFile, err := os.ReadFile(fmt.Sprintf("%s/genesis.json", configDir))
-	if err != nil {
-		return network.Config{}, err
-	}
-	netcfg := rconfig.NetworkConfig
-	netcfg.Genesis = string(genesisFile)
-	netcfg.NodeConfigs = make([]node.Config, 0)
-	for i, k := range rconfig.K8sConfig {
-		nodeConfigDir := fmt.Sprintf("%s/node%d", configDir, i)
-		key, err := os.ReadFile(fmt.Sprintf("%s/staking.key", nodeConfigDir))
-		if err != nil {
-			return network.Config{}, err
-		}
-		cert, err := os.ReadFile(fmt.Sprintf("%s/staking.crt", nodeConfigDir))
-		if err != nil {
-			return network.Config{}, err
-		}
-		configFile, err := os.ReadFile(fmt.Sprintf("%s/config.json", nodeConfigDir))
-		if err != nil {
-			return network.Config{}, err
-		}
-		c := node.Config{
-			Name:        fmt.Sprintf("validator-%d", i),
-			StakingCert: string(cert),
-			StakingKey:  string(key),
-			ConfigFile:  string(configFile),
-			ImplSpecificConfig: utils.NewK8sNodeConfigJsonRaw(
-				k.APIVersion,
-				k.Identifier,
-				k.Image,
-				k.Kind,
-				k.Namespace,
-				k.Tag,
-			),
-			IsBeacon: i == 0,
-		}
-		netcfg.NodeConfigs = append(netcfg.NodeConfigs, c)
-	}
-	return netcfg, nil
-}

examples/k8s/simple-netrunner-pod.yaml

@@ -1,13 +0,0 @@
----
-apiVersion: v1
-kind: Pod 
-metadata:
-  name: k8s-netrunner 
-  namespace: ci-avalanchego 
-spec:
-  containers:
-  - name: k8s-netrunner 
-    image: k8s-netrunner:alpha
-    imagePullPolicy: IfNotPresent 
-  restartPolicy: Never
-  serviceAccountName:  k8s-netrunner

examples/k8s/svc-rbac.yaml

@@ -1,116 +0,0 @@
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: k8s-netrunner
-  namespace: ci-avalanchego
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: k8s-netrunner
-rules:
-  - apiGroups:
-      - apps
-    resources:
-      - statefulsets
-    verbs:
-      - create
-      - delete
-      - get
-      - list
-      - patch
-      - update
-      - watch
-  - apiGroups:
-      - chain.avax.network
-    resources:
-      - avalanchegoes
-      - deployments
-    verbs:
-      - create
-      - delete
-      - get
-      - list
-      - patch
-      - update
-      - watch
-  - apiGroups:
-      - chain.avax.network
-    resources:
-      - avalanchegoes/finalizers
-    verbs:
-      - update
-  - apiGroups:
-      - chain.avax.network
-    resources:
-      - avalanchegoes/status
-    verbs:
-      - get
-      - patch
-      - update
-  - apiGroups:
-      - ""
-    resources:
-      - configmaps
-      - pods
-    verbs:
-      - create
-      - delete
-      - get
-      - list
-      - patch
-      - update
-      - watch
-  - apiGroups:
-      - ""
-    resources:
-      - persistentvolumeclaims
-    verbs:
-      - create
-      - delete
-      - get
-      - list
-      - patch
-      - update
-      - watch
-  - apiGroups:
-      - ""
-    resources:
-      - secrets
-    verbs:
-      - create
-      - delete
-      - get
-      - list
-      - patch
-      - update
-      - watch
-  - apiGroups:
-      - ""
-    resources:
-      - services
-    verbs:
-      - create
-      - delete
-      - get
-      - list
-      - patch
-      - update
-      - watch
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: k8s-netrunner
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: k8s-netrunner
-subjects:
-  - kind: ServiceAccount
-    name: k8s-netrunner
-    namespace: ci-avalanchego
----
-

examples/local/fivenodenetwork/main.go

@@ -46,7 +46,10 @@ func shutdownOnSignal(
 // The network runs until the user provides a SIGINT or SIGTERM.
 func main() {
 	// Create the logger
-	logFactory := logging.NewFactory(logging.DefaultConfig)
+	logFactory := logging.NewFactory(logging.Config{
+		DisplayLevel: logging.Info,
+		LogLevel:     logging.Debug,
+	})
 	log, err := logFactory.Make("main")
 	if err != nil {
 		fmt.Println(err)
@@ -86,9 +89,8 @@ func run(log logging.Logger, binaryPath string) error {
 	// Wait until the nodes in the network are ready
 	ctx, cancel := context.WithTimeout(context.Background(), healthyTimeout)
 	defer cancel()
-	healthyChan := nw.Healthy(ctx)
 	log.Info("waiting for all nodes to report healthy...")
-	if err := <-healthyChan; err != nil {
+	if err := nw.Healthy(ctx); err != nil {
 		return err
 	}
 

examples/local/indepth/main.go

@@ -11,7 +11,6 @@ import (
 	"github.com/ava-labs/avalanche-network-runner/local"
 	"github.com/ava-labs/avalanche-network-runner/network"
 	"github.com/ava-labs/avalanche-network-runner/network/node"
-	"github.com/ava-labs/avalanche-network-runner/utils"
 	"github.com/ava-labs/avalanchego/config"
 	"github.com/ava-labs/avalanchego/staking"
 	"github.com/ava-labs/avalanchego/utils/logging"
@@ -54,7 +53,10 @@ func shutdownOnSignal(
 // The network runs until the user provides a SIGINT or SIGTERM.
 func main() {
 	// Create the logger
-	logFactory := logging.NewFactory(logging.DefaultConfig)
+	logFactory := logging.NewFactory(logging.Config{
+		DisplayLevel: logging.Info,
+		LogLevel:     logging.Debug,
+	})
 	log, err := logFactory.Make("main")
 	if err != nil {
 		fmt.Println(err)
@@ -91,10 +93,10 @@ func run(log logging.Logger, binaryPath string) error {
 	// Wait until the nodes in the network are ready
 	ctx, cancel := context.WithTimeout(context.Background(), healthyTimeout)
 	defer cancel()
-	healthyChan := nw.Healthy(ctx)
 	log.Info("waiting for all nodes to report healthy...")
-	if err := <-healthyChan; err != nil {
+	if err := nw.Healthy(ctx); err != nil {
 		return err
+
 	}
 
 	// Print the node names
@@ -105,17 +107,17 @@ func run(log logging.Logger, binaryPath string) error {
 	log.Info("current network's nodes: %s", nodeNames)
 
 	// Get one node
-	node0, err := nw.GetNode(nodeNames[0])
+	node1, err := nw.GetNode(nodeNames[0])
 	if err != nil {
 		return err
 	}
 
 	// Get its node ID through its API and print it
-	node0ID, err := node0.GetAPIClient().InfoAPI().GetNodeID(context.Background())
+	node1ID, err := node1.GetAPIClient().InfoAPI().GetNodeID(context.Background())
 	if err != nil {
 		return err
 	}
-	log.Info("one node's ID is: %s", node0ID)
+	log.Info("one node's ID is: %s", node1ID)
 
 	// Add a new node with generated cert/key/nodeid
 	stakingCert, stakingKey, err := staking.NewCertAndKeyBytes()
@@ -123,10 +125,10 @@ func run(log logging.Logger, binaryPath string) error {
 		return err
 	}
 	nodeConfig := node.Config{
-		Name:               "New Node",
-		ImplSpecificConfig: utils.NewLocalNodeConfigJsonRaw(binaryPath),
-		StakingKey:         string(stakingKey),
-		StakingCert:        string(stakingCert),
+		Name:        "New Node",
+		BinaryPath:  binaryPath,
+		StakingKey:  string(stakingKey),
+		StakingCert: string(stakingCert),
 		// The flags below would override the config in this node's config file,
 		// if it had one.
 		Flags: map[string]interface{}{
@@ -148,9 +150,8 @@ func run(log logging.Logger, binaryPath string) error {
 	// Wait until the nodes in the updated network are ready
 	ctx, cancel = context.WithTimeout(context.Background(), healthyTimeout)
 	defer cancel()
-	healthyChan = nw.Healthy(ctx)
 	log.Info("waiting for updated network to report healthy...")
-	if err := <-healthyChan; err != nil {
+	if err := nw.Healthy(ctx); err != nil {
 		return err
 	}
 

go.mod

@@ -2,36 +2,30 @@ module github.com/ava-labs/avalanche-network-runner
 
 go 1.17
 
-// ignores the dependency for Rust SDK
-// only used for protocol buffer submodule
-exclude google.golang.org/api v0.62.0
-
 require (
-	github.com/ava-labs/avalanchego v1.7.11-0.20220416161358-8755486a274e
-	github.com/ava-labs/avalanchego-operator v0.0.0-20211115144351-99f07d2570bf
-	github.com/ava-labs/coreth v0.8.9-rc.1
+	github.com/ava-labs/avalanchego v1.7.13
+	github.com/ava-labs/coreth v0.8.12-rc.1
 	github.com/ethereum/go-ethereum v1.10.16
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.2
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.10.3
 	github.com/onsi/ginkgo/v2 v2.1.3
 	github.com/onsi/gomega v1.19.0
-	github.com/prometheus/client_golang v1.11.0
+	github.com/otiai10/copy v1.7.0
+	github.com/prometheus/client_golang v1.12.2
 	github.com/spf13/cobra v1.3.0
 	github.com/stretchr/testify v1.7.0
-	go.uber.org/zap v1.19.0
+	go.uber.org/zap v1.21.0
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa
-	google.golang.org/grpc v1.43.0
-	google.golang.org/protobuf v1.27.1
-	k8s.io/api v0.22.3
-	k8s.io/apimachinery v0.22.3
-	sigs.k8s.io/controller-runtime v0.10.2
+	google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8
+	google.golang.org/grpc v1.47.0
+	google.golang.org/protobuf v1.28.0
 )
 
 require (
 	github.com/Microsoft/go-winio v0.4.16 // indirect
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
 	github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
-	github.com/VictoriaMetrics/fastcache v1.9.0 // indirect
+	github.com/VictoriaMetrics/fastcache v1.10.0 // indirect
+	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/btcsuite/btcd v0.21.0-beta // indirect
 	github.com/btcsuite/btcutil v1.0.2 // indirect
@@ -39,24 +33,17 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/deckarep/golang-set v1.8.0 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v3 v3.0.0-20200627015759-01fd2de07837 // indirect
-	github.com/evanphx/json-patch v4.11.0+incompatible // indirect
 	github.com/fatih/color v1.13.0 // indirect
 	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 // indirect
-	github.com/go-logr/logr v0.4.0 // indirect
 	github.com/go-ole/go-ole v1.2.1 // indirect
 	github.com/go-stack/stack v1.8.0 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt v3.2.1+incompatible // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/mock v1.6.0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/go-cmp v0.5.6 // indirect
-	github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa // indirect
 	github.com/google/uuid v1.1.5 // indirect
-	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/gorilla/rpc v1.2.0 // indirect
 	github.com/gorilla/websocket v1.4.2 // indirect
@@ -70,11 +57,10 @@ require (
 	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
 	github.com/holiman/uint256 v1.2.0 // indirect
 	github.com/huin/goupnp v1.0.2 // indirect
-	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jackpal/gateway v1.0.6 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
-	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/jhump/protoreflect v1.11.1-0.20220213155251-0c2aedc66cf4 // indirect
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
 	github.com/linxGnu/grocksdb v1.6.34 // indirect
 	github.com/magiconair/properties v1.8.5 // indirect
@@ -85,18 +71,15 @@ require (
 	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
 	github.com/mitchellh/mapstructure v1.4.3 // indirect
 	github.com/mitchellh/pointerstructure v1.2.0 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/mr-tron/base58 v1.2.0 // indirect
 	github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d // indirect
 	github.com/oklog/run v1.1.0 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/pelletier/go-toml v1.9.4 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.26.0 // indirect
-	github.com/prometheus/procfs v0.6.0 // indirect
+	github.com/prometheus/common v0.32.1 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/rjeczalik/notify v0.9.2 // indirect
 	github.com/rs/cors v1.7.0 // indirect
 	github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
@@ -113,29 +96,18 @@ require (
 	github.com/tklauser/go-sysconf v0.3.5 // indirect
 	github.com/tklauser/numcpus v0.2.2 // indirect
 	github.com/tyler-smith/go-bip39 v1.0.2 // indirect
-	go.uber.org/atomic v1.7.0 // indirect
-	go.uber.org/multierr v1.6.0 // indirect
-	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/multierr v1.8.0 // indirect
+	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
 	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
-	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
-	golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a // indirect
+	golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64 // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
-	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	gonum.org/v1/gonum v0.9.1 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.66.2 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/urfave/cli.v1 v1.20.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	k8s.io/apiextensions-apiserver v0.22.2 // indirect
-	k8s.io/client-go v0.22.3 // indirect
-	k8s.io/component-base v0.22.2 // indirect
-	k8s.io/klog/v2 v2.9.0 // indirect
-	k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e // indirect
-	k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
-	sigs.k8s.io/yaml v1.3.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

k8s/constants.go

@@ -1,20 +0,0 @@
-package k8s
-
-import "time"
-
-// TODO make these configurable
-const (
-	resourceLimitsCPU      = "1"
-	resourceLimitsMemory   = "4Gi"
-	resourceRequestCPU     = "500m"
-	resourceRequestMemory  = "2Gi"
-	stopTimeout            = 10 * time.Second
-	removeTimeout          = 30 * time.Second
-	nodeReachableCheckFreq = 5 * time.Second
-	healthCheckFreq        = 3 * time.Second
-	// TODO export these default ports from the
-	// AvalancheGo operator and use the imported
-	// values instead of re-defining them below.
-	defaultAPIPort = uint16(9650)
-	defaultP2PPort = uint16(9651)
-)

k8s/mocks/dns_checker.go

@@ -1,28 +0,0 @@
-// Code generated by mockery v2.9.4. DO NOT EDIT.
-
-package mocks
-
-import (
-	context "context"
-
-	mock "github.com/stretchr/testify/mock"
-)
-
-// DnsReachableChecker is an autogenerated mock type for the dnsReachableChecker type
-type DnsReachableChecker struct {
-	mock.Mock
-}
-
-// Reachable provides a mock function with given fields: ctx, url
-func (_m *DnsReachableChecker) Reachable(ctx context.Context, url string) bool {
-	ret := _m.Called(ctx, url)
-
-	var r0 bool
-	if rf, ok := ret.Get(0).(func(context.Context, string) bool); ok {
-		r0 = rf(ctx, url)
-	} else {
-		r0 = ret.Get(0).(bool)
-	}
-
-	return r0
-}

k8s/mocks/k8s_client.go

@@ -1,210 +0,0 @@
-// Code generated by mockery v2.9.4. DO NOT EDIT.
-
-package mocks
-
-import (
-	context "context"
-
-	client "sigs.k8s.io/controller-runtime/pkg/client"
-
-	meta "k8s.io/apimachinery/pkg/api/meta"
-
-	mock "github.com/stretchr/testify/mock"
-
-	runtime "k8s.io/apimachinery/pkg/runtime"
-
-	types "k8s.io/apimachinery/pkg/types"
-)
-
-// Client is an autogenerated mock type for the Client type
-type Client struct {
-	mock.Mock
-}
-
-// Create provides a mock function with given fields: ctx, obj, opts
-func (_m *Client) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error {
-	_va := make([]interface{}, len(opts))
-	for _i := range opts {
-		_va[_i] = opts[_i]
-	}
-	var _ca []interface{}
-	_ca = append(_ca, ctx, obj)
-	_ca = append(_ca, _va...)
-	ret := _m.Called(_ca...)
-
-	var r0 error
-	if rf, ok := ret.Get(0).(func(context.Context, client.Object, ...client.CreateOption) error); ok {
-		r0 = rf(ctx, obj, opts...)
-	} else {
-		r0 = ret.Error(0)
-	}
-
-	return r0
-}
-
-// Delete provides a mock function with given fields: ctx, obj, opts
-func (_m *Client) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error {
-	_va := make([]interface{}, len(opts))
-	for _i := range opts {
-		_va[_i] = opts[_i]
-	}
-	var _ca []interface{}
-	_ca = append(_ca, ctx, obj)
-	_ca = append(_ca, _va...)
-	ret := _m.Called(_ca...)
-
-	var r0 error
-	if rf, ok := ret.Get(0).(func(context.Context, client.Object, ...client.DeleteOption) error); ok {
-		r0 = rf(ctx, obj, opts...)
-	} else {
-		r0 = ret.Error(0)
-	}
-
-	return r0
-}
-
-// DeleteAllOf provides a mock function with given fields: ctx, obj, opts
-func (_m *Client) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error {
-	_va := make([]interface{}, len(opts))
-	for _i := range opts {
-		_va[_i] = opts[_i]
-	}
-	var _ca []interface{}
-	_ca = append(_ca, ctx, obj)
-	_ca = append(_ca, _va...)
-	ret := _m.Called(_ca...)
-
-	var r0 error
-	if rf, ok := ret.Get(0).(func(context.Context, client.Object, ...client.DeleteAllOfOption) error); ok {
-		r0 = rf(ctx, obj, opts...)
-	} else {
-		r0 = ret.Error(0)
-	}
-
-	return r0
-}
-
-// Get provides a mock function with given fields: ctx, key, obj
-func (_m *Client) Get(ctx context.Context, key types.NamespacedName, obj client.Object) error {
-	ret := _m.Called(ctx, key, obj)
-
-	var r0 error
-	if rf, ok := ret.Get(0).(func(context.Context, types.NamespacedName, client.Object) error); ok {
-		r0 = rf(ctx, key, obj)
-	} else {
-		r0 = ret.Error(0)
-	}
-
-	return r0
-}
-
-// List provides a mock function with given fields: ctx, list, opts
-func (_m *Client) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error {
-	_va := make([]interface{}, len(opts))
-	for _i := range opts {
-		_va[_i] = opts[_i]
-	}
-	var _ca []interface{}
-	_ca = append(_ca, ctx, list)
-	_ca = append(_ca, _va...)
-	ret := _m.Called(_ca...)
-
-	var r0 error
-	if rf, ok := ret.Get(0).(func(context.Context, client.ObjectList, ...client.ListOption) error); ok {
-		r0 = rf(ctx, list, opts...)
-	} else {
-		r0 = ret.Error(0)
-	}
-
-	return r0
-}
-
-// Patch provides a mock function with given fields: ctx, obj, patch, opts
-func (_m *Client) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
-	_va := make([]interface{}, len(opts))
-	for _i := range opts {
-		_va[_i] = opts[_i]
-	}
-	var _ca []interface{}
-	_ca = append(_ca, ctx, obj, patch)
-	_ca = append(_ca, _va...)
-	ret := _m.Called(_ca...)
-
-	var r0 error
-	if rf, ok := ret.Get(0).(func(context.Context, client.Object, client.Patch, ...client.PatchOption) error); ok {
-		r0 = rf(ctx, obj, patch, opts...)
-	} else {
-		r0 = ret.Error(0)
-	}
-
-	return r0
-}
-
-// RESTMapper provides a mock function with given fields:
-func (_m *Client) RESTMapper() meta.RESTMapper {
-	ret := _m.Called()
-
-	var r0 meta.RESTMapper
-	if rf, ok := ret.Get(0).(func() meta.RESTMapper); ok {
-		r0 = rf()
-	} else {
-		if ret.Get(0) != nil {
-			r0 = ret.Get(0).(meta.RESTMapper)
-		}
-	}
-
-	return r0
-}
-
-// Scheme provides a mock function with given fields:
-func (_m *Client) Scheme() *runtime.Scheme {
-	ret := _m.Called()
-
-	var r0 *runtime.Scheme
-	if rf, ok := ret.Get(0).(func() *runtime.Scheme); ok {
-		r0 = rf()
-	} else {
-		if ret.Get(0) != nil {
-			r0 = ret.Get(0).(*runtime.Scheme)
-		}
-	}
-
-	return r0
-}
-
-// Status provides a mock function with given fields:
-func (_m *Client) Status() client.StatusWriter {
-	ret := _m.Called()
-
-	var r0 client.StatusWriter
-	if rf, ok := ret.Get(0).(func() client.StatusWriter); ok {
-		r0 = rf()
-	} else {
-		if ret.Get(0) != nil {
-			r0 = ret.Get(0).(client.StatusWriter)
-		}
-	}
-
-	return r0
-}
-
-// Update provides a mock function with given fields: ctx, obj, opts
-func (_m *Client) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error {
-	_va := make([]interface{}, len(opts))
-	for _i := range opts {
-		_va[_i] = opts[_i]
-	}
-	var _ca []interface{}
-	_ca = append(_ca, ctx, obj)
-	_ca = append(_ca, _va...)
-	ret := _m.Called(_ca...)
-
-	var r0 error
-	if rf, ok := ret.Get(0).(func(context.Context, client.Object, ...client.UpdateOption) error); ok {
-		r0 = rf(ctx, obj, opts...)
-	} else {
-		r0 = ret.Error(0)
-	}
-
-	return r0
-}

k8s/network.go

@@ -1,461 +0,0 @@
-package k8s
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/ava-labs/avalanche-network-runner/api"
-	"github.com/ava-labs/avalanche-network-runner/network"
-	"github.com/ava-labs/avalanche-network-runner/network/node"
-	"github.com/ava-labs/avalanchego/ids"
-	"github.com/ava-labs/avalanchego/utils/constants"
-	"github.com/ava-labs/avalanchego/utils/logging"
-	"golang.org/x/sync/errgroup"
-
-	k8sapi "github.com/ava-labs/avalanchego-operator/api/v1alpha1"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/types"
-	ctrl "sigs.k8s.io/controller-runtime"
-	k8scli "sigs.k8s.io/controller-runtime/pkg/client"
-)
-
-const (
-	// How long we'll wait for a kubernetes pod to become reachable
-	nodeReachableTimeout = 2 * time.Minute
-	// Time between checks to see if a node is reachable
-	nodeReachableRetryFreq = 3 * time.Second
-	// Prefix the avalanchego-operator uses to pass params to avalanchego nodes
-	envVarPrefix = "AVAGO_"
-)
-
-var _ network.Network = (*networkImpl)(nil)
-
-// networkParams encapsulate params to create a network
-type networkParams struct {
-	conf          network.Config
-	log           logging.Logger
-	k8sClient     k8scli.Client
-	dnsChecker    dnsReachableChecker
-	apiClientFunc api.NewAPIClientF
-}
-
-// networkImpl is the kubernetes data type representing a kubernetes network adapter.
-// It implements the network.Network interface.
-type networkImpl struct {
-	log    logging.Logger
-	config network.Config
-	// the kubernetes client
-	k8scli k8scli.Client
-	// Must be held when [nodes.lock] is accessed
-	nodesLock sync.RWMutex
-	// Node name --> The node.
-	// If there is a running k8s pod for a node, it's in [nodes]
-	nodes map[string]*Node
-	// URI of the beacon node
-	// TODO allow multiple beacons
-	beaconURL string
-	// Closed when network is done shutting down
-	closedOnStopCh chan struct{}
-	// Checks if a node is reachable via DNS
-	dnsChecker dnsReachableChecker
-	// Create the K8s API client
-	apiClientFunc api.NewAPIClientF
-}
-
-func newK8sClient() (k8scli.Client, error) {
-	// init k8s client
-	scheme := runtime.NewScheme()
-	if err := k8sapi.AddToScheme(scheme); err != nil {
-		return nil, err
-	}
-	kubeconfig := ctrl.GetConfigOrDie()
-	return k8scli.New(kubeconfig, k8scli.Options{Scheme: scheme})
-}
-
-// If this function returns a nil error, you *must* eventually call
-// Stop() on the returned network. Failure to do so will cause old
-// state to linger in k8s.
-func newNetwork(params networkParams) (network.Network, error) {
-	beacons, nonBeacons, err := createDeploymentFromConfig(params)
-	if err != nil {
-		return nil, err
-	}
-	if len(beacons) == 0 {
-		return nil, errors.New("NodeConfigs don't have any beacon nodes")
-	}
-	net := &networkImpl{
-		config:         params.conf,
-		k8scli:         params.k8sClient,
-		closedOnStopCh: make(chan struct{}),
-		log:            params.log,
-		nodes:          make(map[string]*Node, len(params.conf.NodeConfigs)),
-		dnsChecker:     params.dnsChecker,
-		apiClientFunc:  params.apiClientFunc,
-	}
-	net.log.Debug("launching beacon nodes...")
-	// Start the beacon nodes and wait until they're reachable
-	if err := net.launchNodes(beacons); err != nil {
-		ctx, cancel := context.WithTimeout(context.Background(), stopTimeout)
-		defer cancel()
-		if err := net.Stop(ctx); err != nil {
-			net.log.Warn("error stopping network: %s", err)
-		}
-		return nil, fmt.Errorf("error launching beacons: %w", err)
-	}
-	// Tell future nodes the IP of the beacon node
-	// TODO add support for multiple beacons
-	// TODO don't rely on [beacons] being updated in launchNodes.
-	//      It's not a very clean pattern.
-	net.beaconURL = beacons[0].Status.NetworkMembersURI[0]
-	if net.beaconURL == "" {
-		ctx, cancel := context.WithTimeout(context.Background(), stopTimeout)
-		defer cancel()
-		if err := net.Stop(ctx); err != nil {
-			net.log.Warn("error stopping network: %s", err)
-		}
-		return nil, errors.New("Bootstrap URI is set to empty")
-	}
-	net.log.Info("Beacon node started")
-	// Start the non-beacon nodes and wait until they're reachable
-	if err := net.launchNodes(nonBeacons); err != nil {
-		ctx, cancel := context.WithTimeout(context.Background(), stopTimeout)
-		defer cancel()
-		if err := net.Stop(ctx); err != nil {
-			net.log.Warn("error stopping network: %s", err)
-		}
-		return nil, fmt.Errorf("Error launching non-beacons: %s", err)
-	}
-	net.log.Info("All nodes started. Network: %s", net)
-	return net, nil
-}
-
-// NewNetwork returns a new network whose initial state is specified in the config
-func NewNetwork(log logging.Logger, conf network.Config) (network.Network, error) {
-	k8sClient, err := newK8sClient()
-	if err != nil {
-		return nil, fmt.Errorf("couldn't create k8s client: %w", err)
-	}
-	return newNetwork(networkParams{
-		conf:      conf,
-		log:       log,
-		k8sClient: k8sClient,
-		// TODO is there a better way to wait until the node is reachable?
-		dnsChecker:    &defaultDNSReachableChecker{},
-		apiClientFunc: api.NewAPIClient,
-	})
-}
-
-// See network.Network
-func (a *networkImpl) GetNodeNames() ([]string, error) {
-	a.nodesLock.RLock()
-	defer a.nodesLock.RUnlock()
-
-	if a.isStopped() {
-		return nil, network.ErrStopped
-	}
-
-	nodes := make([]string, len(a.nodes))
-	i := 0
-	for _, n := range a.nodes {
-		nodes[i] = n.name
-		i++
-	}
-	return nodes, nil
-}
-
-// See network.Network
-func (a *networkImpl) Healthy(ctx context.Context) chan error {
-	a.nodesLock.RLock()
-	defer a.nodesLock.RUnlock()
-
-	errCh := make(chan error, 1)
-	if a.isStopped() {
-		errCh <- network.ErrStopped
-		return errCh
-	}
-	nodes := make([]*Node, 0, len(a.nodes))
-	for _, node := range a.nodes {
-		nodes = append(nodes, node)
-	}
-
-	go func() {
-		errGr, ctx := errgroup.WithContext(ctx)
-		for _, node := range nodes {
-			node := node
-			errGr.Go(func() error {
-				// Every constants.HealthCheckInterval, query node for health status.
-				// Do this until ctx timeout
-				for {
-					select {
-					case <-a.closedOnStopCh:
-						return network.ErrStopped
-					case <-ctx.Done():
-						return fmt.Errorf("node %q failed to become healthy within timeout", node.GetName())
-					case <-time.After(healthCheckFreq):
-					}
-					health, err := node.apiClient.HealthAPI().Health(ctx)
-					if err == nil && health.Healthy {
-						a.log.Info("node %q became healthy", node.GetName())
-						return nil
-					}
-				}
-			})
-		}
-		// Wait until all nodes are ready or timeout
-		if err := errGr.Wait(); err != nil {
-			errCh <- err
-		}
-		close(errCh)
-	}()
-	return errCh
-}
-
-// See network.Network
-func (a *networkImpl) Stop(ctx context.Context) error {
-	a.nodesLock.Lock()
-	defer a.nodesLock.Unlock()
-
-	if a.isStopped() {
-		return network.ErrStopped
-	}
-
-	failCount := 0
-	for nodeName, node := range a.nodes {
-		a.log.Debug("Shutting down node %q...", nodeName)
-		if err := a.k8scli.Delete(ctx, node.k8sObjSpec); err != nil {
-			a.log.Error("error while stopping node %s: %s", node.name, err)
-			failCount++
-		}
-		delete(a.nodes, nodeName)
-	}
-	close(a.closedOnStopCh)
-	if failCount > 0 {
-		return fmt.Errorf("%d nodes failed shutting down", failCount)
-	}
-	a.log.Info("Network stopped")
-	return nil
-}
-
-// AddNode starts a new node with the given config and blocks
-// until it is reachable.
-// Assumes [a.nodesLock] isn't held.
-func (a *networkImpl) AddNode(cfg node.Config) (node.Node, error) {
-	a.nodesLock.RLock()
-	isStopped := a.isStopped()
-	a.nodesLock.RUnlock()
-
-	// TODO fix this is race condition:
-	// Stop() can be called after the lock is released above.
-	// If that happens, we'll create the pod inside launchNodes
-	// and then never destroy it.
-	// launchNodes assumes [a.nodesLock] isn't held so we can't just
-	// hold the lock inside this method to fix this race condition.
-	if isStopped {
-		return nil, network.ErrStopped
-	}
-
-	nodeSpec, err := buildK8sObjSpec(a.log, []byte(a.config.Genesis), cfg)
-	if err != nil {
-		return nil, err
-	}
-
-	a.log.Debug("Launching new node %s to network...", cfg.Name)
-	if err := a.launchNodes([]*k8sapi.Avalanchego{nodeSpec}); err != nil {
-		return nil, err
-	}
-
-	a.nodesLock.RLock()
-	defer a.nodesLock.RUnlock()
-	return a.nodes[nodeSpec.Name], nil
-}
-
-// See network.Network
-func (a *networkImpl) RemoveNode(name string) error {
-	a.nodesLock.Lock()
-	defer a.nodesLock.Unlock()
-
-	if a.isStopped() {
-		return network.ErrStopped
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), removeTimeout)
-	defer cancel()
-
-	if node, ok := a.nodes[name]; ok {
-		if err := a.k8scli.Delete(ctx, node.k8sObjSpec); err != nil {
-			return err
-		}
-		a.log.Info("Removed node %q", name)
-		delete(a.nodes, name)
-		return nil
-	}
-	return fmt.Errorf("node %q not found", name)
-}
-
-// GetAllNodes returns all nodes
-func (a *networkImpl) GetAllNodes() (map[string]node.Node, error) {
-	a.nodesLock.RLock()
-	defer a.nodesLock.RUnlock()
-
-	if a.isStopped() {
-		return nil, network.ErrStopped
-	}
-
-	nodesCopy := make(map[string]node.Node, len(a.nodes))
-	for nodeName, node := range a.nodes {
-		nodesCopy[nodeName] = node
-	}
-	return nodesCopy, nil
-}
-
-// See network.Network
-func (a *networkImpl) GetNode(name string) (node.Node, error) {
-	a.nodesLock.RLock()
-	defer a.nodesLock.RUnlock()
-
-	if a.isStopped() {
-		return nil, network.ErrStopped
-	}
-	if n, ok := a.nodes[name]; ok {
-		return n, nil
-	}
-	return nil, fmt.Errorf("node %q not found", name)
-}
-
-// Assumes [a.nodesLock] is held
-func (net *networkImpl) isStopped() bool {
-	select {
-	case <-net.closedOnStopCh:
-		return true
-	default:
-		return false
-	}
-}
-
-// Creates the given nodes and blocks until they're all reachable.
-// Assumes [a.nodesLock] isn't held.
-func (a *networkImpl) launchNodes(nodeSpecs []*k8sapi.Avalanchego) error {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	errGr, ctx := errgroup.WithContext(ctx)
-	for _, nodeSpec := range nodeSpecs {
-		nodeSpec := nodeSpec
-		errGr.Go(func() error {
-			if err := a.launchNode(ctx, nodeSpec); err != nil {
-				return fmt.Errorf("error launching node %q: %w", nodeSpec.Spec.DeploymentName, err)
-			}
-			return nil
-		})
-	}
-	return errGr.Wait()
-}
-
-// Create the given node in k8s and block until it's reachable.
-// Assumes [a.nodesLock] isn't held.
-func (a *networkImpl) launchNode(ctx context.Context, nodeSpec *k8sapi.Avalanchego) error {
-	ctx, cancel := context.WithTimeout(ctx, nodeReachableTimeout)
-	defer cancel()
-
-	a.nodesLock.Lock()
-	if a.beaconURL != "" {
-		nodeSpec.Spec.BootstrapperURL = a.beaconURL
-	}
-	// Update [a.nodes] so that we'll delete this node on stop
-	a.nodes[nodeSpec.Spec.DeploymentName] = &Node{
-		name:       nodeSpec.Spec.DeploymentName,
-		k8sObjSpec: nodeSpec,
-	}
-	// Create a Kubernetes pod for this node
-	err := a.k8scli.Create(ctx, nodeSpec)
-	a.nodesLock.Unlock()
-	if err != nil {
-		return fmt.Errorf("k8scli.Create failed: %w", err)
-	}
-
-	a.log.Debug("Waiting for pod to be created for node %q...", nodeSpec.Spec.DeploymentName)
-	for len(nodeSpec.Status.NetworkMembersURI) != 1 {
-		if err := a.k8scli.Get(ctx, types.NamespacedName{
-			Name:      nodeSpec.Name,
-			Namespace: nodeSpec.Namespace,
-		}, nodeSpec); err != nil {
-			return fmt.Errorf("k8scli.Get failed: %w", err)
-		}
-		select {
-		case <-time.After(nodeReachableCheckFreq):
-		case <-ctx.Done():
-			return ctx.Err()
-		}
-	}
-
-	a.log.Debug("pod created. Waiting to be reachable...")
-	// Try connecting to nodes until the DNS resolves,
-	// otherwise we have to sleep indiscriminately, we can't just use the API right away:
-	// the kubernetes cluster has already created the pod(s) but not the DNS names,
-	// so using the API Client too early results in an error.
-	url := nodeSpec.Status.NetworkMembersURI[0]
-	apiURL := fmt.Sprintf("http://%s:%d", url, defaultAPIPort)
-reachableLoop:
-	for {
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		default:
-			a.log.Debug("checking if %q is reachable at %s...", nodeSpec.Spec.DeploymentName, apiURL)
-			if reachable := a.dnsChecker.Reachable(ctx, apiURL); reachable {
-				a.log.Debug("%q has become reachable", nodeSpec.Spec.DeploymentName)
-				break reachableLoop
-			}
-			// Wait before checking again
-			select {
-			case <-ctx.Done():
-				return ctx.Err()
-			case <-time.After(nodeReachableRetryFreq):
-			}
-		}
-	}
-
-	// Create an API client
-	a.log.Debug("creating network node and client for %s", url)
-	apiClient := a.apiClientFunc(url, defaultAPIPort)
-	// Get this node's ID
-	// TODO should we get this by parsing the key/cert?
-	nodeIDStr, err := apiClient.InfoAPI().GetNodeID(ctx)
-	if err != nil {
-		return fmt.Errorf("couldn't get node ID: %w", err)
-	}
-	nodeID, err := ids.ShortFromPrefixedString(nodeIDStr, constants.NodeIDPrefix)
-	if err != nil {
-		return fmt.Errorf("could not parse node ID %q from string: %s", nodeIDStr, err)
-	}
-	// Update node info
-	a.nodesLock.Lock()
-	a.nodes[nodeSpec.Spec.DeploymentName].uri = url
-	a.nodes[nodeSpec.Spec.DeploymentName].apiClient = apiClient
-	a.nodes[nodeSpec.Spec.DeploymentName].nodeID = nodeID
-	a.nodesLock.Unlock()
-	a.log.Debug("Name: %s, NodeID: %s, URI: %s", nodeSpec.Spec.DeploymentName, nodeID, url)
-	return nil
-}
-
-// String returns a string representing the network nodes
-func (a *networkImpl) String() string {
-	a.nodesLock.RLock()
-	defer a.nodesLock.RUnlock()
-
-	s := strings.Builder{}
-	_, _ = s.WriteString("\n****************************************************************************************************\n")
-	_, _ = s.WriteString("     List of nodes in the network: \n")
-	_, _ = s.WriteString("  +------------------------------------------------------------------------------------------------+\n")
-	_, _ = s.WriteString("  +  NodeID                           |     Label         |      Cluster URI                       +\n")
-	_, _ = s.WriteString("  +------------------------------------------------------------------------------------------------+\n")
-	for _, n := range a.nodes {
-		s.WriteString(fmt.Sprintf("     %s    %s    %s\n", n.nodeID, n.name, n.uri))
-	}
-	s.WriteString("****************************************************************************************************\n")
-	return s.String()
-}

k8s/network_test.go

@@ -1,507 +0,0 @@
-package k8s
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/ava-labs/avalanche-network-runner/api"
-	apimocks "github.com/ava-labs/avalanche-network-runner/api/mocks"
-	"github.com/ava-labs/avalanche-network-runner/k8s/mocks"
-	"github.com/ava-labs/avalanche-network-runner/local"
-	"github.com/ava-labs/avalanche-network-runner/network"
-	"github.com/ava-labs/avalanche-network-runner/network/node"
-	"github.com/ava-labs/avalanche-network-runner/utils"
-	"github.com/ava-labs/avalanchego/api/health"
-	healthmocks "github.com/ava-labs/avalanchego/api/health/mocks"
-	infomocks "github.com/ava-labs/avalanchego/api/info/mocks"
-	"github.com/ava-labs/avalanchego/ids"
-	"github.com/ava-labs/avalanchego/staking"
-	"github.com/ava-labs/avalanchego/utils/constants"
-	"github.com/ava-labs/avalanchego/utils/logging"
-
-	k8sapi "github.com/ava-labs/avalanchego-operator/api/v1alpha1"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/mock"
-	k8scli "sigs.k8s.io/controller-runtime/pkg/client"
-)
-
-const (
-	defaultTestNetworkID   = uint32(1337)
-	defaultTestNetworkSize = 5
-)
-
-var (
-	_                  api.NewAPIClientF = newMockAPISuccessful
-	_                  api.NewAPIClientF = newMockAPIUnhealthy
-	defaultTestGenesis []byte            = []byte(
-		`{
-			"networkID": 1337,
-			"allocations": [
-			  {
-				"ethAddr": "0xb3d82b1367d362de99ab59a658165aff520cbd4d",
-				"avaxAddr": "X-local1g65uqn6t77p656w64023nh8nd9updzmxyymev2",
-				"initialAmount": 0,
-				"unlockSchedule": [
-				  {
-					"amount": 10000000000000000,
-					"locktime": 1633824000
-				  }
-				]
-			  },
-			  {
-				"ethAddr": "0xb3d82b1367d362de99ab59a658165aff520cbd4d",
-				"avaxAddr": "X-custom18jma8ppw3nhx5r4ap8clazz0dps7rv5u9xde7p",
-				"initialAmount": 300000000000000000,
-				"unlockSchedule": [
-				  {
-					"amount": 20000000000000000
-				  },
-				  {
-					"amount": 10000000000000000,
-					"locktime": 1633824000
-				  }
-				]
-			  },
-			  {
-				"ethAddr": "0xb3d82b1367d362de99ab59a658165aff520cbd4d",
-				"avaxAddr": "X-custom1ur873jhz9qnaqv5qthk5sn3e8nj3e0kmzpjrhp",
-				"initialAmount": 10000000000000000,
-				"unlockSchedule": [
-				  {
-					"amount": 10000000000000000,
-					"locktime": 1633824000
-				  }
-				]
-			  }
-			],
-			"startTime": 1630987200,
-			"initialStakeDuration": 31536000,
-			"initialStakeDurationOffset": 5400,
-			"initialStakedFunds": [
-			  "X-custom1g65uqn6t77p656w64023nh8nd9updzmxwd59gh"
-			],
-			"initialStakers": [
-			  {
-				"nodeID": "NodeID-7Xhw2mDxuDS44j42TCB6U5579esbSt3Lg",
-				"rewardAddress": "X-custom18jma8ppw3nhx5r4ap8clazz0dps7rv5u9xde7p",
-				"delegationFee": 1000000
-			  },
-			  {
-				"nodeID": "NodeID-MFrZFVCXPv5iCn6M9K6XduxGTYp891xXZ",
-				"rewardAddress": "X-custom18jma8ppw3nhx5r4ap8clazz0dps7rv5u9xde7p",
-				"delegationFee": 500000
-			  },
-			  {
-				"nodeID": "NodeID-NFBbbJ4qCmNaCzeW7sxErhvWqvEQMnYcN",
-				"rewardAddress": "X-custom18jma8ppw3nhx5r4ap8clazz0dps7rv5u9xde7p",
-				"delegationFee": 250000
-			  },
-			  {
-				"nodeID": "NodeID-GWPcbFJZFfZreETSoWjPimr846mXEKCtu",
-				"rewardAddress": "X-custom18jma8ppw3nhx5r4ap8clazz0dps7rv5u9xde7p",
-				"delegationFee": 125000
-			  },
-			  {
-				"nodeID": "NodeID-P7oB2McjBGgW2NXXWVYjV8JEDFoW9xDE5",
-				"rewardAddress": "X-custom18jma8ppw3nhx5r4ap8clazz0dps7rv5u9xde7p",
-				"delegationFee": 62500
-			  }
-			],
-			"cChainGenesis": "{\"config\":{\"chainId\":43112,\"homesteadBlock\":0,\"daoForkBlock\":0,\"daoForkSupport\":true,\"eip150Block\":0,\"eip150Hash\":\"0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0\",\"eip155Block\":0,\"eip158Block\":0,\"byzantiumBlock\":0,\"constantinopleBlock\":0,\"petersburgBlock\":0,\"istanbulBlock\":0,\"muirGlacierBlock\":0,\"apricotPhase1BlockTimestamp\":0,\"apricotPhase2BlockTimestamp\":0},\"nonce\":\"0x0\",\"timestamp\":\"0x0\",\"extraData\":\"0x00\",\"gasLimit\":\"0x5f5e100\",\"difficulty\":\"0x0\",\"mixHash\":\"0x0000000000000000000000000000000000000000000000000000000000000000\",\"coinbase\":\"0x0000000000000000000000000000000000000000\",\"alloc\":{\"8db97C7cEcE249c2b98bDC0226Cc4C2A57BF52FC\":{\"balance\":\"0x295BE96E64066972000000\"}},\"number\":\"0x0\",\"gasUsed\":\"0x0\",\"parentHash\":\"0x0000000000000000000000000000000000000000000000000000000000000000\"}",
-			"message": "{{ fun_quote }}"
-		  }
-		  `,
-	)
-)
-
-// newMockK8sClient creates a new mock client
-func newMockK8sClient() k8scli.Client {
-	client := &mocks.Client{}
-	client.On("Get", mock.Anything, mock.Anything, mock.Anything).Run(
-		func(args mock.Arguments) {
-			arg := args.Get(2).(*k8sapi.Avalanchego)
-			arg.Status.NetworkMembersURI = []string{"localhost"}
-		}).Return(nil)
-	client.On("Delete", mock.Anything, mock.Anything, mock.Anything).Return(nil)
-	client.On("DeleteAllOf", mock.Anything, mock.Anything, mock.Anything).Return(nil)
-	client.On("Create", mock.Anything, mock.Anything).Return(nil)
-	client.On("Status").Return(nil)
-	client.On("Scheme").Return(nil)
-	client.On("RESTMapper").Return(nil)
-	return client
-}
-
-// newDNSChecker creates a mock for checking the DNS (really just a http.Get mock)
-func newDNSChecker() *mocks.DnsReachableChecker {
-	dnsChecker := &mocks.DnsReachableChecker{}
-	dnsChecker.On("Reachable", mock.Anything, mock.AnythingOfType("string")).Return(true)
-	return dnsChecker
-}
-
-// Returns an API client where:
-// * The Health API's Health method always returns healthy
-// * The CChainEthAPI's Close method may be called
-// * Only the above 2 methods may be called
-// TODO have this method return an API Client that has all
-// APIs and methods implemented
-func newMockAPISuccessful(ipAddr string, port uint16) api.Client {
-	healthReply := &health.APIHealthReply{Healthy: true}
-	healthClient := &healthmocks.Client{}
-	healthClient.On("Health", mock.Anything).Return(healthReply, nil)
-
-	id := ids.GenerateTestShortID().String()
-	infoReply := fmt.Sprintf("%s%s", constants.NodeIDPrefix, id)
-	infoClient := &infomocks.Client{}
-	infoClient.On("GetNodeID", mock.Anything).Return(infoReply, nil)
-
-	client := &apimocks.Client{}
-	client.On("HealthAPI").Return(healthClient)
-	client.On("InfoAPI").Return(infoClient)
-	return client
-}
-
-// Returns an API client where the Health API's Health method always returns unhealthy
-func newMockAPIUnhealthy(ipAddr string, port uint16) api.Client {
-	healthReply := &health.APIHealthReply{Healthy: false}
-	healthClient := &healthmocks.Client{}
-	healthClient.On("Health", mock.Anything).Return(healthReply, nil)
-	client := &apimocks.Client{}
-	client.On("HealthAPI").Return(healthClient)
-	return client
-}
-
-func newDefaultTestNetwork(t *testing.T) (network.Network, error) {
-	conf := defaultTestNetworkConfig(t)
-	return newTestNetworkWithConfig(conf)
-}
-
-func newTestNetworkWithConfig(conf network.Config) (network.Network, error) {
-	return newNetwork(networkParams{
-		conf:          conf,
-		log:           logging.NoLog{},
-		k8sClient:     newMockK8sClient(),
-		dnsChecker:    newDNSChecker(),
-		apiClientFunc: newMockAPISuccessful,
-	})
-}
-
-// cleanup closes the channel to shutdown the HTTP server
-func cleanup(n network.Network) {
-	ctx, cancel := context.WithTimeout(context.Background(), stopTimeout)
-	defer cancel()
-	if err := n.Stop(ctx); err != nil {
-		fmt.Printf("Error stopping network: %s\n", err)
-	}
-}
-
-// TestNewNetworkEmpty tests that an empty config results in an error
-func TestNewNetworkEmpty(t *testing.T) {
-	t.Parallel()
-	conf := network.Config{}
-	_, err := newTestNetworkWithConfig(conf)
-	assert.Error(t, err)
-}
-
-// TestHealthy tests that a default network can be created and becomes healthy
-func TestHealthy(t *testing.T) {
-	t.Parallel()
-	n, err := newDefaultTestNetwork(t)
-	assert.NoError(t, err)
-	defer cleanup(n)
-	err = awaitNetworkHealthy(n, 30*time.Second)
-	assert.NoError(t, err)
-}
-
-// TestNetworkDefault tests the default operations on a network:
-// * it creates a network and waits until it's healthy
-// * it adds a new node
-// * it gets a single node
-// * it gets all nodes
-// * it removes a node
-// * it stops the network
-func TestNetworkDefault(t *testing.T) {
-	t.Parallel()
-	assert := assert.New(t)
-	conf := defaultTestNetworkConfig(t)
-	n, err := newTestNetworkWithConfig(conf)
-	assert.NoError(err)
-	defer cleanup(n)
-	err = awaitNetworkHealthy(n, 30*time.Second)
-	assert.NoError(err)
-
-	net, ok := n.(*networkImpl)
-	assert.True(ok)
-	assert.Len(net.nodes, len(conf.NodeConfigs))
-	for _, node := range net.nodes {
-		assert.NotNil(node.apiClient)
-		assert.NotNil(node.k8sObjSpec)
-		assert.True(len(node.name) > 0)
-		assert.True(len(node.uri) > 0)
-		assert.NotEqualValues(ids.ShortEmpty, node.nodeID)
-	}
-
-	names, err := n.GetNodeNames()
-	assert.NoError(err)
-	netSize := len(names)
-	assert.EqualValues(defaultTestNetworkSize, netSize)
-	for _, name := range names {
-		assert.Greater(len(name), 0)
-	}
-	stakingCert, stakingKey, err := staking.NewCertAndKeyBytes()
-	assert.NoError(err)
-
-	newNodeConfig := node.Config{
-		Name:        "new-node",
-		IsBeacon:    false,
-		StakingKey:  string(stakingKey),
-		StakingCert: string(stakingCert),
-		ImplSpecificConfig: utils.NewK8sNodeConfigJsonRaw(
-			"chain.avax.network/v1alpha1",
-			"new-node",
-			"avaplatform/avalanchego",
-			"Avalanchego",
-			"ci-network-runner",
-			"9.99.9999",
-		),
-	}
-	newNode, err := n.AddNode(newNodeConfig)
-	assert.NoError(err)
-	names, err = n.GetNodeNames()
-	assert.NoError(err)
-	assert.Len(names, netSize+1)
-
-	nn, err := n.GetNode(newNodeConfig.Name)
-	assert.NoError(err)
-	assert.Equal(newNodeConfig.Name, nn.GetName())
-
-	_, err = n.GetNode("this does not exist")
-	assert.Error(err)
-
-	err = n.RemoveNode(newNode.GetName())
-	assert.NoError(err)
-	names, err = n.GetNodeNames()
-	assert.NoError(err)
-	assert.Len(names, netSize)
-}
-
-// TestWrongNetworkConfigs checks configs that are expected to be invalid at network creation time
-// This is adapted from the local test suite
-func TestWrongNetworkConfigs(t *testing.T) {
-	t.Parallel()
-	tests := map[string]struct {
-		config network.Config
-	}{
-		"no ImplSpecificConfig": {
-			config: network.Config{
-				Genesis: "nonempty",
-				NodeConfigs: []node.Config{
-					{
-						IsBeacon:    true,
-						StakingKey:  "nonempty",
-						StakingCert: "nonempty",
-					},
-				},
-			},
-		},
-		"empty name": {
-			config: network.Config{
-				Genesis: "nonempty",
-				NodeConfigs: []node.Config{
-					{
-						ImplSpecificConfig: utils.NewK8sNodeConfigJsonRaw("0.00.0000", "testnode", "somerepo/someimage", "anykind", "noname", "testingversion"),
-						IsBeacon:           true,
-						StakingKey:         "nonempty",
-						StakingCert:        "nonempty",
-					},
-				},
-			},
-		},
-		"StakingKey but no StakingCert": {
-			config: network.Config{
-				Genesis: "nonempty",
-				NodeConfigs: []node.Config{
-					{
-						ImplSpecificConfig: utils.NewK8sNodeConfigJsonRaw("0.00.0000", "testnode", "somerepo/someimage", "anykind", "noname", "testingversion"),
-						IsBeacon:           true,
-						StakingKey:         "nonempty",
-					},
-				},
-			},
-		},
-		"StakingCert but no StakingKey": {
-			config: network.Config{
-				Genesis: "nonempty",
-				NodeConfigs: []node.Config{
-					{
-						ImplSpecificConfig: utils.NewK8sNodeConfigJsonRaw("0.00.0000", "testnode", "somerepo/someimage", "anykind", "noname", "testingversion"),
-						IsBeacon:           true,
-						StakingCert:        "nonempty",
-					},
-				},
-			},
-		},
-		"no beacon node": {
-			config: network.Config{
-				Genesis: "nonempty",
-				NodeConfigs: []node.Config{
-					{
-						ImplSpecificConfig: utils.NewK8sNodeConfigJsonRaw("0.00.0000", "testnode", "somerepo/someimage", "anykind", "noname", "testingversion"),
-						StakingKey:         "nonempty",
-						StakingCert:        "nonempty",
-					},
-				},
-			},
-		},
-		"repeated name": {
-			config: network.Config{
-				Genesis: "nonempty",
-				NodeConfigs: []node.Config{
-					{
-						ImplSpecificConfig: utils.NewK8sNodeConfigJsonRaw("0.00.0000", "testnode", "somerepo/someimage", "anykind", "noname", "testingversion"),
-						Name:               "node0",
-						IsBeacon:           true,
-						StakingKey:         "nonempty",
-						StakingCert:        "nonempty",
-					},
-					{
-						ImplSpecificConfig: utils.NewK8sNodeConfigJsonRaw("0.00.0000", "testnode", "somerepo/someimage", "anykind", "noname", "testingversion"),
-						Name:               "node0",
-						IsBeacon:           false,
-						StakingKey:         "nonempty",
-						StakingCert:        "nonempty",
-					},
-				},
-			},
-		},
-	}
-	assert := assert.New(t)
-	for name, tt := range tests {
-		t.Run(name, func(t *testing.T) {
-			_, err := newTestNetworkWithConfig(tt.config)
-			assert.Error(err)
-		})
-	}
-}
-
-// TestFlags tests that we can pass flags through the network.Config
-// but also via node.Config and that the latter overrides the former
-// if same keys exist.
-func TestFlags(t *testing.T) {
-	t.Parallel()
-	assert := assert.New(t)
-	networkConfig := defaultTestNetworkConfig(t)
-
-	networkConfig.Flags = map[string]interface{}{
-		"test-network-config-flag": "something",
-		"common-config-flag":       "should not be added",
-	}
-	for i := range networkConfig.NodeConfigs {
-		v := &networkConfig.NodeConfigs[i]
-		v.Flags = map[string]interface{}{
-			"test-node-config-flag":  "node",
-			"test2-node-config-flag": "config",
-			"common-config-flag":     "this should be added",
-		}
-	}
-	nw, err := newTestNetworkWithConfig(networkConfig)
-	assert.NoError(err)
-	// after creating the network, one flag should have been overridden by the node configs
-	for _, n := range networkConfig.NodeConfigs {
-		assert.Len(n.Flags, 4)
-		assert.Contains(n.Flags, "test-network-config-flag")
-		assert.Equal(n.Flags["test-network-config-flag"], "something")
-		assert.Contains(n.Flags, "common-config-flag")
-		assert.Equal(n.Flags["common-config-flag"], "this should be added")
-		assert.Contains(n.Flags, "test-node-config-flag")
-		assert.Equal(n.Flags["test-node-config-flag"], "node")
-		assert.Contains(n.Flags, "test2-node-config-flag")
-		assert.Equal(n.Flags["test2-node-config-flag"], "config")
-	}
-	err = nw.Stop(context.Background())
-	assert.NoError(err)
-
-	// submit only node.Config flags
-	networkConfig.Flags = nil
-	for i := range networkConfig.NodeConfigs {
-		v := &networkConfig.NodeConfigs[i]
-		v.Flags = map[string]interface{}{
-			"test-node-config-flag":  "node",
-			"test2-node-config-flag": "config",
-			"common-config-flag":     "this should be added",
-		}
-	}
-	nw, err = newTestNetworkWithConfig(networkConfig)
-	assert.NoError(err)
-	// after creating the network, only node configs should exist
-	for _, n := range networkConfig.NodeConfigs {
-		assert.Len(n.Flags, 3)
-		assert.NotContains(n.Flags, "test-network-config-flag")
-		assert.Contains(n.Flags, "common-config-flag")
-		assert.Equal(n.Flags["common-config-flag"], "this should be added")
-		assert.Contains(n.Flags, "test-node-config-flag")
-		assert.Equal(n.Flags["test-node-config-flag"], "node")
-		assert.Contains(n.Flags, "test2-node-config-flag")
-		assert.Equal(n.Flags["test2-node-config-flag"], "config")
-	}
-	err = nw.Stop(context.Background())
-	assert.NoError(err)
-
-	// submit only network.Config flags
-	networkConfig.Flags = map[string]interface{}{
-		"test-network-config-flag": "something",
-		"common-config-flag":       "else",
-	}
-	for i := range networkConfig.NodeConfigs {
-		v := &networkConfig.NodeConfigs[i]
-		v.Flags = nil
-	}
-	nw, err = newTestNetworkWithConfig(networkConfig)
-	assert.NoError(err)
-	// after creating the network, only flags from the network config should exist
-	for _, n := range networkConfig.NodeConfigs {
-		assert.True(len(n.Flags) == 2)
-		assert.Contains(n.Flags, "test-network-config-flag")
-		assert.Equal(n.Flags["test-network-config-flag"], "something")
-		assert.Contains(n.Flags, "common-config-flag")
-		assert.Equal(n.Flags["common-config-flag"], "else")
-	}
-	err = nw.Stop(context.Background())
-	assert.NoError(err)
-}
-
-// TestImplSpecificConfigInterface checks incorrect type to interface{} ImplSpecificConfig
-// This is adapted from the local test suite
-func TestImplSpecificConfigInterface(t *testing.T) {
-	t.Parallel()
-	assert := assert.New(t)
-	networkConfig := defaultTestNetworkConfig(t)
-	networkConfig.NodeConfigs[0].ImplSpecificConfig = json.RawMessage("should be a JSON")
-	_, err := newTestNetworkWithConfig(networkConfig)
-	assert.Error(err)
-}
-
-// defaultTestNetworkConfig creates a default size network for testing
-func defaultTestNetworkConfig(t *testing.T) network.Config {
-	assert := assert.New(t)
-	networkConfig, err := local.NewDefaultConfigNNodes("pepito", defaultTestNetworkSize)
-	assert.NoError(err)
-	for i := 0; i < defaultTestNetworkSize; i++ {
-		networkConfig.NodeConfigs[i].Name = fmt.Sprintf("node%d", i)
-		networkConfig.NodeConfigs[i].ImplSpecificConfig = utils.NewK8sNodeConfigJsonRaw("0.00.0000", fmt.Sprintf("testnode-%d", i), "somerepo/someimage", "Avalanchego", "ci-networkrunner", "testingversion")
-	}
-	return networkConfig
-}
-
-// Returns nil when all the nodes in [net] are healthy,
-// or an error if one doesn't become healthy within
-// the timeout.
-func awaitNetworkHealthy(net network.Network, timeout time.Duration) error {
-	ctx, cancel := context.WithTimeout(context.Background(), timeout)
-	defer cancel()
-	healthyCh := net.Healthy(ctx)
-	return <-healthyCh
-}

k8s/node.go

@@ -1,83 +0,0 @@
-package k8s
-
-import (
-	"context"
-	"errors"
-
-	"github.com/ava-labs/avalanche-network-runner/api"
-	"github.com/ava-labs/avalanche-network-runner/network/node"
-	k8sapi "github.com/ava-labs/avalanchego-operator/api/v1alpha1"
-	"github.com/ava-labs/avalanchego/ids"
-	"github.com/ava-labs/avalanchego/network/peer"
-	"github.com/ava-labs/avalanchego/snow/networking/router"
-)
-
-var _ node.Node = &Node{}
-
-// ObjectSpec is the K8s-specifc object config. This is the "implementation-specic config"
-// for the K8s-based network runner. See struct Config in network/node/node.go.
-type ObjectSpec struct {
-	Namespace  string `json:"namespace"`  // The kubernetes Namespace
-	Identifier string `json:"identifier"` // Identifies this network in the cluster
-	Kind       string `json:"kind"`       // Identifies the object Kind for the operator
-	APIVersion string `json:"apiVersion"` // The APIVersion of the kubernetes object
-	Image      string `json:"image"`      // The docker image to use
-	Tag        string `json:"tag"`        // The docker tag to use
-}
-
-// Node is a Avalanchego representation on k8s
-type Node struct {
-	// This node's AvalancheGo node ID
-	nodeID ids.ShortID
-	// Unique name of this node
-	name string
-	// URI of this node from Kubernetes
-	uri string
-	// Use to send API calls to this node
-	apiClient api.Client
-	// K8s description of this node
-	k8sObjSpec *k8sapi.Avalanchego
-}
-
-// AttachPeer see Network
-// TODO: Not yet implemented
-func (n *Node) AttachPeer(ctx context.Context, handler router.InboundHandler) (peer.Peer, error) {
-	return nil, errors.New("AttachPeer is not implemented")
-}
-
-// See node.Node
-func (n *Node) GetAPIClient() api.Client {
-	return n.apiClient
-}
-
-// See node.Node
-func (n *Node) GetName() string {
-	return n.name
-}
-
-// See node.Node
-func (n *Node) GetNodeID() ids.ShortID {
-	return n.nodeID
-}
-
-// See node.Node
-func (n *Node) GetURL() string {
-	return n.uri
-}
-
-// See node.Node
-func (n *Node) GetP2PPort() uint16 {
-	// TODO don't hard-code this
-	return defaultP2PPort
-}
-
-func (n *Node) GetAPIPort() uint16 {
-	// TODO don't hard-code this
-	return defaultAPIPort
-}
-
-// GetK8sObjSpec returns the kubernetes object spec
-// representation of this node
-func (n *Node) GetK8sObjSpec() *k8sapi.Avalanchego {
-	return n.k8sObjSpec
-}

k8s/reachable_checker.go

@@ -1,49 +0,0 @@
-package k8s
-
-import (
-	"context"
-	"net/http"
-	"time"
-)
-
-const (
-	consecutiveReachableSuccess = 5
-	reachableCheckFreq          = time.Second
-)
-
-var _ dnsReachableChecker = &defaultDNSReachableChecker{}
-
-// dnsReachableChecker checks if a URL is reachable.
-// We use this to check whether a K8s pod is reachable.
-// We need this as long as we use plain HTTP Get calls
-// to check for DNS working (there is a TODO to change this).
-type dnsReachableChecker interface {
-	// Returns true if we can successfully
-	// make an HTTP call to [url]
-	Reachable(ctx context.Context, url string) bool
-}
-
-// defaultDNSReachableChecker is the default implementation of the DNS check.
-// It just uses http.Get.
-type defaultDNSReachableChecker struct{}
-
-// It seems that sometimes, we are able to send a GET request to [url]
-// but then an immediately subsequent HTTP request to [url] fails.
-// This method only returns true if [consecutiveReachableSuccess]
-// consecutive GET requests to [url], with [reachableCheckFreq]
-// between each check, succeed, in order to ensure that [url] is stably reachable.
-func (d *defaultDNSReachableChecker) Reachable(ctx context.Context, url string) bool {
-	for i := 0; i < consecutiveReachableSuccess; i++ {
-		if resp, err := http.Get(url); err != nil {
-			return false
-		} else {
-			_ = resp.Body.Close()
-		}
-		select {
-		case <-ctx.Done():
-			return false
-		case <-time.After(reachableCheckFreq): // Wait
-		}
-	}
-	return true
-}

k8s/utils.go

@@ -1,206 +0,0 @@
-package k8s
-
-import (
-	"encoding/base64"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"strings"
-
-	"github.com/ava-labs/avalanche-network-runner/network/node"
-	"github.com/ava-labs/avalanche-network-runner/utils"
-	k8sapi "github.com/ava-labs/avalanchego-operator/api/v1alpha1"
-	"github.com/ava-labs/avalanchego/config"
-	"github.com/ava-labs/avalanchego/utils/logging"
-	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/resource"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-// Convert a config flag to the format AvalancheGo expects
-// environment variable config flags in.
-// e.g. bootstrap-ips --> AVAGO_BOOTSTRAP_IPS
-// e.g. log-level --> AVAGO_LOG_LEVEL
-func convertKey(key string) string {
-	key = strings.Replace(key, "-", "_", -1)
-	key = strings.ToUpper(key)
-	newKey := fmt.Sprintf("%s%s", envVarPrefix, key)
-	return newKey
-}
-
-// Given a node's config and genesis, returns the environment
-// variables (i.e. config flags) to give to the node
-func buildNodeEnv(log logging.Logger, genesis []byte, c node.Config) ([]corev1.EnvVar, error) {
-	conf := map[string]interface{}{}
-
-	// Parse config from file if one given
-	if c.ConfigFile != "" {
-		if err := json.Unmarshal([]byte(c.ConfigFile), &conf); err != nil {
-			return nil, err
-		}
-	}
-
-	// If a key is in the config file and flags, the flag takes precedence.
-	for key, val := range c.Flags {
-		if _, ok := conf[key]; ok {
-			log.Info("overwriting key %s in config file with value given in flag", key)
-		}
-		conf[key] = val
-	}
-
-	// Parse network ID from genesis
-	networkID, err := utils.NetworkIDFromGenesis(genesis)
-	if err != nil {
-		return nil, err
-	}
-
-	// Make sure network ID in config file / flag, if given,
-	// matches genesis network ID
-	if gotNetworkID, ok := conf[config.NetworkNameKey]; ok {
-		if gotNetworkID, ok := gotNetworkID.(float64); ok && uint32(gotNetworkID) != networkID {
-			return nil, fmt.Errorf(
-				"network ID in config file / flag (%d) != network ID in genesis (%d)",
-				uint32(gotNetworkID), networkID,
-			)
-		}
-	}
-
-	// AvalancheGo expects environment variable config keys in.
-	// e.g. bootstrap-ips --> AVAGO_BOOTSTRAP_IPS
-	// e.g. log-level --> AVAGO_LOG_LEVEL
-	env := []corev1.EnvVar{
-		{
-			// Provide environment variable giving the network ID
-			Name:  convertKey(config.NetworkNameKey),
-			Value: fmt.Sprint(networkID),
-		},
-	}
-	for key, val := range conf {
-		// For each config key, convert it to the right format
-		env = append(env, corev1.EnvVar{
-			Name:  convertKey(key),
-			Value: fmt.Sprintf("%v", val),
-		})
-	}
-	return env, nil
-}
-
-// Takes a node's config and genesis and returns the node as a k8s object spec
-func buildK8sObjSpec(log logging.Logger, genesis []byte, c node.Config) (*k8sapi.Avalanchego, error) {
-	env, err := buildNodeEnv(log, genesis, c)
-	if err != nil {
-		return nil, err
-	}
-	certs := []k8sapi.Certificate{
-		{
-			Cert: base64.StdEncoding.EncodeToString([]byte(c.StakingCert)),
-			Key:  base64.StdEncoding.EncodeToString([]byte(c.StakingKey)),
-		},
-	}
-	var k8sConf ObjectSpec
-	if err := json.Unmarshal(c.ImplSpecificConfig, &k8sConf); err != nil {
-		return nil, fmt.Errorf("Unmarshalling an expected k8s.ObjectSpec failed: %w", err)
-	}
-
-	if err := validateObjectSpec(k8sConf); err != nil {
-		return nil, err
-	}
-
-	return &k8sapi.Avalanchego{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       k8sConf.Kind,
-			APIVersion: k8sConf.APIVersion,
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      k8sConf.Identifier,
-			Namespace: k8sConf.Namespace,
-		},
-		Spec: k8sapi.AvalanchegoSpec{
-			BootstrapperURL: "",
-			DeploymentName:  k8sConf.Identifier,
-			Image:           k8sConf.Image,
-			Tag:             k8sConf.Tag,
-			Env:             env,
-			NodeCount:       1,
-			Certificates:    certs,
-			Genesis:         string(genesis),
-			Resources: corev1.ResourceRequirements{
-				Limits: corev1.ResourceList{
-					corev1.ResourceCPU:    resource.MustParse(resourceLimitsCPU), // TODO: Should these be supplied by Opts rather than const?
-					corev1.ResourceMemory: resource.MustParse(resourceLimitsMemory),
-				},
-				Requests: corev1.ResourceList{
-					corev1.ResourceCPU:    resource.MustParse(resourceRequestCPU),
-					corev1.ResourceMemory: resource.MustParse(resourceRequestMemory),
-				},
-			},
-		},
-	}, nil
-}
-
-// Validates an ObjectSpec.
-// The tag value can be empty so not checked.
-func validateObjectSpec(k8sobj ObjectSpec) error {
-	switch {
-	case k8sobj.Identifier == "":
-		return errors.New("name should not be empty")
-	case k8sobj.APIVersion == "":
-		return errors.New("APIVersion should not be empty")
-	case k8sobj.Kind != "Avalanchego":
-		// only "AvalancheGo" currently supported -- mandated by avalanchego-operator
-		return fmt.Errorf("expected \"Avalanchego\" but got %q", k8sobj.Kind)
-	case k8sobj.Namespace == "":
-		return errors.New("namespace should be defined to avoid unintended consequences")
-	case k8sobj.Image == "" || strings.Index(k8sobj.Image, "/") == 1:
-		return fmt.Errorf("image string %q is invalid, it can't be empty and must contain a %q to describe a valid image repo", k8sobj.Image, "/")
-	default:
-		return nil
-	}
-}
-
-// Takes the genesis of a network and node configs and returns:
-// 1) The beacon nodes
-// 2) The non-beacon nodes
-// as avalanchego-operator compatible descriptions.
-// May return nil slices.
-func createDeploymentFromConfig(params networkParams) ([]*k8sapi.Avalanchego, []*k8sapi.Avalanchego, error) {
-	// Give each flag in the network config to each node's config.
-	// If a flag is defined in both the network config and the node config,
-	// the value given in the node config takes precedence.
-	for flagName, flagVal := range params.conf.Flags {
-		for i := range params.conf.NodeConfigs {
-			nodeConfig := &params.conf.NodeConfigs[i]
-			if len(nodeConfig.Flags) == 0 {
-				nodeConfig.Flags = make(map[string]interface{})
-			}
-			// Do not overwrite flags described in the nodeConfig
-			if val, ok := nodeConfig.Flags[flagName]; !ok {
-				nodeConfig.Flags[flagName] = flagVal
-			} else {
-				params.log.Info(
-					"not overwriting node config flag %s (value %v) with network config flag (value %v)",
-					flagName, val, flagVal,
-				)
-			}
-		}
-	}
-
-	var beacons, nonBeacons []*k8sapi.Avalanchego
-	names := make(map[string]struct{})
-	for _, nodeConfig := range params.conf.NodeConfigs {
-		spec, err := buildK8sObjSpec(params.log, []byte(params.conf.Genesis), nodeConfig)
-		if err != nil {
-			return nil, nil, err
-		}
-		if _, exists := names[spec.Name]; exists {
-			return nil, nil, fmt.Errorf("node with name name %q already exists", spec.Name)
-		}
-		names[spec.Name] = struct{}{}
-		if nodeConfig.IsBeacon {
-			beacons = append(beacons, spec)
-		} else {
-			nonBeacons = append(nonBeacons, spec)
-		}
-	}
-	return beacons, nonBeacons, nil
-}

k8s/utils_test.go

@@ -1,146 +0,0 @@
-package k8s
-
-import (
-	"encoding/base64"
-	"fmt"
-	"testing"
-
-	"github.com/ava-labs/avalanche-network-runner/network"
-	"github.com/ava-labs/avalanche-network-runner/network/node"
-	"github.com/ava-labs/avalanche-network-runner/utils"
-	"github.com/ava-labs/avalanchego/utils/logging"
-	"github.com/stretchr/testify/assert"
-	v1 "k8s.io/api/core/v1"
-)
-
-// TestBuildNodeEnv tests the internal buildNodeEnv method which creates the env vars for the avalanche nodes
-func TestBuildNodeEnv(t *testing.T) {
-	t.Parallel()
-	genesis := defaultTestGenesis
-	testConfig := `
-	{
-		"network-peer-list-gossip-frequency": "250ms",
-		"network-max-reconnect-delay": "1s",
-		"health-check-frequency": "2s"
-	}`
-	c := node.Config{
-		ConfigFile: testConfig,
-	}
-
-	envVars, err := buildNodeEnv(logging.NoLog{}, genesis, c)
-	assert.NoError(t, err)
-	controlVars := []v1.EnvVar{
-		{
-			Name:  "AVAGO_NETWORK_PEER_LIST_GOSSIP_FREQUENCY",
-			Value: "250ms",
-		},
-		{
-			Name:  "AVAGO_NETWORK_MAX_RECONNECT_DELAY",
-			Value: "1s",
-		},
-		{
-			Name:  "AVAGO_HEALTH_CHECK_FREQUENCY",
-			Value: "2s",
-		},
-		{
-			Name:  "AVAGO_NETWORK_ID",
-			Value: fmt.Sprint(defaultTestNetworkID),
-		},
-	}
-
-	assert.ElementsMatch(t, envVars, controlVars)
-}
-
-// TestConvertKey tests the internal convertKey method which is used
-// to convert from the avalanchego config file format to env vars
-func TestConvertKey(t *testing.T) {
-	t.Parallel()
-	testKey := "network-peer-list-gossip-frequency"
-	controlKey := "AVAGO_NETWORK_PEER_LIST_GOSSIP_FREQUENCY"
-	convertedKey := convertKey(testKey)
-	assert.Equal(t, convertedKey, controlKey)
-}
-
-// TestCreateDeploymentConfig tests the internal createDeploymentFromConfig method which creates the k8s objects
-func TestCreateDeploymentConfig(t *testing.T) {
-	t.Parallel()
-	assert := assert.New(t)
-	genesis := defaultTestGenesis
-
-	nodeConfigs := []node.Config{
-		{
-			Name:        "test1",
-			IsBeacon:    true,
-			StakingKey:  "fooKey",
-			StakingCert: "fooCert",
-			ConfigFile:  "{}",
-			ImplSpecificConfig: utils.NewK8sNodeConfigJsonRaw(
-				"v1",
-				"test11",
-				"img1",
-				"Avalanchego",
-				"test01",
-				"t1",
-			),
-		},
-		{
-			Name:        "test2",
-			IsBeacon:    false,
-			StakingKey:  "barKey",
-			StakingCert: "barCert",
-			ConfigFile:  "{}",
-			ImplSpecificConfig: utils.NewK8sNodeConfigJsonRaw(
-				"v2",
-				"test22",
-				"img2",
-				"Avalanchego",
-				"test02",
-				"t2",
-			),
-		},
-	}
-	params := networkParams{
-		conf: network.Config{
-			Genesis:     string(genesis),
-			NodeConfigs: nodeConfigs,
-		},
-	}
-
-	beacons, nonBeacons, err := createDeploymentFromConfig(params)
-	assert.NoError(err)
-	assert.Len(beacons, 1)
-	assert.Len(nonBeacons, 1)
-
-	b := beacons[0]
-	n := nonBeacons[0]
-
-	assert.Equal(b.Name, "test11")
-	assert.Equal(n.Name, "test22")
-	assert.Equal(b.Kind, "Avalanchego")
-	assert.Equal(n.Kind, "Avalanchego")
-	assert.Equal(b.APIVersion, "v1")
-	assert.Equal(n.APIVersion, "v2")
-	assert.Equal(b.Namespace, "test01")
-	assert.Equal(n.Namespace, "test02")
-	assert.Equal(b.Spec.DeploymentName, "test11")
-	assert.Equal(n.Spec.DeploymentName, "test22")
-	assert.Equal(b.Spec.Image, "img1")
-	assert.Equal(n.Spec.Image, "img2")
-	assert.Equal(b.Spec.Tag, "t1")
-	assert.Equal(n.Spec.Tag, "t2")
-	assert.Equal(b.Spec.BootstrapperURL, "")
-	assert.Equal(n.Spec.BootstrapperURL, "")
-	assert.Equal(b.Spec.Env[0].Name, "AVAGO_NETWORK_ID")
-	assert.Equal(n.Spec.Env[0].Name, "AVAGO_NETWORK_ID")
-	assert.Equal(b.Spec.Env[0].Value, fmt.Sprint(defaultTestNetworkID))
-	assert.Equal(n.Spec.Env[0].Value, fmt.Sprint(defaultTestNetworkID))
-	assert.Equal(b.Spec.NodeCount, 1)
-	assert.Equal(n.Spec.NodeCount, 1)
-	assert.Equal(b.Spec.Certificates[0].Cert, base64.StdEncoding.EncodeToString([]byte("fooCert")))
-	assert.Equal(b.Spec.Certificates[0].Key, base64.StdEncoding.EncodeToString([]byte("fooKey")))
-	assert.Equal(n.Spec.Certificates[0].Cert, base64.StdEncoding.EncodeToString([]byte("barCert")))
-	assert.Equal(n.Spec.Certificates[0].Key, base64.StdEncoding.EncodeToString([]byte("barKey")))
-	assert.Equal(n.Spec.NodeCount, 1)
-	assert.Equal(b.Spec.Genesis, string(genesis))
-	assert.Equal(n.Spec.Genesis, string(genesis))
-}

local/network_test.go

@@ -3,7 +3,6 @@ package local
 import (
 	"bytes"
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
@@ -25,8 +24,8 @@ import (
 	"github.com/ava-labs/avalanchego/ids"
 	"github.com/ava-labs/avalanchego/message"
 	"github.com/ava-labs/avalanchego/snow/networking/router"
-	"github.com/ava-labs/avalanchego/utils/constants"
 	"github.com/ava-labs/avalanchego/utils/logging"
+	"github.com/ava-labs/avalanchego/utils/rpc"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 )
@@ -131,12 +130,14 @@ func TestNewNetworkEmpty(t *testing.T) {
 	networkConfig.NodeConfigs = nil
 	net, err := newNetwork(
 		logging.NoLog{},
-		networkConfig,
 		newMockAPISuccessful,
 		&localTestProcessUndefNodeProcessCreator{},
 		"",
+		"",
 	)
 	assert.NoError(err)
+	err = net.loadConfig(context.Background(), networkConfig)
+	assert.NoError(err)
 	// Assert that GetNodeNames() returns an empty list
 	names, err := net.GetNodeNames()
 	assert.NoError(err)
@@ -162,9 +163,9 @@ func newLocalTestOneNodeCreator(assert *assert.Assertions, networkConfig network
 func (lt *localTestOneNodeCreator) NewNodeProcess(config node.Config, flags ...string) (NodeProcess, error) {
 	lt.assert.True(config.IsBeacon)
 	expectedConfig := lt.networkConfig.NodeConfigs[0]
-	lt.assert.EqualValues(expectedConfig.CChainConfigFile, config.CChainConfigFile)
+	lt.assert.EqualValues(expectedConfig.ChainConfigFiles, config.ChainConfigFiles)
 	lt.assert.EqualValues(expectedConfig.ConfigFile, config.ConfigFile)
-	lt.assert.EqualValues(expectedConfig.ImplSpecificConfig, config.ImplSpecificConfig)
+	lt.assert.EqualValues(expectedConfig.BinaryPath, config.BinaryPath)
 	lt.assert.EqualValues(expectedConfig.IsBeacon, config.IsBeacon)
 	lt.assert.EqualValues(expectedConfig.Name, config.Name)
 	lt.assert.EqualValues(expectedConfig.StakingCert, config.StakingCert)
@@ -187,12 +188,14 @@ func TestNewNetworkOneNode(t *testing.T) {
 	creator := newLocalTestOneNodeCreator(assert, networkConfig)
 	net, err := newNetwork(
 		logging.NoLog{},
-		networkConfig,
 		newMockAPISuccessful,
 		creator,
 		"",
+		"",
 	)
 	assert.NoError(err)
+	err = net.loadConfig(context.Background(), networkConfig)
+	assert.NoError(err)
 
 	// Assert that GetNodeNames() includes only the 1 node's name
 	names, err := net.GetNodeNames()
@@ -201,7 +204,7 @@ func TestNewNetworkOneNode(t *testing.T) {
 	assert.Len(names, 1)
 
 	// Assert that the network's genesis was set
-	assert.EqualValues(networkConfig.Genesis, net.(*localNetwork).genesis)
+	assert.EqualValues(networkConfig.Genesis, net.genesis)
 }
 
 // Test that NewNetwork returns an error when
@@ -210,13 +213,15 @@ func TestNewNetworkFailToStartNode(t *testing.T) {
 	t.Parallel()
 	assert := assert.New(t)
 	networkConfig := testNetworkConfig(t)
-	_, err := newNetwork(
+	net, err := newNetwork(
 		logging.NoLog{},
-		networkConfig,
 		newMockAPISuccessful,
 		&localTestFailedStartProcessCreator{},
 		"",
+		"",
 	)
+	assert.NoError(err)
+	err = net.loadConfig(context.Background(), networkConfig)
 	assert.Error(err)
 }
 
@@ -227,28 +232,16 @@ func TestWrongNetworkConfigs(t *testing.T) {
 	tests := map[string]struct {
 		config network.Config
 	}{
-		"no ImplSpecificConfig": {
-			config: network.Config{
-				Genesis: "{\"networkID\": 0}",
-				NodeConfigs: []node.Config{
-					{
-						IsBeacon:    true,
-						StakingKey:  refNetworkConfig.NodeConfigs[0].StakingKey,
-						StakingCert: refNetworkConfig.NodeConfigs[0].StakingCert,
-					},
-				},
-			},
-		},
 		"config file unmarshal": {
 			config: network.Config{
 				Genesis: "{\"networkID\": 0}",
 				NodeConfigs: []node.Config{
 					{
-						ImplSpecificConfig: utils.NewLocalNodeConfigJsonRaw("pepe"),
-						IsBeacon:           true,
-						StakingKey:         refNetworkConfig.NodeConfigs[0].StakingKey,
-						StakingCert:        refNetworkConfig.NodeConfigs[0].StakingCert,
-						ConfigFile:         "nonempty",
+						BinaryPath:  "pepe",
+						IsBeacon:    true,
+						StakingKey:  refNetworkConfig.NodeConfigs[0].StakingKey,
+						StakingCert: refNetworkConfig.NodeConfigs[0].StakingCert,
+						ConfigFile:  "nonempty",
 					},
 				},
 			},
@@ -258,11 +251,11 @@ func TestWrongNetworkConfigs(t *testing.T) {
 				Genesis: "{\"networkID\": 0}",
 				NodeConfigs: []node.Config{
 					{
-						ImplSpecificConfig: utils.NewLocalNodeConfigJsonRaw("pepe"),
-						IsBeacon:           true,
-						StakingKey:         refNetworkConfig.NodeConfigs[0].StakingKey,
-						StakingCert:        refNetworkConfig.NodeConfigs[0].StakingCert,
-						ConfigFile:         "{\"network-id\": \"0\"}",
+						BinaryPath:  "pepe",
+						IsBeacon:    true,
+						StakingKey:  refNetworkConfig.NodeConfigs[0].StakingKey,
+						StakingCert: refNetworkConfig.NodeConfigs[0].StakingCert,
+						ConfigFile:  "{\"network-id\": \"0\"}",
 					},
 				},
 			},
@@ -272,11 +265,11 @@ func TestWrongNetworkConfigs(t *testing.T) {
 				Genesis: "{\"networkID\": 0}",
 				NodeConfigs: []node.Config{
 					{
-						ImplSpecificConfig: utils.NewLocalNodeConfigJsonRaw("pepe"),
-						IsBeacon:           true,
-						StakingKey:         refNetworkConfig.NodeConfigs[0].StakingKey,
-						StakingCert:        refNetworkConfig.NodeConfigs[0].StakingCert,
-						ConfigFile:         "{\"db-dir\": 0}",
+						BinaryPath:  "pepe",
+						IsBeacon:    true,
+						StakingKey:  refNetworkConfig.NodeConfigs[0].StakingKey,
+						StakingCert: refNetworkConfig.NodeConfigs[0].StakingCert,
+						ConfigFile:  "{\"db-dir\": 0}",
 					},
 				},
 			},
@@ -286,11 +279,11 @@ func TestWrongNetworkConfigs(t *testing.T) {
 				Genesis: "{\"networkID\": 0}",
 				NodeConfigs: []node.Config{
 					{
-						ImplSpecificConfig: utils.NewLocalNodeConfigJsonRaw("pepe"),
-						IsBeacon:           true,
-						StakingKey:         refNetworkConfig.NodeConfigs[0].StakingKey,
-						StakingCert:        refNetworkConfig.NodeConfigs[0].StakingCert,
-						ConfigFile:         "{\"log-dir\": 0}",
+						BinaryPath:  "pepe",
+						IsBeacon:    true,
+						StakingKey:  refNetworkConfig.NodeConfigs[0].StakingKey,
+						StakingCert: refNetworkConfig.NodeConfigs[0].StakingCert,
+						ConfigFile:  "{\"log-dir\": 0}",
 					},
 				},
 			},
@@ -300,11 +293,11 @@ func TestWrongNetworkConfigs(t *testing.T) {
 				Genesis: "{\"networkID\": 0}",
 				NodeConfigs: []node.Config{
 					{
-						ImplSpecificConfig: utils.NewLocalNodeConfigJsonRaw("pepe"),
-						IsBeacon:           true,
-						StakingKey:         refNetworkConfig.NodeConfigs[0].StakingKey,
-						StakingCert:        refNetworkConfig.NodeConfigs[0].StakingCert,
-						ConfigFile:         "{\"http-port\": \"0\"}",
+						BinaryPath:  "pepe",
+						IsBeacon:    true,
+						StakingKey:  refNetworkConfig.NodeConfigs[0].StakingKey,
+						StakingCert: refNetworkConfig.NodeConfigs[0].StakingCert,
+						ConfigFile:  "{\"http-port\": \"0\"}",
 					},
 				},
 			},
@@ -314,11 +307,11 @@ func TestWrongNetworkConfigs(t *testing.T) {
 				Genesis: "{\"networkID\": 0}",
 				NodeConfigs: []node.Config{
 					{
-						ImplSpecificConfig: utils.NewLocalNodeConfigJsonRaw("pepe"),
-						IsBeacon:           true,
-						StakingKey:         refNetworkConfig.NodeConfigs[0].StakingKey,
-						StakingCert:        refNetworkConfig.NodeConfigs[0].StakingCert,
-						ConfigFile:         "{\"staking-port\": \"0\"}",
+						BinaryPath:  "pepe",
+						IsBeacon:    true,
+						StakingKey:  refNetworkConfig.NodeConfigs[0].StakingKey,
+						StakingCert: refNetworkConfig.NodeConfigs[0].StakingCert,
+						ConfigFile:  "{\"staking-port\": \"0\"}",
 					},
 				},
 			},
@@ -328,11 +321,11 @@ func TestWrongNetworkConfigs(t *testing.T) {
 				Genesis: "{\"networkID\": 0}",
 				NodeConfigs: []node.Config{
 					{
-						ImplSpecificConfig: utils.NewLocalNodeConfigJsonRaw("pepe"),
-						IsBeacon:           true,
-						StakingKey:         refNetworkConfig.NodeConfigs[0].StakingKey,
-						StakingCert:        refNetworkConfig.NodeConfigs[0].StakingCert,
-						ConfigFile:         "{\"network-id\": 1}",
+						BinaryPath:  "pepe",
+						IsBeacon:    true,
+						StakingKey:  refNetworkConfig.NodeConfigs[0].StakingKey,
+						StakingCert: refNetworkConfig.NodeConfigs[0].StakingCert,
+						ConfigFile:  "{\"network-id\": 1}",
 					},
 				},
 			},
@@ -342,10 +335,10 @@ func TestWrongNetworkConfigs(t *testing.T) {
 				Genesis: "nonempty",
 				NodeConfigs: []node.Config{
 					{
-						ImplSpecificConfig: utils.NewLocalNodeConfigJsonRaw("pepe"),
-						IsBeacon:           true,
-						StakingKey:         refNetworkConfig.NodeConfigs[0].StakingKey,
-						StakingCert:        refNetworkConfig.NodeConfigs[0].StakingCert,
+						BinaryPath:  "pepe",
+						IsBeacon:    true,
+						StakingKey:  refNetworkConfig.NodeConfigs[0].StakingKey,
+						StakingCert: refNetworkConfig.NodeConfigs[0].StakingCert,
 					},
 				},
 			},
@@ -355,10 +348,10 @@ func TestWrongNetworkConfigs(t *testing.T) {
 				Genesis: "{}",
 				NodeConfigs: []node.Config{
 					{
-						ImplSpecificConfig: utils.NewLocalNodeConfigJsonRaw("pepe"),
-						IsBeacon:           true,
-						StakingKey:         refNetworkConfig.NodeConfigs[0].StakingKey,
-						StakingCert:        refNetworkConfig.NodeConfigs[0].StakingCert,
+						BinaryPath:  "pepe",
+						IsBeacon:    true,
+						StakingKey:  refNetworkConfig.NodeConfigs[0].StakingKey,
+						StakingCert: refNetworkConfig.NodeConfigs[0].StakingCert,
 					},
 				},
 			},
@@ -368,10 +361,10 @@ func TestWrongNetworkConfigs(t *testing.T) {
 				Genesis: "{\"networkID\": \"0\"}",
 				NodeConfigs: []node.Config{
 					{
-						ImplSpecificConfig: utils.NewLocalNodeConfigJsonRaw("pepe"),
-						IsBeacon:           true,
-						StakingKey:         refNetworkConfig.NodeConfigs[0].StakingKey,
-						StakingCert:        refNetworkConfig.NodeConfigs[0].StakingCert,
+						BinaryPath:  "pepe",
+						IsBeacon:    true,
+						StakingKey:  refNetworkConfig.NodeConfigs[0].StakingKey,
+						StakingCert: refNetworkConfig.NodeConfigs[0].StakingCert,
 					},
 				},
 			},
@@ -380,10 +373,10 @@ func TestWrongNetworkConfigs(t *testing.T) {
 			config: network.Config{
 				NodeConfigs: []node.Config{
 					{
-						ImplSpecificConfig: utils.NewLocalNodeConfigJsonRaw("pepe"),
-						IsBeacon:           true,
-						StakingKey:         refNetworkConfig.NodeConfigs[0].StakingKey,
-						StakingCert:        refNetworkConfig.NodeConfigs[0].StakingCert,
+						BinaryPath:  "pepe",
+						IsBeacon:    true,
+						StakingKey:  refNetworkConfig.NodeConfigs[0].StakingKey,
+						StakingCert: refNetworkConfig.NodeConfigs[0].StakingCert,
 					},
 				},
 			},
@@ -393,9 +386,9 @@ func TestWrongNetworkConfigs(t *testing.T) {
 				Genesis: "{\"networkID\": 0}",
 				NodeConfigs: []node.Config{
 					{
-						ImplSpecificConfig: utils.NewLocalNodeConfigJsonRaw("pepe"),
-						IsBeacon:           true,
-						StakingKey:         refNetworkConfig.NodeConfigs[0].StakingKey,
+						BinaryPath: "pepe",
+						IsBeacon:   true,
+						StakingKey: refNetworkConfig.NodeConfigs[0].StakingKey,
 					},
 				},
 			},
@@ -405,9 +398,9 @@ func TestWrongNetworkConfigs(t *testing.T) {
 				Genesis: "{\"networkID\": 0}",
 				NodeConfigs: []node.Config{
 					{
-						ImplSpecificConfig: utils.NewLocalNodeConfigJsonRaw("pepe"),
-						IsBeacon:           true,
-						StakingCert:        refNetworkConfig.NodeConfigs[0].StakingCert,
+						BinaryPath:  "pepe",
+						IsBeacon:    true,
+						StakingCert: refNetworkConfig.NodeConfigs[0].StakingCert,
 					},
 				},
 			},
@@ -417,10 +410,10 @@ func TestWrongNetworkConfigs(t *testing.T) {
 				Genesis: "{\"networkID\": 0}",
 				NodeConfigs: []node.Config{
 					{
-						ImplSpecificConfig: utils.NewLocalNodeConfigJsonRaw("pepe"),
-						IsBeacon:           true,
-						StakingKey:         "nonempty",
-						StakingCert:        "nonempty",
+						BinaryPath:  "pepe",
+						IsBeacon:    true,
+						StakingKey:  "nonempty",
+						StakingCert: "nonempty",
 					},
 				},
 			},
@@ -430,9 +423,9 @@ func TestWrongNetworkConfigs(t *testing.T) {
 				Genesis: "{\"networkID\": 0}",
 				NodeConfigs: []node.Config{
 					{
-						ImplSpecificConfig: utils.NewLocalNodeConfigJsonRaw("pepe"),
-						StakingKey:         refNetworkConfig.NodeConfigs[0].StakingKey,
-						StakingCert:        refNetworkConfig.NodeConfigs[0].StakingCert,
+						BinaryPath:  "pepe",
+						StakingKey:  refNetworkConfig.NodeConfigs[0].StakingKey,
+						StakingCert: refNetworkConfig.NodeConfigs[0].StakingCert,
 					},
 				},
 			},
@@ -442,18 +435,18 @@ func TestWrongNetworkConfigs(t *testing.T) {
 				Genesis: "{\"networkID\": 0}",
 				NodeConfigs: []node.Config{
 					{
-						Name:               "node0",
-						ImplSpecificConfig: utils.NewLocalNodeConfigJsonRaw("pepe"),
-						IsBeacon:           true,
-						StakingKey:         refNetworkConfig.NodeConfigs[0].StakingKey,
-						StakingCert:        refNetworkConfig.NodeConfigs[0].StakingCert,
+						Name:        "node1",
+						BinaryPath:  "pepe",
+						IsBeacon:    true,
+						StakingKey:  refNetworkConfig.NodeConfigs[0].StakingKey,
+						StakingCert: refNetworkConfig.NodeConfigs[0].StakingCert,
 					},
 					{
-						Name:               "node0",
-						ImplSpecificConfig: utils.NewLocalNodeConfigJsonRaw("pepe"),
-						IsBeacon:           true,
-						StakingKey:         refNetworkConfig.NodeConfigs[1].StakingKey,
-						StakingCert:        refNetworkConfig.NodeConfigs[1].StakingCert,
+						Name:        "node1",
+						BinaryPath:  "pepe",
+						IsBeacon:    true,
+						StakingKey:  refNetworkConfig.NodeConfigs[1].StakingKey,
+						StakingCert: refNetworkConfig.NodeConfigs[1].StakingCert,
 					},
 				},
 			},
@@ -462,29 +455,23 @@ func TestWrongNetworkConfigs(t *testing.T) {
 	assert := assert.New(t)
 	for name, tt := range tests {
 		t.Run(name, func(t *testing.T) {
-			_, err := newNetwork(logging.NoLog{}, tt.config, newMockAPISuccessful, &localTestSuccessfulNodeProcessCreator{}, "")
+			net, err := newNetwork(logging.NoLog{}, newMockAPISuccessful, &localTestSuccessfulNodeProcessCreator{}, "", "")
+			assert.NoError(err)
+			err = net.loadConfig(context.Background(), tt.config)
 			assert.Error(err)
 		})
 	}
 }
 
-// Give incorrect type to interface{} ImplSpecificConfig
-func TestInvalidImplSpecificConfig(t *testing.T) {
-	t.Parallel()
-	assert := assert.New(t)
-	networkConfig := testNetworkConfig(t)
-	networkConfig.NodeConfigs[0].ImplSpecificConfig = json.RawMessage("just a string")
-	_, err := newNetwork(logging.NoLog{}, networkConfig, newMockAPISuccessful, &localTestSuccessfulNodeProcessCreator{}, "")
-	assert.Error(err)
-}
-
 // Assert that the network's Healthy() method returns an
 // error when all nodes' Health API return unhealthy
 func TestUnhealthyNetwork(t *testing.T) {
 	t.Parallel()
 	assert := assert.New(t)
 	networkConfig := testNetworkConfig(t)
-	net, err := newNetwork(logging.NoLog{}, networkConfig, newMockAPIUnhealthy, &localTestSuccessfulNodeProcessCreator{}, "")
+	net, err := newNetwork(logging.NoLog{}, newMockAPIUnhealthy, &localTestSuccessfulNodeProcessCreator{}, "", "")
+	assert.NoError(err)
+	err = net.loadConfig(context.Background(), networkConfig)
 	assert.NoError(err)
 	assert.Error(awaitNetworkHealthy(net, defaultHealthyTimeout))
 }
@@ -498,7 +485,9 @@ func TestGeneratedNodesNames(t *testing.T) {
 	for i := range networkConfig.NodeConfigs {
 		networkConfig.NodeConfigs[i].Name = ""
 	}
-	net, err := newNetwork(logging.NoLog{}, networkConfig, newMockAPISuccessful, &localTestSuccessfulNodeProcessCreator{}, "")
+	net, err := newNetwork(logging.NoLog{}, newMockAPISuccessful, &localTestSuccessfulNodeProcessCreator{}, "", "")
+	assert.NoError(err)
+	err = net.loadConfig(context.Background(), networkConfig)
 	assert.NoError(err)
 	nodeNameMap := make(map[string]bool)
 	nodeNames, err := net.GetNodeNames()
@@ -509,13 +498,16 @@ func TestGeneratedNodesNames(t *testing.T) {
 	assert.EqualValues(len(nodeNameMap), len(networkConfig.NodeConfigs))
 }
 
-// TestGenerateDefaultNetwork create a default network with GenerateDefaultNetwork and
+// TestGenerateDefaultNetwork create a default network with config from NewDefaultConfig and
 // check expected number of nodes, node names, and avalanchego node ids
 func TestGenerateDefaultNetwork(t *testing.T) {
 	t.Parallel()
 	assert := assert.New(t)
 	binaryPath := "pepito"
-	net, err := newDefaultNetwork(logging.NoLog{}, binaryPath, newMockAPISuccessful, &localTestSuccessfulNodeProcessCreator{})
+	networkConfig := NewDefaultConfig(binaryPath)
+	net, err := newNetwork(logging.NoLog{}, newMockAPISuccessful, &localTestSuccessfulNodeProcessCreator{}, "", "")
+	assert.NoError(err)
+	err = net.loadConfig(context.Background(), networkConfig)
 	assert.NoError(err)
 	assert.NoError(awaitNetworkHealthy(net, defaultHealthyTimeout))
 	names, err := net.GetNodeNames()
@@ -526,23 +518,23 @@ func TestGenerateDefaultNetwork(t *testing.T) {
 		ID   string
 	}{
 		{
-			"node-0",
+			"node1",
 			"NodeID-7Xhw2mDxuDS44j42TCB6U5579esbSt3Lg",
 		},
 		{
-			"node-1",
+			"node2",
 			"NodeID-MFrZFVCXPv5iCn6M9K6XduxGTYp891xXZ",
 		},
 		{
-			"node-2",
+			"node3",
 			"NodeID-NFBbbJ4qCmNaCzeW7sxErhvWqvEQMnYcN",
 		},
 		{
-			"node-3",
+			"node4",
 			"NodeID-GWPcbFJZFfZreETSoWjPimr846mXEKCtu",
 		},
 		{
-			"node-4",
+			"node5",
 			"NodeID-P7oB2McjBGgW2NXXWVYjV8JEDFoW9xDE5",
 		},
 	} {
@@ -550,7 +542,7 @@ func TestGenerateDefaultNetwork(t *testing.T) {
 		node, err := net.GetNode(nodeInfo.name)
 		assert.NoError(err)
 		assert.EqualValues(nodeInfo.name, node.GetName())
-		expectedID, err := ids.ShortFromPrefixedString(nodeInfo.ID, constants.NodeIDPrefix)
+		expectedID, err := ids.NodeIDFromString(nodeInfo.ID)
 		assert.NoError(err)
 		assert.EqualValues(expectedID, node.GetNodeID())
 	}
@@ -563,7 +555,9 @@ func TestNetworkFromConfig(t *testing.T) {
 	t.Parallel()
 	assert := assert.New(t)
 	networkConfig := testNetworkConfig(t)
-	net, err := newNetwork(logging.NoLog{}, networkConfig, newMockAPISuccessful, &localTestSuccessfulNodeProcessCreator{}, "")
+	net, err := newNetwork(logging.NoLog{}, newMockAPISuccessful, &localTestSuccessfulNodeProcessCreator{}, "", "")
+	assert.NoError(err)
+	err = net.loadConfig(context.Background(), networkConfig)
 	assert.NoError(err)
 	assert.NoError(awaitNetworkHealthy(net, defaultHealthyTimeout))
 	runningNodes := make(map[string]struct{})
@@ -585,7 +579,9 @@ func TestNetworkNodeOps(t *testing.T) {
 	// Start a new, empty network
 	emptyNetworkConfig, err := emptyNetworkConfig()
 	assert.NoError(err)
-	net, err := newNetwork(logging.NoLog{}, emptyNetworkConfig, newMockAPISuccessful, &localTestSuccessfulNodeProcessCreator{}, "")
+	net, err := newNetwork(logging.NoLog{}, newMockAPISuccessful, &localTestSuccessfulNodeProcessCreator{}, "", "")
+	assert.NoError(err)
+	err = net.loadConfig(context.Background(), emptyNetworkConfig)
 	assert.NoError(err)
 	runningNodes := make(map[string]struct{})
 
@@ -621,7 +617,9 @@ func TestNodeNotFound(t *testing.T) {
 	emptyNetworkConfig, err := emptyNetworkConfig()
 	assert.NoError(err)
 	networkConfig := testNetworkConfig(t)
-	net, err := newNetwork(logging.NoLog{}, emptyNetworkConfig, newMockAPISuccessful, &localTestSuccessfulNodeProcessCreator{}, "")
+	net, err := newNetwork(logging.NoLog{}, newMockAPISuccessful, &localTestSuccessfulNodeProcessCreator{}, "", "")
+	assert.NoError(err)
+	err = net.loadConfig(context.Background(), emptyNetworkConfig)
 	assert.NoError(err)
 	_, err = net.AddNode(networkConfig.NodeConfigs[0])
 	assert.NoError(err)
@@ -652,7 +650,9 @@ func TestStoppedNetwork(t *testing.T) {
 	emptyNetworkConfig, err := emptyNetworkConfig()
 	assert.NoError(err)
 	networkConfig := testNetworkConfig(t)
-	net, err := newNetwork(logging.NoLog{}, emptyNetworkConfig, newMockAPISuccessful, &localTestSuccessfulNodeProcessCreator{}, "")
+	net, err := newNetwork(logging.NoLog{}, newMockAPISuccessful, &localTestSuccessfulNodeProcessCreator{}, "", "")
+	assert.NoError(err)
+	err = net.loadConfig(context.Background(), emptyNetworkConfig)
 	assert.NoError(err)
 	_, err = net.AddNode(networkConfig.NodeConfigs[0])
 	assert.NoError(err)
@@ -665,32 +665,34 @@ func TestStoppedNetwork(t *testing.T) {
 	assert.EqualValues(net.Stop(context.Background()), network.ErrStopped)
 	// AddNode failure
 	_, err = net.AddNode(networkConfig.NodeConfigs[1])
-	assert.EqualValues(err, network.ErrStopped)
+	assert.EqualValues(network.ErrStopped, err)
 	// GetNode failure
 	_, err = net.GetNode(networkConfig.NodeConfigs[0].Name)
 	assert.EqualValues(err, network.ErrStopped)
 	// second GetNodeNames should return no nodes
 	_, err = net.GetNodeNames()
-	assert.EqualValues(err, network.ErrStopped)
+	assert.EqualValues(network.ErrStopped, err)
 	// RemoveNode failure
-	assert.EqualValues(net.RemoveNode(networkConfig.NodeConfigs[0].Name), network.ErrStopped)
+	assert.EqualValues(network.ErrStopped, net.RemoveNode(networkConfig.NodeConfigs[0].Name))
 	// Healthy failure
 	assert.EqualValues(awaitNetworkHealthy(net, defaultHealthyTimeout), network.ErrStopped)
 	_, err = net.GetAllNodes()
-	assert.EqualValues(network.ErrStopped, err)
+	assert.EqualValues(err, network.ErrStopped)
 }
 
 func TestGetAllNodes(t *testing.T) {
 	t.Parallel()
 	assert := assert.New(t)
 	networkConfig := testNetworkConfig(t)
-	net, err := newNetwork(logging.NoLog{}, networkConfig, newMockAPISuccessful, &localTestSuccessfulNodeProcessCreator{}, "")
+	net, err := newNetwork(logging.NoLog{}, newMockAPISuccessful, &localTestSuccessfulNodeProcessCreator{}, "", "")
+	assert.NoError(err)
+	err = net.loadConfig(context.Background(), networkConfig)
 	assert.NoError(err)
 
 	nodes, err := net.GetAllNodes()
 	assert.NoError(err)
-	assert.Len(nodes, len(net.(*localNetwork).nodes))
-	for name, node := range net.(*localNetwork).nodes {
+	assert.Len(nodes, len(net.nodes))
+	for name, node := range net.nodes {
 		assert.EqualValues(node, nodes[name])
 	}
 }
@@ -716,7 +718,7 @@ func TestFlags(t *testing.T) {
 			"common-config-flag":     "this should be added",
 		}
 	}
-	nw, err := newNetwork(logging.NoLog{}, networkConfig, newMockAPISuccessful, &localTestFlagCheckProcessCreator{
+	nw, err := newNetwork(logging.NoLog{}, newMockAPISuccessful, &localTestFlagCheckProcessCreator{
 		// after creating the network, one flag should have been overridden by the node configs
 		expectedFlags: map[string]interface{}{
 			"test-network-config-flag": "something",
@@ -727,7 +729,10 @@ func TestFlags(t *testing.T) {
 		assert: assert,
 	},
 		"",
+		"",
 	)
+	assert.NoError(err)
+	err = nw.loadConfig(context.Background(), networkConfig)
 	if ok := assert.NoError(err); !ok {
 		t.Fatal("assertion failed")
 	}
@@ -745,13 +750,16 @@ func TestFlags(t *testing.T) {
 		v := &networkConfig.NodeConfigs[i]
 		v.Flags = flags
 	}
-	nw, err = newNetwork(logging.NoLog{}, networkConfig, newMockAPISuccessful, &localTestFlagCheckProcessCreator{
+	nw, err = newNetwork(logging.NoLog{}, newMockAPISuccessful, &localTestFlagCheckProcessCreator{
 		// after creating the network, only node configs should exist
 		expectedFlags: flags,
 		assert:        assert,
 	},
 		"",
+		"",
 	)
+	assert.NoError(err)
+	err = nw.loadConfig(context.Background(), networkConfig)
 	if ok := assert.NoError(err); !ok {
 		t.Fatal("assertion failed")
 	}
@@ -768,14 +776,17 @@ func TestFlags(t *testing.T) {
 		v := &networkConfig.NodeConfigs[i]
 		v.Flags = nil
 	}
-	nw, err = newNetwork(logging.NoLog{}, networkConfig, newMockAPISuccessful, &localTestFlagCheckProcessCreator{
+	nw, err = newNetwork(logging.NoLog{}, newMockAPISuccessful, &localTestFlagCheckProcessCreator{
 		// after creating the network, only flags from the network config should exist
 		expectedFlags: flags,
 		assert:        assert,
 	},
 		"",
+		"",
 	)
 	assert.NoError(err)
+	err = nw.loadConfig(context.Background(), networkConfig)
+	assert.NoError(err)
 	err = nw.Stop(context.Background())
 	assert.NoError(err)
 }
@@ -825,8 +836,10 @@ func TestChildCmdRedirection(t *testing.T) {
 
 	// now create the node process and check it will be prepended and colored
 	testConfig := node.Config{
-		ImplSpecificConfig: json.RawMessage(`{"binaryPath":"echo","redirectStdout":true,"redirectStderr":true}`),
-		Name:               mockNodeName,
+		BinaryPath:     "echo",
+		RedirectStdout: true,
+		RedirectStderr: true,
+		Name:           mockNodeName,
 	}
 	proc, err := npc.NewNodeProcess(testConfig, testOutput)
 	if err != nil {
@@ -896,15 +909,13 @@ func emptyNetworkConfig() (network.Config, error) {
 			},
 		},
 		nil,
-		[]ids.ShortID{ids.GenerateTestShortID()},
+		[]ids.NodeID{ids.GenerateTestNodeID()},
 	)
 	if err != nil {
 		return network.Config{}, err
 	}
 	return network.Config{
-		LogLevel: "DEBUG",
-		Name:     "My Network",
-		Genesis:  string(genesis),
+		Genesis: string(genesis),
 	}, nil
 }
 
@@ -927,8 +938,7 @@ func testNetworkConfig(t *testing.T) network.Config {
 func awaitNetworkHealthy(net network.Network, timeout time.Duration) error {
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
-	healthyCh := net.Healthy(ctx)
-	return <-healthyCh
+	return net.Healthy(ctx)
 }
 
 func TestAddNetworkFlags(t *testing.T) {
@@ -979,20 +989,21 @@ func TestSetNodeName(t *testing.T) {
 	assert := assert.New(t)
 
 	ln := &localNetwork{
-		nodes: make(map[string]*localNode),
+		nodes:          make(map[string]*localNode),
+		nextNodeSuffix: 1,
 	}
 
 	// Case: No name given
 	config := &node.Config{Name: ""}
 	err := ln.setNodeName(config)
 	assert.NoError(err)
-	assert.Equal("node-0", config.Name)
+	assert.Equal("node1", config.Name)
 
 	// Case: No name given again
 	config.Name = ""
 	err = ln.setNodeName(config)
 	assert.NoError(err)
-	assert.Equal("node-1", config.Name)
+	assert.Equal("node2", config.Name)
 
 	// Case: name given
 	config.Name = "hi"
@@ -1004,7 +1015,7 @@ func TestSetNodeName(t *testing.T) {
 	config.Name = ""
 	err = ln.setNodeName(config)
 	assert.NoError(err)
-	assert.Equal("node-2", config.Name)
+	assert.Equal("node3", config.Name)
 
 	// Case: name already present
 	config.Name = "hi"
@@ -1019,6 +1030,7 @@ func TestGetConfigEntry(t *testing.T) {
 
 	// case: key not present
 	val, err := getConfigEntry(
+		map[string]interface{}{},
 		map[string]interface{}{"2": "2"},
 		"1",
 		"1",
@@ -1028,6 +1040,7 @@ func TestGetConfigEntry(t *testing.T) {
 
 	// case: key present
 	val, err = getConfigEntry(
+		map[string]interface{}{},
 		map[string]interface{}{"1": "hi", "2": "2"},
 		"1",
 		"1",
@@ -1037,6 +1050,7 @@ func TestGetConfigEntry(t *testing.T) {
 
 	// case: key present wrong type
 	_, err = getConfigEntry(
+		map[string]interface{}{},
 		map[string]interface{}{"1": 1, "2": "2"},
 		"1",
 		"1",
@@ -1104,7 +1118,9 @@ func TestWriteFiles(t *testing.T) {
 	stakingCert := "stakingCert"
 	genesis := []byte("genesis")
 	configFile := "config file"
-	cChainConfigFile := "c-chain config file"
+	chainConfigFiles := map[string]string{
+		"C": "c-chain config file",
+	}
 	tmpDir, err := os.MkdirTemp("", "avalanche-network-runner-tests-*")
 	if err != nil {
 		t.Fatal(err)
@@ -1168,7 +1184,7 @@ func TestWriteFiles(t *testing.T) {
 				StakingKey:       stakingKey,
 				StakingCert:      stakingCert,
 				ConfigFile:       configFile,
-				CChainConfigFile: cChainConfigFile,
+				ChainConfigFiles: chainConfigFiles,
 			},
 			expectedFlags: []string{
 				stakingKeyFlag,
@@ -1206,10 +1222,10 @@ func TestWriteFiles(t *testing.T) {
 				assert.NoError(err)
 				assert.Equal([]byte(configFile), gotConfigFile)
 			}
-			if len(tt.nodeConfig.CChainConfigFile) > 0 {
+			if tt.nodeConfig.ChainConfigFiles != nil {
 				gotCChainConfigFile, err := os.ReadFile(cChainConfigPath)
 				assert.NoError(err)
-				assert.Equal([]byte(cChainConfigFile), gotCChainConfigFile)
+				assert.Equal([]byte(chainConfigFiles["C"]), gotCChainConfigFile)
 			}
 		})
 	}
@@ -1222,9 +1238,10 @@ func TestRemoveBeacon(t *testing.T) {
 	// create a network with no nodes in it
 	emptyNetworkConfig, err := emptyNetworkConfig()
 	assert.NoError(err)
-	netIntf, err := newNetwork(logging.NoLog{}, emptyNetworkConfig, newMockAPISuccessful, &localTestSuccessfulNodeProcessCreator{}, "")
+	net, err := newNetwork(logging.NoLog{}, newMockAPISuccessful, &localTestSuccessfulNodeProcessCreator{}, "", "")
+	assert.NoError(err)
+	net.loadConfig(context.Background(), emptyNetworkConfig)
 	assert.NoError(err)
-	net := netIntf.(*localNetwork)
 
 	// a network config for a 3 node staking network, and add the bootstrapper
 	// to the exesting network
@@ -1237,3 +1254,58 @@ func TestRemoveBeacon(t *testing.T) {
 	assert.NoError(err)
 	assert.Equal(0, net.bootstraps.Len())
 }
+
+// Returns an API client where:
+// * The Health API's Health method always returns an error after the
+//   given context is cancelled.
+// * The CChainEthAPI's Close method may be called
+// * Only the above 2 methods may be called
+func newMockAPIHealthyBlocks(ipAddr string, port uint16) api.Client {
+	healthClient := &healthmocks.Client{}
+	healthClient.On("Health", mock.MatchedBy(func(_ context.Context) bool { return true }), mock.Anything).Return(
+		func(ctx context.Context, _ ...rpc.Option) *health.APIHealthReply {
+			<-ctx.Done()
+			return nil
+		},
+		func(ctx context.Context, _ ...rpc.Option) error {
+			<-ctx.Done()
+			return ctx.Err()
+		},
+	)
+	// ethClient used when removing nodes, to close websocket connection
+	ethClient := &apimocks.EthClient{}
+	ethClient.On("Close").Return()
+	client := &apimocks.Client{}
+	client.On("HealthAPI").Return(healthClient)
+	client.On("CChainEthAPI").Return(ethClient)
+	return client
+}
+
+// Assert that if the network's Stop method is called while
+// a call to Healthy is ongoing, Healthy returns immediately.
+func TestHealthyDuringNetworkStop(t *testing.T) {
+	assert := assert.New(t)
+	networkConfig := testNetworkConfig(t)
+	// Calls to a node's Healthy() function blocks until context cancelled
+	net, err := newNetwork(logging.NoLog{}, newMockAPIHealthyBlocks, &localTestSuccessfulNodeProcessCreator{}, "", "")
+	assert.NoError(err)
+	err = net.loadConfig(context.Background(), networkConfig)
+	assert.NoError(err)
+
+	healthyChan := make(chan error)
+	go func() {
+		healthyChan <- net.Healthy(context.Background())
+	}()
+	// Wait to make sure we're actually blocking on Health API call
+	time.Sleep(500 * time.Millisecond)
+	err = net.Stop(context.Background())
+	assert.NoError(err)
+	select {
+	case err := <-healthyChan:
+		assert.Error(err)
+	case <-time.After(1 * time.Second):
+		// Since [net.Stop] was called, [net.Healthy] should immediately return.
+		// We assume that it will do so within 1 second.
+		assert.Fail("Healthy should've returned immediately because network closed")
+	}
+}

local/node.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net"
 	"os/exec"
+	"path/filepath"
 	"syscall"
 	"time"
 
@@ -16,11 +17,14 @@ import (
 	"github.com/ava-labs/avalanchego/network/peer"
 	"github.com/ava-labs/avalanchego/network/throttling"
 	"github.com/ava-labs/avalanchego/snow/networking/router"
+	"github.com/ava-labs/avalanchego/snow/networking/tracker"
 	"github.com/ava-labs/avalanchego/snow/validators"
 	"github.com/ava-labs/avalanchego/staking"
-	avago_utils "github.com/ava-labs/avalanchego/utils"
 	"github.com/ava-labs/avalanchego/utils/constants"
+	"github.com/ava-labs/avalanchego/utils/ips"
 	"github.com/ava-labs/avalanchego/utils/logging"
+	"github.com/ava-labs/avalanchego/utils/math/meter"
+	"github.com/ava-labs/avalanchego/utils/resource"
 	"github.com/ava-labs/avalanchego/version"
 	"github.com/prometheus/client_golang/prometheus"
 )
@@ -34,17 +38,6 @@ var (
 
 type getConnFunc func(context.Context, node.Node) (net.Conn, error)
 
-// NodeConfig configurations which are specific to the
-// local implementation of a network / node.
-type NodeConfig struct {
-	// What type of node this is
-	BinaryPath string `json:"binaryPath"`
-	// If non-nil, direct this node's Stdout to os.Stdout
-	RedirectStdout bool `json:"redirectStdout"`
-	// If non-nil, direct this node's Stderr to os.Stderr
-	RedirectStderr bool `json:"redirectStderr"`
-}
-
 // NodeProcess as an interface so we can mock running
 // AvalancheGo binaries in tests
 type NodeProcess interface {
@@ -56,6 +49,11 @@ type NodeProcess interface {
 	Wait() error
 }
 
+const (
+	peerMsgQueueBufferSize      = 1024
+	peerResourceTrackerDuration = 10 * time.Second
+)
+
 type nodeProcessImpl struct {
 	cmd *exec.Cmd
 }
@@ -78,7 +76,7 @@ type localNode struct {
 	name string
 	// [nodeID] is this node's Avalannche Node ID.
 	// Set in network.AddNode
-	nodeID ids.ShortID
+	nodeID ids.NodeID
 	// The ID of the network this node exists in
 	networkID uint32
 	// Allows user to make API calls to this node.
@@ -91,6 +89,14 @@ type localNode struct {
 	p2pPort uint16
 	// Returns a connection to this node
 	getConnFunc getConnFunc
+	// The db dir of the node
+	dbDir string
+	// The logs dir of the node
+	logsDir string
+	// The build dir of the node
+	buildDir string
+	// The node config
+	config node.Config
 }
 
 func defaultGetConnFunc(ctx context.Context, node node.Node) (net.Conn, error) {
@@ -128,16 +134,24 @@ func (node *localNode) AttachPeer(ctx context.Context, router router.InboundHand
 	if err != nil {
 		return nil, err
 	}
-	ip := avago_utils.IPDesc{
+	ip := ips.IPPort{
 		IP:   net.IPv6zero,
 		Port: 0,
 	}
+	resourceTracker, err := tracker.NewResourceTracker(
+		prometheus.NewRegistry(),
+		resource.NoUsage,
+		meter.ContinuousFactory{},
+		peerResourceTrackerDuration,
+	)
+	if err != nil {
+		return nil, err
+	}
 	config := &peer.Config{
-		Metrics:              metrics,
-		MessageCreator:       mc,
-		Log:                  logging.NoLog{},
-		InboundMsgThrottler:  throttling.NewNoInboundThrottler(),
-		OutboundMsgThrottler: throttling.NewNoOutboundThrottler(),
+		Metrics:             metrics,
+		MessageCreator:      mc,
+		Log:                 logging.NoLog{},
+		InboundMsgThrottler: throttling.NewNoInboundThrottler(),
 		Network: peer.NewTestNetwork(
 			mc,
 			node.networkID,
@@ -149,13 +163,14 @@ func (node *localNode) AttachPeer(ctx context.Context, router router.InboundHand
 		),
 		Router:               router,
 		VersionCompatibility: version.GetCompatibility(node.networkID),
-		VersionParser:        version.NewDefaultApplicationParser(),
+		VersionParser:        version.DefaultApplicationParser,
 		MySubnets:            ids.Set{},
 		Beacons:              validators.NewSet(),
 		NetworkID:            node.networkID,
 		PingFrequency:        constants.DefaultPingFrequency,
 		PongTimeout:          constants.DefaultPingPongTimeout,
 		MaxClockDifference:   time.Minute,
+		ResourceTracker:      resourceTracker,
 	}
 	_, conn, cert, err := clientUpgrader.Upgrade(conn)
 	if err != nil {
@@ -166,7 +181,12 @@ func (node *localNode) AttachPeer(ctx context.Context, router router.InboundHand
 		config,
 		conn,
 		cert,
-		peer.CertToID(tlsCert.Leaf),
+		ids.NodeIDFromCert(tlsCert.Leaf),
+		peer.NewBlockingMessageQueue(
+			config.Metrics,
+			logging.NoLog{},
+			peerMsgQueueBufferSize,
+		),
 	)
 	if err != nil {
 		return nil, err
@@ -181,7 +201,7 @@ func (node *localNode) GetName() string {
 }
 
 // See node.Node
-func (node *localNode) GetNodeID() ids.ShortID {
+func (node *localNode) GetNodeID() ids.NodeID {
 	return node.nodeID
 }
 
@@ -204,3 +224,36 @@ func (node *localNode) GetP2PPort() uint16 {
 func (node *localNode) GetAPIPort() uint16 {
 	return node.apiPort
 }
+
+// See node.Node
+func (node *localNode) GetBinaryPath() string {
+	return node.config.BinaryPath
+}
+
+// See node.Node
+func (node *localNode) GetBuildDir() string {
+	if node.buildDir == "" {
+		return filepath.Dir(node.GetBinaryPath())
+	}
+	return node.buildDir
+}
+
+// See node.Node
+func (node *localNode) GetDbDir() string {
+	return node.dbDir
+}
+
+// See node.Node
+func (node *localNode) GetLogsDir() string {
+	return node.logsDir
+}
+
+// See node.Node
+func (node *localNode) GetConfigFile() string {
+	return node.config.ConfigFile
+}
+
+// See node.Node
+func (node *localNode) GetConfig() node.Config {
+	return node.config
+}

local/node_test.go

@@ -16,16 +16,15 @@ import (
 	"github.com/ava-labs/avalanchego/message"
 	"github.com/ava-labs/avalanchego/network/peer"
 	"github.com/ava-labs/avalanchego/staking"
-	"github.com/ava-labs/avalanchego/utils"
-	avago_utils "github.com/ava-labs/avalanchego/utils"
 	"github.com/ava-labs/avalanchego/utils/constants"
+	"github.com/ava-labs/avalanchego/utils/ips"
 	"github.com/ava-labs/avalanchego/utils/wrappers"
 	"github.com/ava-labs/avalanchego/version"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
 )
 
-func upgradeConn(myTLSCert *tls.Certificate, conn net.Conn) (ids.ShortID, net.Conn, error) {
+func upgradeConn(myTLSCert *tls.Certificate, conn net.Conn) (ids.NodeID, net.Conn, error) {
 	tlsConfig := peer.TLSConfig(*myTLSCert)
 	upgrader := peer.NewTLSServerUpgrader(tlsConfig)
 	// this will block until the ssh handshake is done
@@ -66,7 +65,7 @@ func verifyProtocol(
 	// send the peer our version and peerlist
 
 	// create the version message
-	myIP := avago_utils.IPDesc{
+	myIP := ips.IPPort{
 		IP:   net.IPv6zero,
 		Port: 0,
 	}
@@ -96,7 +95,7 @@ func verifyProtocol(
 	}
 
 	// create the PeerList message
-	plMsg, err := mc.PeerList([]utils.IPCertDesc{}, true)
+	plMsg, err := mc.PeerList([]ips.ClaimedIPPort{}, true)
 	if err != nil {
 		errCh <- err
 		return
@@ -186,7 +185,7 @@ func TestAttachPeer(t *testing.T) {
 	}()
 
 	node := localNode{
-		nodeID:    ids.GenerateTestShortID(),
+		nodeID:    ids.GenerateTestNodeID(),
 		networkID: constants.MainnetID,
 		getConnFunc: func(ctx context.Context, n node.Node) (net.Conn, error) {
 			return peerConn, nil
@@ -237,7 +236,7 @@ func TestAttachPeer(t *testing.T) {
 	msg, err := mc.Chits(chainID, requestID, containerIDs)
 	assert.NoError(err)
 	// send chits to [node]
-	ok := p.Send(msg)
+	ok := p.Send(context.Background(), msg)
 	assert.True(ok)
 	// wait until the go routines are done
 	// also ensures that [assert] calls will be reflected in test results if failed

network/config.go

@@ -12,7 +12,7 @@ import (
 	"github.com/ava-labs/avalanchego/genesis"
 	"github.com/ava-labs/avalanchego/ids"
 	"github.com/ava-labs/avalanchego/utils/constants"
-	"github.com/ava-labs/avalanchego/utils/formatting"
+	"github.com/ava-labs/avalanchego/utils/formatting/address"
 	"github.com/ava-labs/avalanchego/utils/units"
 )
 
@@ -35,40 +35,6 @@ type AddrAndBalance struct {
 	Balance uint64
 }
 
-// Backend is the type of network runner to use
-type Backend byte
-
-const (
-	// Local network runner
-	Local Backend = iota + 1
-	// Kubernetes network runner
-	Kubernetes
-)
-
-func (b Backend) MarshalJSON() ([]byte, error) {
-	switch b {
-	case Local:
-		return []byte("\"local\""), nil
-	case Kubernetes:
-		return []byte("\"k8s\""), nil
-	default:
-		return nil, fmt.Errorf("got unexpected backend %v", b)
-	}
-}
-
-func (b *Backend) UnmarshalJSON(bytes []byte) error {
-	switch string(bytes) {
-	case "\"local\"":
-		*b = Local
-		return nil
-	case "\"k8s\"":
-		*b = Kubernetes
-		return nil
-	default:
-		return fmt.Errorf("got unexpected backend %s", string(bytes))
-	}
-}
-
 // Config that defines a network when it is created.
 type Config struct {
 	// Must not be empty
@@ -76,12 +42,6 @@ type Config struct {
 	// May have length 0
 	// (i.e. network may have no nodes on creation.)
 	NodeConfigs []node.Config `json:"nodeConfigs"`
-	// Log level for the whole network
-	LogLevel string `json:"logLevel"`
-	// Name for the network
-	Name string `json:"name"`
-	// Backend specifies the backend for the network
-	Backend Backend `json:"backend"`
 	// Flags that will be passed to each node in this network.
 	// It can be empty.
 	// Config flags may also be passed in a node's config struct
@@ -138,7 +98,7 @@ func NewAvalancheGoGenesis(
 	networkID uint32,
 	xChainBalances []AddrAndBalance,
 	cChainBalances []AddrAndBalance,
-	genesisVdrs []ids.ShortID,
+	genesisVdrs []ids.NodeID,
 ) ([]byte, error) {
 	switch networkID {
 	case constants.TestnetID, constants.MainnetID, constants.LocalID:
@@ -152,7 +112,11 @@ func NewAvalancheGoGenesis(
 	}
 
 	// Address that controls stake doesn't matter -- generate it randomly
-	genesisVdrStakeAddr, _ := formatting.FormatAddress("X", constants.GetHRP(networkID), ids.GenerateTestShortID().Bytes())
+	genesisVdrStakeAddr, _ := address.Format(
+		"X",
+		constants.GetHRP(networkID),
+		ids.GenerateTestShortID().Bytes(),
+	)
 	config := genesis.UnparsedConfig{
 		NetworkID: networkID,
 		Allocations: []genesis.UnparsedAllocation{
@@ -175,7 +139,7 @@ func NewAvalancheGoGenesis(
 	}
 
 	for _, xChainBal := range xChainBalances {
-		xChainAddr, _ := formatting.FormatAddress("X", constants.GetHRP(networkID), xChainBal.Addr[:])
+		xChainAddr, _ := address.Format("X", constants.GetHRP(networkID), xChainBal.Addr[:])
 		config.Allocations = append(
 			config.Allocations,
 			genesis.UnparsedAllocation{
@@ -212,12 +176,12 @@ func NewAvalancheGoGenesis(
 
 	// Set initial validators.
 	// Give staking rewards to random address.
-	rewardAddr, _ := formatting.FormatAddress("X", constants.GetHRP(networkID), ids.GenerateTestShortID().Bytes())
+	rewardAddr, _ := address.Format("X", constants.GetHRP(networkID), ids.GenerateTestShortID().Bytes())
 	for _, genesisVdr := range genesisVdrs {
 		config.InitialStakers = append(
 			config.InitialStakers,
 			genesis.UnparsedStaker{
-				NodeID:        genesisVdr.PrefixedString(constants.NodeIDPrefix),
+				NodeID:        genesisVdr,
 				RewardAddress: rewardAddr,
 				DelegationFee: 10_000,
 			},

network/config_test.go

@@ -4,90 +4,48 @@ import (
 	"encoding/json"
 	"testing"
 
-	"github.com/ava-labs/avalanche-network-runner/k8s"
-	"github.com/ava-labs/avalanche-network-runner/local"
 	"github.com/ava-labs/avalanche-network-runner/network"
 	"github.com/ava-labs/avalanche-network-runner/network/node"
-	"github.com/ava-labs/avalanche-network-runner/utils"
-	"github.com/ava-labs/avalanchego/utils/logging"
 	"github.com/stretchr/testify/assert"
 )
 
-func TestBackendMarshalJSON(t *testing.T) {
-	assert := assert.New(t)
-
-	backend := network.Local
-	backendJSON, err := json.Marshal(backend)
-	assert.NoError(err)
-	assert.Equal("\"local\"", string(backendJSON))
-	err = json.Unmarshal(backendJSON, &backend)
-	assert.NoError(err)
-	assert.EqualValues(network.Local, backend)
-
-	backend = network.Kubernetes
-	backendJSON, err = json.Marshal(backend)
-	assert.NoError(err)
-	assert.Equal("\"k8s\"", string(backendJSON))
-	err = json.Unmarshal(backendJSON, &backend)
-	assert.NoError(err)
-	assert.EqualValues(network.Kubernetes, backend)
-
-	backend = network.Backend(200) // non-existent backend
-	_, err = json.Marshal(backend)
-	assert.Error(err)
-	err = json.Unmarshal([]byte("invalid"), &backend)
-	assert.Error(err)
-}
-
 func TestConfigMarshalJSON(t *testing.T) {
-	jsonNetcfg := "{\"implSpecificConfig\":\"\",\"genesis\":\"in the beginning there was a token\",\"nodeConfigs\":[{\"implSpecificConfig\":{\"binaryPath\":\"/tmp/some/file/path\"},\"name\":\"node0\",\"isBeacon\":true,\"stakingKey\":\"key123\",\"stakingCert\":\"cert123\",\"configFile\":\"config-file-blablabla1\",\"cchainConfigFile\":\"cchain-config-file-blablabla1\",\"flags\":{\"flag-one\":\"val-one\",\"flag-two\":2}},{\"implSpecificConfig\":{\"apiVersion\":\"0.99.999\",\"identifier\":\"k8s-node-1\",\"image\":\"therepo/theimage\",\"kind\":\"imaginary\",\"namespace\":\"outer-space\",\"tag\":\"omega\"},\"name\":\"node1\",\"isBeacon\":false,\"stakingKey\":\"key456\",\"stakingCert\":\"cert456\",\"configFile\":\"config-file-blablabla2\",\"cchainConfigFile\":\"cchain-config-file-blablabla2\",\"flags\":{\"flag-one\":\"val-one\",\"flag-two\":2}},{\"implSpecificConfig\":{\"binaryPath\":\"/tmp/some/other/path\"},\"name\":\"node2\",\"isBeacon\":false,\"stakingKey\":\"key789\",\"stakingCert\":\"cert789\",\"configFile\":\"config-file-blablabla3\",\"cchainConfigFile\":\"cchain-config-file-blablabla3\",\"flags\":{\"flag-one\":\"val-one\",\"flag-two\":2}}],\"logLevel\":\"DEBUG\",\"name\":\"abcxyz\",\"backend\":\"k8s\",\"flags\":{\"flag-three\":\"val-three\"}}"
+	jsonNetcfg := "{\"genesis\":\"in the beginning there was a token\",\"nodeConfigs\":[{\"binaryPath\":\"/tmp/some/file/path\",\"name\":\"node1\",\"isBeacon\":true,\"stakingKey\":\"key123\",\"stakingCert\":\"cert123\",\"configFile\":\"config-file-blablabla1\",\"chainConfigFiles\":{\"C\": \"cchain-config-file-blablabla1\"},\"flags\":{\"flag-one\":\"val-one\",\"flag-two\":2}},{\"binaryPath\":\"/tmp/some/other/path\",\"name\":\"node2\",\"isBeacon\":false,\"stakingKey\":\"key789\",\"stakingCert\":\"cert789\",\"configFile\":\"config-file-blablabla3\",\"chainConfigFiles\":{\"C\": \"cchain-config-file-blablabla3\"},\"flags\":{\"flag-one\":\"val-one\",\"flag-two\":2}}],\"logLevel\":\"DEBUG\",\"name\":\"abcxyz\",\"flags\":{\"flag-three\":\"val-three\"}}"
 
 	control := network.Config{
 		Genesis: "in the beginning there was a token",
 		NodeConfigs: []node.Config{
 			{
-				ImplSpecificConfig: utils.NewLocalNodeConfigJsonRaw("/tmp/some/file/path"),
-				Name:               "node0",
-				IsBeacon:           true,
-				StakingKey:         "key123",
-				StakingCert:        "cert123",
-				ConfigFile:         "config-file-blablabla1",
-				CChainConfigFile:   "cchain-config-file-blablabla1",
+				Name:        "node1",
+				IsBeacon:    true,
+				StakingKey:  "key123",
+				StakingCert: "cert123",
+				ConfigFile:  "config-file-blablabla1",
+				ChainConfigFiles: map[string]string{
+					"C": "cchain-config-file-blablabla1",
+				},
 				Flags: map[string]interface{}{
 					"flag-one": "val-one",
 					"flag-two": float64(2),
 				},
+				BinaryPath: "/tmp/some/file/path",
 			},
 			{
-				ImplSpecificConfig: newTestK8sNodeConfigJSONRaw(),
-				Name:               "node1",
-				IsBeacon:           false,
-				StakingKey:         "key456",
-				StakingCert:        "cert456",
-				ConfigFile:         "config-file-blablabla2",
-				CChainConfigFile:   "cchain-config-file-blablabla2",
-				Flags: map[string]interface{}{
-					"flag-one": "val-one",
-					"flag-two": float64(2),
+				Name:        "node2",
+				IsBeacon:    false,
+				StakingKey:  "key789",
+				StakingCert: "cert789",
+				ConfigFile:  "config-file-blablabla3",
+				ChainConfigFiles: map[string]string{
+					"C": "cchain-config-file-blablabla3",
 				},
-			},
-			{
-				ImplSpecificConfig: utils.NewLocalNodeConfigJsonRaw("/tmp/some/other/path"),
-				Name:               "node2",
-				IsBeacon:           false,
-				StakingKey:         "key789",
-				StakingCert:        "cert789",
-				ConfigFile:         "config-file-blablabla3",
-				CChainConfigFile:   "cchain-config-file-blablabla3",
 				Flags: map[string]interface{}{
 					"flag-one": "val-one",
 					"flag-two": float64(2),
 				},
+				BinaryPath: "/tmp/some/other/path",
 			},
 		},
-		LogLevel: "DEBUG",
-		Name:     "abcxyz",
-		Backend:  network.Kubernetes,
 		Flags: map[string]interface{}{
 			"flag-three": "val-three",
 		},
@@ -100,38 +58,4 @@ func TestConfigMarshalJSON(t *testing.T) {
 
 	assert := assert.New(t)
 	assert.EqualValues(control, netcfg)
-
-	// At this point unmarshalling should succeed because *the json.RawMessages are ignored*.
-	// Let's try creating a local network first: it should fail as the second node is for k8s
-	_, err := local.NewNetwork(logging.NoLog{}, netcfg, "")
-	assert.Error(err)
-
-	var localcfg local.NodeConfig
-	err = json.Unmarshal([]byte(control.NodeConfigs[0].ImplSpecificConfig), &localcfg)
-	assert.NoError(err)
-	assert.NotEmpty(localcfg.BinaryPath)
-	localcfg.BinaryPath = ""
-	err = json.Unmarshal([]byte(control.NodeConfigs[2].ImplSpecificConfig), &localcfg)
-	assert.NoError(err)
-	assert.NotEmpty(localcfg.BinaryPath)
-	localcfg.BinaryPath = ""
-	err = json.Unmarshal([]byte(control.NodeConfigs[1].ImplSpecificConfig), &localcfg)
-	assert.NoError(err)
-	assert.Empty(localcfg.BinaryPath)
-
-	var k8scfg k8s.ObjectSpec
-	assert.Empty(k8scfg.APIVersion)
-	err = json.Unmarshal([]byte(control.NodeConfigs[0].ImplSpecificConfig), &k8scfg)
-	assert.NoError(err)
-	assert.Empty(k8scfg.APIVersion)
-	err = json.Unmarshal([]byte(control.NodeConfigs[2].ImplSpecificConfig), &k8scfg)
-	assert.NoError(err)
-	assert.Empty(k8scfg.APIVersion)
-	err = json.Unmarshal([]byte(control.NodeConfigs[1].ImplSpecificConfig), &k8scfg)
-	assert.NoError(err)
-	assert.NotEmpty(k8scfg.APIVersion)
-}
-
-func newTestK8sNodeConfigJSONRaw() json.RawMessage {
-	return utils.NewK8sNodeConfigJsonRaw("0.99.999", "k8s-node-1", "therepo/theimage", "imaginary", "outer-space", "omega")
 }

network/network.go

@@ -7,19 +7,15 @@ import (
 	"github.com/ava-labs/avalanche-network-runner/network/node"
 )
 
+var ErrUndefined = errors.New("undefined network")
 var ErrStopped = errors.New("network stopped")
 
 // Network is an abstraction of an Avalanche network
 type Network interface {
-	// Returns a chan that is closed when
-	// all the nodes in the network are healthy.
-	// If an error is sent on this channel, at least 1
-	// node didn't become healthy before the timeout.
-	// If an error isn't sent on the channel before it
-	// closes, all the nodes are healthy.
+	// Returns nil if all the nodes in the network are healthy.
 	// A stopped network is considered unhealthy.
 	// Timeout is given by the context parameter.
-	Healthy(context.Context) chan error
+	Healthy(context.Context) error
 	// Stop all the nodes.
 	// Returns ErrStopped if Stop() was previously called.
 	Stop(context.Context) error
@@ -39,4 +35,12 @@ type Network interface {
 	// Returns the names of all nodes in this network.
 	// Returns ErrStopped if Stop() was previously called.
 	GetNodeNames() ([]string, error)
+	// Save network snapshot
+	// Network is stopped in order to do a safe preservation
+	// Returns the full local path to the snapshot dir
+	SaveSnapshot(context.Context, string) (string, error)
+	// Remove network snapshot
+	RemoveSnapshot(string) error
+	// Get name of available snapshots
+	GetSnapshotNames() ([]string, error)
 }

network/node/node.go

@@ -19,16 +19,14 @@ type Node interface {
 	// across all the nodes in its network.
 	GetName() string
 	// Return this node's Avalanche node ID.
-	GetNodeID() ids.ShortID
+	GetNodeID() ids.NodeID
 	// Return a client that can be used to make API calls.
 	GetAPIClient() api.Client
-	// Return this node's URL.
-	// For a local network, this is the node's IP (e.g. 127.0.0.1).
-	// For a k8s network, this is the DNS name of the pod hosting the node.
+	// Return this node's IP (e.g. 127.0.0.1).
 	GetURL() string
 	// Return this node's P2P (staking) port.
 	GetP2PPort() uint16
-	// Return this node's HTP API port.
+	// Return this node's HTTP API port.
 	GetAPIPort() uint16
 	// Starts a new test peer, connects it to the given node, and returns the peer.
 	// [handler] defines how the test peer handles messages it receives.
@@ -36,12 +34,22 @@ type Node interface {
 	// It's left to the caller to maintain a reference to the returned peer.
 	// The caller should call StartClose() on the peer when they're done with it.
 	AttachPeer(ctx context.Context, handler router.InboundHandler) (peer.Peer, error)
+	// Return this node's avalanchego binary path
+	GetBinaryPath() string
+	// Return this node's db dir
+	GetDbDir() string
+	// Return this node's logs dir
+	GetLogsDir() string
+	// Return this node's build dir
+	GetBuildDir() string
+	// Return this node's config file contents
+	GetConfigFile() string
+	// Return this node's config
+	GetConfig() Config
 }
 
 // Config encapsulates an avalanchego configuration
 type Config struct {
-	// Configuration specific to a particular implementation of a node.
-	ImplSpecificConfig json.RawMessage `json:"implSpecificConfig"`
 	// A node's name must be unique from all other nodes
 	// in a network. If Name is the empty string, a
 	// unique name is assigned on node creation.
@@ -56,7 +64,7 @@ type Config struct {
 	// May be nil.
 	ConfigFile string `json:"configFile"`
 	// May be nil.
-	CChainConfigFile string `json:"cChainConfigFile"`
+	ChainConfigFiles map[string]string `json:"chainConfigFiles"`
 	// Flags can hold additional flags for the node.
 	// It can be empty.
 	// The precedence of flags handling is:
@@ -64,13 +72,17 @@ type Config struct {
 	// 2. Flags defined in network.Config override
 	// 3. Flags defined in the json config file
 	Flags map[string]interface{} `json:"flags"`
+	// What type of node this is
+	BinaryPath string `json:"binaryPath"`
+	// If non-nil, direct this node's Stdout to os.Stdout
+	RedirectStdout bool `json:"redirectStdout"`
+	// If non-nil, direct this node's Stderr to os.Stderr
+	RedirectStderr bool `json:"redirectStderr"`
 }
 
 // Validate returns an error if this config is invalid
 func (c *Config) Validate(expectedNetworkID uint32) error {
 	switch {
-	case c.ImplSpecificConfig == nil:
-		return errors.New("implementation-specific node config not given")
 	case c.StakingKey == "":
 		return errors.New("staking key not given")
 	case c.StakingCert == "":

rpcpb/rpc.pb.gw.go

@@ -99,6 +99,74 @@ func local_request_ControlService_Start_0(ctx context.Context, marshaler runtime
 
 }
 
+func request_ControlService_CreateBlockchains_0(ctx context.Context, marshaler runtime.Marshaler, client ControlServiceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) {
+	var protoReq CreateBlockchainsRequest
+	var metadata runtime.ServerMetadata
+
+	newReader, berr := utilities.IOReaderFactory(req.Body)
+	if berr != nil {
+		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", berr)
+	}
+	if err := marshaler.NewDecoder(newReader()).Decode(&protoReq); err != nil && err != io.EOF {
+		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
+	}
+
+	msg, err := client.CreateBlockchains(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD))
+	return msg, metadata, err
+
+}
+
+func local_request_ControlService_CreateBlockchains_0(ctx context.Context, marshaler runtime.Marshaler, server ControlServiceServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) {
+	var protoReq CreateBlockchainsRequest
+	var metadata runtime.ServerMetadata
+
+	newReader, berr := utilities.IOReaderFactory(req.Body)
+	if berr != nil {
+		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", berr)
+	}
+	if err := marshaler.NewDecoder(newReader()).Decode(&protoReq); err != nil && err != io.EOF {
+		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
+	}
+
+	msg, err := server.CreateBlockchains(ctx, &protoReq)
+	return msg, metadata, err
+
+}
+
+func request_ControlService_CreateSubnets_0(ctx context.Context, marshaler runtime.Marshaler, client ControlServiceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) {
+	var protoReq CreateSubnetsRequest
+	var metadata runtime.ServerMetadata
+
+	newReader, berr := utilities.IOReaderFactory(req.Body)
+	if berr != nil {
+		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", berr)
+	}
+	if err := marshaler.NewDecoder(newReader()).Decode(&protoReq); err != nil && err != io.EOF {
+		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
+	}
+
+	msg, err := client.CreateSubnets(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD))
+	return msg, metadata, err
+
+}
+
+func local_request_ControlService_CreateSubnets_0(ctx context.Context, marshaler runtime.Marshaler, server ControlServiceServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) {
+	var protoReq CreateSubnetsRequest
+	var metadata runtime.ServerMetadata
+
+	newReader, berr := utilities.IOReaderFactory(req.Body)
+	if berr != nil {
+		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", berr)
+	}
+	if err := marshaler.NewDecoder(newReader()).Decode(&protoReq); err != nil && err != io.EOF {
+		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
+	}
+
+	msg, err := server.CreateSubnets(ctx, &protoReq)
+	return msg, metadata, err
+
+}
+
 func request_ControlService_Health_0(ctx context.Context, marshaler runtime.Marshaler, client ControlServiceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) {
 	var protoReq HealthRequest
 	var metadata runtime.ServerMetadata
@@ -430,6 +498,142 @@ func local_request_ControlService_SendOutboundMessage_0(ctx context.Context, mar
 
 }
 
+func request_ControlService_SaveSnapshot_0(ctx context.Context, marshaler runtime.Marshaler, client ControlServiceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) {
+	var protoReq SaveSnapshotRequest
+	var metadata runtime.ServerMetadata
+
+	newReader, berr := utilities.IOReaderFactory(req.Body)
+	if berr != nil {
+		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", berr)
+	}
+	if err := marshaler.NewDecoder(newReader()).Decode(&protoReq); err != nil && err != io.EOF {
+		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
+	}
+
+	msg, err := client.SaveSnapshot(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD))
+	return msg, metadata, err
+
+}
+
+func local_request_ControlService_SaveSnapshot_0(ctx context.Context, marshaler runtime.Marshaler, server ControlServiceServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) {
+	var protoReq SaveSnapshotRequest
+	var metadata runtime.ServerMetadata
+
+	newReader, berr := utilities.IOReaderFactory(req.Body)
+	if berr != nil {
+		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", berr)
+	}
+	if err := marshaler.NewDecoder(newReader()).Decode(&protoReq); err != nil && err != io.EOF {
+		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
+	}
+
+	msg, err := server.SaveSnapshot(ctx, &protoReq)
+	return msg, metadata, err
+
+}
+
+func request_ControlService_LoadSnapshot_0(ctx context.Context, marshaler runtime.Marshaler, client ControlServiceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) {
+	var protoReq LoadSnapshotRequest
+	var metadata runtime.ServerMetadata
+
+	newReader, berr := utilities.IOReaderFactory(req.Body)
+	if berr != nil {
+		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", berr)
+	}
+	if err := marshaler.NewDecoder(newReader()).Decode(&protoReq); err != nil && err != io.EOF {
+		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
+	}
+
+	msg, err := client.LoadSnapshot(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD))
+	return msg, metadata, err
+
+}
+
+func local_request_ControlService_LoadSnapshot_0(ctx context.Context, marshaler runtime.Marshaler, server ControlServiceServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) {
+	var protoReq LoadSnapshotRequest
+	var metadata runtime.ServerMetadata
+
+	newReader, berr := utilities.IOReaderFactory(req.Body)
+	if berr != nil {
+		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", berr)
+	}
+	if err := marshaler.NewDecoder(newReader()).Decode(&protoReq); err != nil && err != io.EOF {
+		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
+	}
+
+	msg, err := server.LoadSnapshot(ctx, &protoReq)
+	return msg, metadata, err
+
+}
+
+func request_ControlService_RemoveSnapshot_0(ctx context.Context, marshaler runtime.Marshaler, client ControlServiceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) {
+	var protoReq RemoveSnapshotRequest
+	var metadata runtime.ServerMetadata
+
+	newReader, berr := utilities.IOReaderFactory(req.Body)
+	if berr != nil {
+		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", berr)
+	}
+	if err := marshaler.NewDecoder(newReader()).Decode(&protoReq); err != nil && err != io.EOF {
+		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
+	}
+
+	msg, err := client.RemoveSnapshot(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD))
+	return msg, metadata, err
+
+}
+
+func local_request_ControlService_RemoveSnapshot_0(ctx context.Context, marshaler runtime.Marshaler, server ControlServiceServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) {
+	var protoReq RemoveSnapshotRequest
+	var metadata runtime.ServerMetadata
+
+	newReader, berr := utilities.IOReaderFactory(req.Body)
+	if berr != nil {
+		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", berr)
+	}
+	if err := marshaler.NewDecoder(newReader()).Decode(&protoReq); err != nil && err != io.EOF {
+		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
+	}
+
+	msg, err := server.RemoveSnapshot(ctx, &protoReq)
+	return msg, metadata, err
+
+}
+
+func request_ControlService_GetSnapshotNames_0(ctx context.Context, marshaler runtime.Marshaler, client ControlServiceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) {
+	var protoReq GetSnapshotNamesRequest
+	var metadata runtime.ServerMetadata
+
+	newReader, berr := utilities.IOReaderFactory(req.Body)
+	if berr != nil {
+		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", berr)
+	}
+	if err := marshaler.NewDecoder(newReader()).Decode(&protoReq); err != nil && err != io.EOF {
+		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
+	}
+
+	msg, err := client.GetSnapshotNames(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD))
+	return msg, metadata, err
+
+}
+
+func local_request_ControlService_GetSnapshotNames_0(ctx context.Context, marshaler runtime.Marshaler, server ControlServiceServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) {
+	var protoReq GetSnapshotNamesRequest
+	var metadata runtime.ServerMetadata
+
+	newReader, berr := utilities.IOReaderFactory(req.Body)
+	if berr != nil {
+		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", berr)
+	}
+	if err := marshaler.NewDecoder(newReader()).Decode(&protoReq); err != nil && err != io.EOF {
+		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
+	}
+
+	msg, err := server.GetSnapshotNames(ctx, &protoReq)
+	return msg, metadata, err
+
+}
+
 // RegisterPingServiceHandlerServer registers the http handlers for service PingService to "mux".
 // UnaryRPC     :call PingServiceServer directly.
 // StreamingRPC :currently unsupported pending https://github.com/grpc/grpc-go/issues/906.
@@ -493,6 +697,54 @@ func RegisterControlServiceHandlerServer(ctx context.Context, mux *runtime.Serve
 
 	})
 
+	mux.Handle("POST", pattern_ControlService_CreateBlockchains_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
+		ctx, cancel := context.WithCancel(req.Context())
+		defer cancel()
+		var stream runtime.ServerTransportStream
+		ctx = grpc.NewContextWithServerTransportStream(ctx, &stream)
+		inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req)
+		var err error
+		ctx, err = runtime.AnnotateIncomingContext(ctx, mux, req, "/rpcpb.ControlService/CreateBlockchains", runtime.WithHTTPPathPattern("/v1/control/createblockchains"))
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+		resp, md, err := local_request_ControlService_CreateBlockchains_0(ctx, inboundMarshaler, server, req, pathParams)
+		md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer())
+		ctx = runtime.NewServerMetadataContext(ctx, md)
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+
+		forward_ControlService_CreateBlockchains_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...)
+
+	})
+
+	mux.Handle("POST", pattern_ControlService_CreateSubnets_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
+		ctx, cancel := context.WithCancel(req.Context())
+		defer cancel()
+		var stream runtime.ServerTransportStream
+		ctx = grpc.NewContextWithServerTransportStream(ctx, &stream)
+		inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req)
+		var err error
+		ctx, err = runtime.AnnotateIncomingContext(ctx, mux, req, "/rpcpb.ControlService/CreateSubnets", runtime.WithHTTPPathPattern("/v1/control/createsubnets"))
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+		resp, md, err := local_request_ControlService_CreateSubnets_0(ctx, inboundMarshaler, server, req, pathParams)
+		md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer())
+		ctx = runtime.NewServerMetadataContext(ctx, md)
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+
+		forward_ControlService_CreateSubnets_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...)
+
+	})
+
 	mux.Handle("POST", pattern_ControlService_Health_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
 		ctx, cancel := context.WithCancel(req.Context())
 		defer cancel()
@@ -716,6 +968,102 @@ func RegisterControlServiceHandlerServer(ctx context.Context, mux *runtime.Serve
 
 	})
 
+	mux.Handle("POST", pattern_ControlService_SaveSnapshot_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
+		ctx, cancel := context.WithCancel(req.Context())
+		defer cancel()
+		var stream runtime.ServerTransportStream
+		ctx = grpc.NewContextWithServerTransportStream(ctx, &stream)
+		inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req)
+		var err error
+		ctx, err = runtime.AnnotateIncomingContext(ctx, mux, req, "/rpcpb.ControlService/SaveSnapshot", runtime.WithHTTPPathPattern("/v1/control/savesnapshot"))
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+		resp, md, err := local_request_ControlService_SaveSnapshot_0(ctx, inboundMarshaler, server, req, pathParams)
+		md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer())
+		ctx = runtime.NewServerMetadataContext(ctx, md)
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+
+		forward_ControlService_SaveSnapshot_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...)
+
+	})
+
+	mux.Handle("POST", pattern_ControlService_LoadSnapshot_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
+		ctx, cancel := context.WithCancel(req.Context())
+		defer cancel()
+		var stream runtime.ServerTransportStream
+		ctx = grpc.NewContextWithServerTransportStream(ctx, &stream)
+		inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req)
+		var err error
+		ctx, err = runtime.AnnotateIncomingContext(ctx, mux, req, "/rpcpb.ControlService/LoadSnapshot", runtime.WithHTTPPathPattern("/v1/control/loadsnapshot"))
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+		resp, md, err := local_request_ControlService_LoadSnapshot_0(ctx, inboundMarshaler, server, req, pathParams)
+		md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer())
+		ctx = runtime.NewServerMetadataContext(ctx, md)
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+
+		forward_ControlService_LoadSnapshot_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...)
+
+	})
+
+	mux.Handle("POST", pattern_ControlService_RemoveSnapshot_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
+		ctx, cancel := context.WithCancel(req.Context())
+		defer cancel()
+		var stream runtime.ServerTransportStream
+		ctx = grpc.NewContextWithServerTransportStream(ctx, &stream)
+		inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req)
+		var err error
+		ctx, err = runtime.AnnotateIncomingContext(ctx, mux, req, "/rpcpb.ControlService/RemoveSnapshot", runtime.WithHTTPPathPattern("/v1/control/removesnapshot"))
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+		resp, md, err := local_request_ControlService_RemoveSnapshot_0(ctx, inboundMarshaler, server, req, pathParams)
+		md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer())
+		ctx = runtime.NewServerMetadataContext(ctx, md)
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+
+		forward_ControlService_RemoveSnapshot_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...)
+
+	})
+
+	mux.Handle("POST", pattern_ControlService_GetSnapshotNames_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
+		ctx, cancel := context.WithCancel(req.Context())
+		defer cancel()
+		var stream runtime.ServerTransportStream
+		ctx = grpc.NewContextWithServerTransportStream(ctx, &stream)
+		inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req)
+		var err error
+		ctx, err = runtime.AnnotateIncomingContext(ctx, mux, req, "/rpcpb.ControlService/GetSnapshotNames", runtime.WithHTTPPathPattern("/v1/control/getsnapshotnames"))
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+		resp, md, err := local_request_ControlService_GetSnapshotNames_0(ctx, inboundMarshaler, server, req, pathParams)
+		md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer())
+		ctx = runtime.NewServerMetadataContext(ctx, md)
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+
+		forward_ControlService_GetSnapshotNames_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...)
+
+	})
+
 	return nil
 }
 
@@ -848,6 +1196,48 @@ func RegisterControlServiceHandlerClient(ctx context.Context, mux *runtime.Serve
 
 	})
 
+	mux.Handle("POST", pattern_ControlService_CreateBlockchains_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
+		ctx, cancel := context.WithCancel(req.Context())
+		defer cancel()
+		inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req)
+		var err error
+		ctx, err = runtime.AnnotateContext(ctx, mux, req, "/rpcpb.ControlService/CreateBlockchains", runtime.WithHTTPPathPattern("/v1/control/createblockchains"))
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+		resp, md, err := request_ControlService_CreateBlockchains_0(ctx, inboundMarshaler, client, req, pathParams)
+		ctx = runtime.NewServerMetadataContext(ctx, md)
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+
+		forward_ControlService_CreateBlockchains_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...)
+
+	})
+
+	mux.Handle("POST", pattern_ControlService_CreateSubnets_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
+		ctx, cancel := context.WithCancel(req.Context())
+		defer cancel()
+		inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req)
+		var err error
+		ctx, err = runtime.AnnotateContext(ctx, mux, req, "/rpcpb.ControlService/CreateSubnets", runtime.WithHTTPPathPattern("/v1/control/createsubnets"))
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+		resp, md, err := request_ControlService_CreateSubnets_0(ctx, inboundMarshaler, client, req, pathParams)
+		ctx = runtime.NewServerMetadataContext(ctx, md)
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+
+		forward_ControlService_CreateSubnets_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...)
+
+	})
+
 	mux.Handle("POST", pattern_ControlService_Health_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
 		ctx, cancel := context.WithCancel(req.Context())
 		defer cancel()
@@ -1058,12 +1448,100 @@ func RegisterControlServiceHandlerClient(ctx context.Context, mux *runtime.Serve
 
 	})
 
+	mux.Handle("POST", pattern_ControlService_SaveSnapshot_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
+		ctx, cancel := context.WithCancel(req.Context())
+		defer cancel()
+		inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req)
+		var err error
+		ctx, err = runtime.AnnotateContext(ctx, mux, req, "/rpcpb.ControlService/SaveSnapshot", runtime.WithHTTPPathPattern("/v1/control/savesnapshot"))
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+		resp, md, err := request_ControlService_SaveSnapshot_0(ctx, inboundMarshaler, client, req, pathParams)
+		ctx = runtime.NewServerMetadataContext(ctx, md)
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+
+		forward_ControlService_SaveSnapshot_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...)
+
+	})
+
+	mux.Handle("POST", pattern_ControlService_LoadSnapshot_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
+		ctx, cancel := context.WithCancel(req.Context())
+		defer cancel()
+		inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req)
+		var err error
+		ctx, err = runtime.AnnotateContext(ctx, mux, req, "/rpcpb.ControlService/LoadSnapshot", runtime.WithHTTPPathPattern("/v1/control/loadsnapshot"))
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+		resp, md, err := request_ControlService_LoadSnapshot_0(ctx, inboundMarshaler, client, req, pathParams)
+		ctx = runtime.NewServerMetadataContext(ctx, md)
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+
+		forward_ControlService_LoadSnapshot_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...)
+
+	})
+
+	mux.Handle("POST", pattern_ControlService_RemoveSnapshot_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
+		ctx, cancel := context.WithCancel(req.Context())
+		defer cancel()
+		inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req)
+		var err error
+		ctx, err = runtime.AnnotateContext(ctx, mux, req, "/rpcpb.ControlService/RemoveSnapshot", runtime.WithHTTPPathPattern("/v1/control/removesnapshot"))
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+		resp, md, err := request_ControlService_RemoveSnapshot_0(ctx, inboundMarshaler, client, req, pathParams)
+		ctx = runtime.NewServerMetadataContext(ctx, md)
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+
+		forward_ControlService_RemoveSnapshot_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...)
+
+	})
+
+	mux.Handle("POST", pattern_ControlService_GetSnapshotNames_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
+		ctx, cancel := context.WithCancel(req.Context())
+		defer cancel()
+		inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req)
+		var err error
+		ctx, err = runtime.AnnotateContext(ctx, mux, req, "/rpcpb.ControlService/GetSnapshotNames", runtime.WithHTTPPathPattern("/v1/control/getsnapshotnames"))
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+		resp, md, err := request_ControlService_GetSnapshotNames_0(ctx, inboundMarshaler, client, req, pathParams)
+		ctx = runtime.NewServerMetadataContext(ctx, md)
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+
+		forward_ControlService_GetSnapshotNames_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...)
+
+	})
+
 	return nil
 }
 
 var (
 	pattern_ControlService_Start_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v1", "control", "start"}, ""))
 
+	pattern_ControlService_CreateBlockchains_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v1", "control", "createblockchains"}, ""))
+
+	pattern_ControlService_CreateSubnets_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v1", "control", "createsubnets"}, ""))
+
 	pattern_ControlService_Health_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v1", "control", "health"}, ""))
 
 	pattern_ControlService_URIs_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v1", "control", "uris"}, ""))
@@ -1083,11 +1561,23 @@ var (
 	pattern_ControlService_AttachPeer_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v1", "control", "attachpeer"}, ""))
 
 	pattern_ControlService_SendOutboundMessage_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v1", "control", "sendoutboundmessage"}, ""))
+
+	pattern_ControlService_SaveSnapshot_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v1", "control", "savesnapshot"}, ""))
+
+	pattern_ControlService_LoadSnapshot_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v1", "control", "loadsnapshot"}, ""))
+
+	pattern_ControlService_RemoveSnapshot_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v1", "control", "removesnapshot"}, ""))
+
+	pattern_ControlService_GetSnapshotNames_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v1", "control", "getsnapshotnames"}, ""))
 )
 
 var (
 	forward_ControlService_Start_0 = runtime.ForwardResponseMessage
 
+	forward_ControlService_CreateBlockchains_0 = runtime.ForwardResponseMessage
+
+	forward_ControlService_CreateSubnets_0 = runtime.ForwardResponseMessage
+
 	forward_ControlService_Health_0 = runtime.ForwardResponseMessage
 
 	forward_ControlService_URIs_0 = runtime.ForwardResponseMessage
@@ -1107,4 +1597,12 @@ var (
 	forward_ControlService_AttachPeer_0 = runtime.ForwardResponseMessage
 
 	forward_ControlService_SendOutboundMessage_0 = runtime.ForwardResponseMessage
+
+	forward_ControlService_SaveSnapshot_0 = runtime.ForwardResponseMessage
+
+	forward_ControlService_LoadSnapshot_0 = runtime.ForwardResponseMessage
+
+	forward_ControlService_RemoveSnapshot_0 = runtime.ForwardResponseMessage
+
+	forward_ControlService_GetSnapshotNames_0 = runtime.ForwardResponseMessage
 )

rpcpb/rpc.proto

@@ -29,6 +29,20 @@ service ControlService {
     };
   }
 
+  rpc CreateBlockchains(CreateBlockchainsRequest) returns (CreateBlockchainsResponse) {
+    option (google.api.http) = {
+      post: "/v1/control/createblockchains"
+      body: "*"
+    };
+  }
+
+  rpc CreateSubnets(CreateSubnetsRequest) returns (CreateSubnetsResponse) {
+    option (google.api.http) = {
+      post: "/v1/control/createsubnets"
+      body: "*"
+    };
+  }
+
   rpc Health(HealthRequest) returns (HealthResponse) {
     option (google.api.http) = {
       post: "/v1/control/health"
@@ -98,6 +112,34 @@ service ControlService {
       body: "*"
     };
   }
+
+  rpc SaveSnapshot(SaveSnapshotRequest) returns (SaveSnapshotResponse) {
+    option (google.api.http) = {
+      post: "/v1/control/savesnapshot"
+      body: "*"
+    };
+  }
+
+  rpc LoadSnapshot(LoadSnapshotRequest) returns (LoadSnapshotResponse) {
+    option (google.api.http) = {
+      post: "/v1/control/loadsnapshot"
+      body: "*"
+    };
+  }
+
+  rpc RemoveSnapshot(RemoveSnapshotRequest) returns (RemoveSnapshotResponse) {
+    option (google.api.http) = {
+      post: "/v1/control/removesnapshot"
+      body: "*"
+    };
+  }
+
+  rpc GetSnapshotNames(GetSnapshotNamesRequest) returns (GetSnapshotNamesResponse) {
+    option (google.api.http) = {
+      post: "/v1/control/getsnapshotnames"
+      body: "*"
+    };
+  }
 }
 
 message ClusterInfo {
@@ -114,6 +156,7 @@ message ClusterInfo {
   bool custom_vms_healthy = 7;
   // The map of custom VM IDs in "ids.ID" format to its VM information.
   map<string, CustomVmInfo> custom_vms = 8;
+  repeated string subnets          = 9;
 }
 
 message CustomVmInfo {
@@ -155,8 +198,7 @@ message StartRequest {
   string exec_path                    = 1;
   optional uint32 num_nodes           = 2;
   optional string whitelisted_subnets = 3;
-  optional string log_level           = 4;
-
+  optional string global_node_config  = 4;
   // Used for both database and log files.
   optional string root_data_dir = 5;
 
@@ -175,18 +217,50 @@ message StartRequest {
   // even if the VM binary exists on the local plugins directory.
   map<string, string> custom_vms = 7;
 
+  map<string, string> custom_node_configs = 8;
+
   // Map of chain name to config file contents.
   // If specified, will create a file "chainname/config.json" with
   // the contents provided here.
   // Currently only supports C chain, pending generalizing logic
   // in avalanche-network-runner.
-  map<string, string> chain_configs = 8;
+  map<string, string> chain_configs = 9;
 }
 
 message StartResponse {
   ClusterInfo cluster_info = 1;
 }
 
+message BlockchainSpec {
+  string vm_name = 1;
+  string genesis = 2;
+  optional string subnet_id = 3;
+}
+
+message CreateBlockchainsRequest {
+  // The list of custom VM name, its genesis file path, and (optional) subnet id to use.
+  //
+  // The matching file with the name in "ids.ID" format must exist.
+  // e.g., ids.ToID(hashing.ComputeHash256("subnetevm")).String()
+  // e.g., subnet-cli create VMID subnetevm
+  //
+  // If this field is set to none (by default), the node/network-runner
+  // will return error
+  repeated BlockchainSpec blockchain_specs = 1;
+}
+
+message CreateBlockchainsResponse {
+  ClusterInfo cluster_info = 1;
+}
+
+message CreateSubnetsRequest {
+  optional uint32 num_subnets = 1;
+}
+
+message CreateSubnetsResponse {
+  ClusterInfo cluster_info = 1;
+}
+
 message HealthRequest {}
 
 message HealthResponse {
@@ -220,10 +294,9 @@ message RestartNodeRequest {
   // Optional fields are set to the previous values if empty.
   optional string exec_path           = 2;
   optional string whitelisted_subnets = 3;
-  optional string log_level           = 4;
 
   // Used for both database and log files.
-  optional string root_data_dir = 5;
+  optional string root_data_dir = 4;
 }
 
 message RestartNodeResponse {
@@ -272,3 +345,36 @@ message SendOutboundMessageRequest {
 message SendOutboundMessageResponse {
   bool sent = 1;
 }
+
+message SaveSnapshotRequest {
+  string snapshot_name = 1;
+}
+
+message SaveSnapshotResponse {
+  string snapshot_path = 1;
+}
+
+message LoadSnapshotRequest {
+  string snapshot_name = 1;
+  optional string exec_path = 2;
+  optional string plugin_dir = 3;
+  optional string root_data_dir = 4;
+}
+
+message LoadSnapshotResponse {
+  ClusterInfo cluster_info = 1;
+}
+
+message RemoveSnapshotRequest {
+  string snapshot_name = 1;
+}
+
+message RemoveSnapshotResponse {
+}
+
+message GetSnapshotNamesRequest {
+}
+
+message GetSnapshotNamesResponse {
+  repeated string snapshot_names = 1;
+}

rpcpb/rpc_grpc.pb.go

@@ -109,6 +109,8 @@ var PingService_ServiceDesc = grpc.ServiceDesc{
 // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
 type ControlServiceClient interface {
 	Start(ctx context.Context, in *StartRequest, opts ...grpc.CallOption) (*StartResponse, error)
+	CreateBlockchains(ctx context.Context, in *CreateBlockchainsRequest, opts ...grpc.CallOption) (*CreateBlockchainsResponse, error)
+	CreateSubnets(ctx context.Context, in *CreateSubnetsRequest, opts ...grpc.CallOption) (*CreateSubnetsResponse, error)
 	Health(ctx context.Context, in *HealthRequest, opts ...grpc.CallOption) (*HealthResponse, error)
 	URIs(ctx context.Context, in *URIsRequest, opts ...grpc.CallOption) (*URIsResponse, error)
 	Status(ctx context.Context, in *StatusRequest, opts ...grpc.CallOption) (*StatusResponse, error)
@@ -119,6 +121,10 @@ type ControlServiceClient interface {
 	Stop(ctx context.Context, in *StopRequest, opts ...grpc.CallOption) (*StopResponse, error)
 	AttachPeer(ctx context.Context, in *AttachPeerRequest, opts ...grpc.CallOption) (*AttachPeerResponse, error)
 	SendOutboundMessage(ctx context.Context, in *SendOutboundMessageRequest, opts ...grpc.CallOption) (*SendOutboundMessageResponse, error)
+	SaveSnapshot(ctx context.Context, in *SaveSnapshotRequest, opts ...grpc.CallOption) (*SaveSnapshotResponse, error)
+	LoadSnapshot(ctx context.Context, in *LoadSnapshotRequest, opts ...grpc.CallOption) (*LoadSnapshotResponse, error)
+	RemoveSnapshot(ctx context.Context, in *RemoveSnapshotRequest, opts ...grpc.CallOption) (*RemoveSnapshotResponse, error)
+	GetSnapshotNames(ctx context.Context, in *GetSnapshotNamesRequest, opts ...grpc.CallOption) (*GetSnapshotNamesResponse, error)
 }
 
 type controlServiceClient struct {
@@ -138,6 +144,24 @@ func (c *controlServiceClient) Start(ctx context.Context, in *StartRequest, opts
 	return out, nil
 }
 
+func (c *controlServiceClient) CreateBlockchains(ctx context.Context, in *CreateBlockchainsRequest, opts ...grpc.CallOption) (*CreateBlockchainsResponse, error) {
+	out := new(CreateBlockchainsResponse)
+	err := c.cc.Invoke(ctx, "/rpcpb.ControlService/CreateBlockchains", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *controlServiceClient) CreateSubnets(ctx context.Context, in *CreateSubnetsRequest, opts ...grpc.CallOption) (*CreateSubnetsResponse, error) {
+	out := new(CreateSubnetsResponse)
+	err := c.cc.Invoke(ctx, "/rpcpb.ControlService/CreateSubnets", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 func (c *controlServiceClient) Health(ctx context.Context, in *HealthRequest, opts ...grpc.CallOption) (*HealthResponse, error) {
 	out := new(HealthResponse)
 	err := c.cc.Invoke(ctx, "/rpcpb.ControlService/Health", in, out, opts...)
@@ -251,11 +275,49 @@ func (c *controlServiceClient) SendOutboundMessage(ctx context.Context, in *Send
 	return out, nil
 }
 
+func (c *controlServiceClient) SaveSnapshot(ctx context.Context, in *SaveSnapshotRequest, opts ...grpc.CallOption) (*SaveSnapshotResponse, error) {
+	out := new(SaveSnapshotResponse)
+	err := c.cc.Invoke(ctx, "/rpcpb.ControlService/SaveSnapshot", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *controlServiceClient) LoadSnapshot(ctx context.Context, in *LoadSnapshotRequest, opts ...grpc.CallOption) (*LoadSnapshotResponse, error) {
+	out := new(LoadSnapshotResponse)
+	err := c.cc.Invoke(ctx, "/rpcpb.ControlService/LoadSnapshot", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *controlServiceClient) RemoveSnapshot(ctx context.Context, in *RemoveSnapshotRequest, opts ...grpc.CallOption) (*RemoveSnapshotResponse, error) {
+	out := new(RemoveSnapshotResponse)
+	err := c.cc.Invoke(ctx, "/rpcpb.ControlService/RemoveSnapshot", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *controlServiceClient) GetSnapshotNames(ctx context.Context, in *GetSnapshotNamesRequest, opts ...grpc.CallOption) (*GetSnapshotNamesResponse, error) {
+	out := new(GetSnapshotNamesResponse)
+	err := c.cc.Invoke(ctx, "/rpcpb.ControlService/GetSnapshotNames", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 // ControlServiceServer is the server API for ControlService service.
 // All implementations must embed UnimplementedControlServiceServer
 // for forward compatibility
 type ControlServiceServer interface {
 	Start(context.Context, *StartRequest) (*StartResponse, error)
+	CreateBlockchains(context.Context, *CreateBlockchainsRequest) (*CreateBlockchainsResponse, error)
+	CreateSubnets(context.Context, *CreateSubnetsRequest) (*CreateSubnetsResponse, error)
 	Health(context.Context, *HealthRequest) (*HealthResponse, error)
 	URIs(context.Context, *URIsRequest) (*URIsResponse, error)
 	Status(context.Context, *StatusRequest) (*StatusResponse, error)
@@ -266,6 +328,10 @@ type ControlServiceServer interface {
 	Stop(context.Context, *StopRequest) (*StopResponse, error)
 	AttachPeer(context.Context, *AttachPeerRequest) (*AttachPeerResponse, error)
 	SendOutboundMessage(context.Context, *SendOutboundMessageRequest) (*SendOutboundMessageResponse, error)
+	SaveSnapshot(context.Context, *SaveSnapshotRequest) (*SaveSnapshotResponse, error)
+	LoadSnapshot(context.Context, *LoadSnapshotRequest) (*LoadSnapshotResponse, error)
+	RemoveSnapshot(context.Context, *RemoveSnapshotRequest) (*RemoveSnapshotResponse, error)
+	GetSnapshotNames(context.Context, *GetSnapshotNamesRequest) (*GetSnapshotNamesResponse, error)
 	mustEmbedUnimplementedControlServiceServer()
 }
 
@@ -276,6 +342,12 @@ type UnimplementedControlServiceServer struct {
 func (UnimplementedControlServiceServer) Start(context.Context, *StartRequest) (*StartResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method Start not implemented")
 }
+func (UnimplementedControlServiceServer) CreateBlockchains(context.Context, *CreateBlockchainsRequest) (*CreateBlockchainsResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method CreateBlockchains not implemented")
+}
+func (UnimplementedControlServiceServer) CreateSubnets(context.Context, *CreateSubnetsRequest) (*CreateSubnetsResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method CreateSubnets not implemented")
+}
 func (UnimplementedControlServiceServer) Health(context.Context, *HealthRequest) (*HealthResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method Health not implemented")
 }
@@ -306,6 +378,18 @@ func (UnimplementedControlServiceServer) AttachPeer(context.Context, *AttachPeer
 func (UnimplementedControlServiceServer) SendOutboundMessage(context.Context, *SendOutboundMessageRequest) (*SendOutboundMessageResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method SendOutboundMessage not implemented")
 }
+func (UnimplementedControlServiceServer) SaveSnapshot(context.Context, *SaveSnapshotRequest) (*SaveSnapshotResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method SaveSnapshot not implemented")
+}
+func (UnimplementedControlServiceServer) LoadSnapshot(context.Context, *LoadSnapshotRequest) (*LoadSnapshotResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method LoadSnapshot not implemented")
+}
+func (UnimplementedControlServiceServer) RemoveSnapshot(context.Context, *RemoveSnapshotRequest) (*RemoveSnapshotResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method RemoveSnapshot not implemented")
+}
+func (UnimplementedControlServiceServer) GetSnapshotNames(context.Context, *GetSnapshotNamesRequest) (*GetSnapshotNamesResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetSnapshotNames not implemented")
+}
 func (UnimplementedControlServiceServer) mustEmbedUnimplementedControlServiceServer() {}
 
 // UnsafeControlServiceServer may be embedded to opt out of forward compatibility for this service.
@@ -337,6 +421,42 @@ func _ControlService_Start_Handler(srv interface{}, ctx context.Context, dec fun
 	return interceptor(ctx, in, info, handler)
 }
 
+func _ControlService_CreateBlockchains_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(CreateBlockchainsRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ControlServiceServer).CreateBlockchains(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/rpcpb.ControlService/CreateBlockchains",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ControlServiceServer).CreateBlockchains(ctx, req.(*CreateBlockchainsRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _ControlService_CreateSubnets_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(CreateSubnetsRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ControlServiceServer).CreateSubnets(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/rpcpb.ControlService/CreateSubnets",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ControlServiceServer).CreateSubnets(ctx, req.(*CreateSubnetsRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 func _ControlService_Health_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(HealthRequest)
 	if err := dec(in); err != nil {
@@ -520,6 +640,78 @@ func _ControlService_SendOutboundMessage_Handler(srv interface{}, ctx context.Co
 	return interceptor(ctx, in, info, handler)
 }
 
+func _ControlService_SaveSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(SaveSnapshotRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ControlServiceServer).SaveSnapshot(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/rpcpb.ControlService/SaveSnapshot",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ControlServiceServer).SaveSnapshot(ctx, req.(*SaveSnapshotRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _ControlService_LoadSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(LoadSnapshotRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ControlServiceServer).LoadSnapshot(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/rpcpb.ControlService/LoadSnapshot",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ControlServiceServer).LoadSnapshot(ctx, req.(*LoadSnapshotRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _ControlService_RemoveSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(RemoveSnapshotRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ControlServiceServer).RemoveSnapshot(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/rpcpb.ControlService/RemoveSnapshot",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ControlServiceServer).RemoveSnapshot(ctx, req.(*RemoveSnapshotRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _ControlService_GetSnapshotNames_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetSnapshotNamesRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ControlServiceServer).GetSnapshotNames(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/rpcpb.ControlService/GetSnapshotNames",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ControlServiceServer).GetSnapshotNames(ctx, req.(*GetSnapshotNamesRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 // ControlService_ServiceDesc is the grpc.ServiceDesc for ControlService service.
 // It's only intended for direct use with grpc.RegisterService,
 // and not to be introspected or modified (even as a copy)
@@ -531,6 +723,14 @@ var ControlService_ServiceDesc = grpc.ServiceDesc{
 			MethodName: "Start",
 			Handler:    _ControlService_Start_Handler,
 		},
+		{
+			MethodName: "CreateBlockchains",
+			Handler:    _ControlService_CreateBlockchains_Handler,
+		},
+		{
+			MethodName: "CreateSubnets",
+			Handler:    _ControlService_CreateSubnets_Handler,
+		},
 		{
 			MethodName: "Health",
 			Handler:    _ControlService_Health_Handler,
@@ -567,6 +767,22 @@ var ControlService_ServiceDesc = grpc.ServiceDesc{
 			MethodName: "SendOutboundMessage",
 			Handler:    _ControlService_SendOutboundMessage_Handler,
 		},
+		{
+			MethodName: "SaveSnapshot",
+			Handler:    _ControlService_SaveSnapshot_Handler,
+		},
+		{
+			MethodName: "LoadSnapshot",
+			Handler:    _ControlService_LoadSnapshot_Handler,
+		},
+		{
+			MethodName: "RemoveSnapshot",
+			Handler:    _ControlService_RemoveSnapshot_Handler,
+		},
+		{
+			MethodName: "GetSnapshotNames",
+			Handler:    _ControlService_GetSnapshotNames_Handler,
+		},
 	},
 	Streams: []grpc.StreamDesc{
 		{

scripts/genmocks.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -e
+
+if ! [[ "$0" =~ scripts/genmocks.sh ]]; then
+  echo "must be run from repository root"
+  exit 255
+fi
+
+mockery --dir api --name Client --output api/mocks/ --filename client.go
+mockery --dir api --name EthClient --output api/mocks/ --filename EthClient.go
+mockery --dir local --name NodeProcess --output local/mocks/ --filename node_process.go
+mockery --dir k8s --name dnsReachableChecker --output k8s/mocks/ --filename dns_checker.go --structname DnsReachableChecker
+
+echo "Successfully generated mock files"

scripts/tests.e2e.sh

@@ -27,58 +27,64 @@ echo "Running e2e tests with:"
 echo VERSION_1: ${VERSION_1}
 echo VERSION_2: ${VERSION_2}
 
-############################
-# download avalanchego
-# https://github.com/ava-labs/avalanchego/releases
-GOARCH=$(go env GOARCH)
-GOOS=$(go env GOOS)
-DOWNLOAD_URL=https://github.com/ava-labs/avalanchego/releases/download/v${VERSION_1}/avalanchego-linux-${GOARCH}-v${VERSION_1}.tar.gz
-DOWNLOAD_PATH=/tmp/avalanchego.tar.gz
-if [[ ${GOOS} == "darwin" ]]; then
-  DOWNLOAD_URL=https://github.com/ava-labs/avalanchego/releases/download/v${VERSION_1}/avalanchego-macos-v${VERSION_1}.zip
-  DOWNLOAD_PATH=/tmp/avalanchego.zip
+if [ ! -f /tmp/avalanchego-v${VERSION_1}/avalanchego ]
+then
+    ############################
+    # download avalanchego
+    # https://github.com/ava-labs/avalanchego/releases
+    GOARCH=$(go env GOARCH)
+    GOOS=$(go env GOOS)
+    DOWNLOAD_URL=https://github.com/ava-labs/avalanchego/releases/download/v${VERSION_1}/avalanchego-linux-${GOARCH}-v${VERSION_1}.tar.gz
+    DOWNLOAD_PATH=/tmp/avalanchego.tar.gz
+    if [[ ${GOOS} == "darwin" ]]; then
+      DOWNLOAD_URL=https://github.com/ava-labs/avalanchego/releases/download/v${VERSION_1}/avalanchego-macos-v${VERSION_1}.zip
+      DOWNLOAD_PATH=/tmp/avalanchego.zip
+    fi
+
+    rm -rf /tmp/avalanchego-v${VERSION_1}
+    rm -rf /tmp/avalanchego-build
+    rm -f ${DOWNLOAD_PATH}
+
+    echo "downloading avalanchego ${VERSION_1} at ${DOWNLOAD_URL}"
+    curl -L ${DOWNLOAD_URL} -o ${DOWNLOAD_PATH}
+
+    echo "extracting downloaded avalanchego"
+    if [[ ${GOOS} == "linux" ]]; then
+      tar xzvf ${DOWNLOAD_PATH} -C /tmp
+    elif [[ ${GOOS} == "darwin" ]]; then
+      unzip ${DOWNLOAD_PATH} -d /tmp/avalanchego-build
+      mv /tmp/avalanchego-build/build /tmp/avalanchego-v${VERSION_1}
+    fi
+    find /tmp/avalanchego-v${VERSION_1}
 fi
 
-rm -rf /tmp/avalanchego-v${VERSION_1}
-rm -rf /tmp/avalanchego-build
-rm -f ${DOWNLOAD_PATH}
+if [ ! -f /tmp/avalanchego-v${VERSION_2}/avalanchego ]
+then
+    ############################
+    # download avalanchego
+    # https://github.com/ava-labs/avalanchego/releases
+    DOWNLOAD_URL=https://github.com/ava-labs/avalanchego/releases/download/v${VERSION_2}/avalanchego-linux-${GOARCH}-v${VERSION_2}.tar.gz
+    if [[ ${GOOS} == "darwin" ]]; then
+      DOWNLOAD_URL=https://github.com/ava-labs/avalanchego/releases/download/v${VERSION_2}/avalanchego-macos-v${VERSION_2}.zip
+      DOWNLOAD_PATH=/tmp/avalanchego.zip
+    fi
 
-echo "downloading avalanchego ${VERSION_1} at ${DOWNLOAD_URL}"
-curl -L ${DOWNLOAD_URL} -o ${DOWNLOAD_PATH}
+    rm -rf /tmp/avalanchego-v${VERSION_2}
+    rm -rf /tmp/avalanchego-build
+    rm -f ${DOWNLOAD_PATH}
 
-echo "extracting downloaded avalanchego"
-if [[ ${GOOS} == "linux" ]]; then
-  tar xzvf ${DOWNLOAD_PATH} -C /tmp
-elif [[ ${GOOS} == "darwin" ]]; then
-  unzip ${DOWNLOAD_PATH} -d /tmp/avalanchego-build
-  mv /tmp/avalanchego-build/build /tmp/avalanchego-v${VERSION_1}
+    echo "downloading avalanchego ${VERSION_2} at ${DOWNLOAD_URL}"
+    curl -L ${DOWNLOAD_URL} -o ${DOWNLOAD_PATH}
+
+    echo "extracting downloaded avalanchego"
+    if [[ ${GOOS} == "linux" ]]; then
+      tar xzvf ${DOWNLOAD_PATH} -C /tmp
+    elif [[ ${GOOS} == "darwin" ]]; then
+      unzip ${DOWNLOAD_PATH} -d /tmp/avalanchego-build
+      mv /tmp/avalanchego-build/build /tmp/avalanchego-v${VERSION_2}
+    fi
+    find /tmp/avalanchego-v${VERSION_2}
 fi
-find /tmp/avalanchego-v${VERSION_1}
-
-############################
-# download avalanchego
-# https://github.com/ava-labs/avalanchego/releases
-DOWNLOAD_URL=https://github.com/ava-labs/avalanchego/releases/download/v${VERSION_2}/avalanchego-linux-${GOARCH}-v${VERSION_2}.tar.gz
-if [[ ${GOOS} == "darwin" ]]; then
-  DOWNLOAD_URL=https://github.com/ava-labs/avalanchego/releases/download/v${VERSION_2}/avalanchego-macos-v${VERSION_2}.zip
-  DOWNLOAD_PATH=/tmp/avalanchego.zip
-fi
-
-rm -rf /tmp/avalanchego-v${VERSION_2}
-rm -rf /tmp/avalanchego-build
-rm -f ${DOWNLOAD_PATH}
-
-echo "downloading avalanchego ${VERSION_2} at ${DOWNLOAD_URL}"
-curl -L ${DOWNLOAD_URL} -o ${DOWNLOAD_PATH}
-
-echo "extracting downloaded avalanchego"
-if [[ ${GOOS} == "linux" ]]; then
-  tar xzvf ${DOWNLOAD_PATH} -C /tmp
-elif [[ ${GOOS} == "darwin" ]]; then
-  unzip ${DOWNLOAD_PATH} -d /tmp/avalanchego-build
-  mv /tmp/avalanchego-build/build /tmp/avalanchego-v${VERSION_2}
-fi
-find /tmp/avalanchego-v${VERSION_2}
 
 ############################
 echo "building runner"
@@ -90,12 +96,19 @@ go install -v github.com/onsi/ginkgo/v2/ginkgo@v2.1.3
 ACK_GINKGO_RC=true ginkgo build ./tests/e2e
 ./tests/e2e/e2e.test --help
 
+snapshots_dir=/tmp/avalanche-network-runner-snapshots-e2e/
+rm -rf $snapshots_dir
+
+killall network.runner || echo
+
 echo "launch local test cluster in the background"
 /tmp/network.runner \
 server \
 --log-level debug \
 --port=":8080" \
+--snapshots-dir=$snapshots_dir \
 --grpc-gateway-port=":8081" &
+#--disable-nodes-output \
 PID=${!}
 
 echo "running e2e tests"

server/custom_vms.go

@@ -15,92 +15,269 @@ import (
 	"github.com/ava-labs/avalanche-network-runner/pkg/color"
 	"github.com/ava-labs/avalanche-network-runner/rpcpb"
 	"github.com/ava-labs/avalanche-network-runner/utils"
+	"github.com/ava-labs/avalanchego/config"
 	"github.com/ava-labs/avalanchego/genesis"
 	"github.com/ava-labs/avalanchego/ids"
 	"github.com/ava-labs/avalanchego/utils/constants"
 	"github.com/ava-labs/avalanchego/utils/units"
 	"github.com/ava-labs/avalanchego/vms/platformvm"
+	"github.com/ava-labs/avalanchego/vms/platformvm/validator"
 	"github.com/ava-labs/avalanchego/vms/secp256k1fx"
 	"github.com/ava-labs/avalanchego/wallet/subnet/primary/common"
 	"go.uber.org/zap"
 )
 
+const (
+	// offset of validation start from current time
+	validationStartOffset = 20 * time.Second
+	// duration for primary network validators
+	validationDuration = 365 * 24 * time.Hour
+	// weight assigned to subnet validators
+	subnetValidatorsWeight = 1000
+	// check period for blockchain logs while waiting for custom VMs to be ready
+	blockchainLogPullFrequency = time.Second
+	// check period while waiting for all validators to be ready
+	waitForValidatorsPullFrequency = time.Second
+)
+
+var defaultPoll = common.WithPollFrequency(100 * time.Millisecond)
+
+type blockchainSpec struct {
+	vmName   string
+	genesis  []byte
+	subnetId *string
+}
+
 // provisions local cluster and install custom VMs if applicable
 // assumes the local cluster is already set up and healthy
-func (lc *localNetwork) installCustomVMs(ctx context.Context) error {
+func (lc *localNetwork) installCustomVMs(
+	ctx context.Context,
+	chainSpecs []blockchainSpec,
+) ([]vmInfo, error) {
 	println()
 	color.Outf("{{blue}}{{bold}}create and install custom VMs{{/}}\n")
 
-	uris := make([]string, 0, len(lc.nodeInfos))
-	for _, i := range lc.nodeInfos {
-		uris = append(uris, i.Uri)
-	}
-	httpRPCEp := uris[0]
-	platformCli := platformvm.NewClient(httpRPCEp)
+	clientURI := lc.nodeInfos[lc.nodeNames[0]].Uri
+	platformCli := platformvm.NewClient(clientURI)
 
-	baseWallet, avaxAssetID, testKeyAddr, err := lc.setupWallet(ctx, httpRPCEp)
+	// wallet needs txs for all previously created subnets
+	pTXs := make(map[ids.ID]*platformvm.Tx)
+	for _, chainSpec := range chainSpecs {
+		// if subnet id for the blockchain is specified, we need to add the subnet id
+		// tx info to the wallet so blockchain creation does not fail
+		// if subnet id is not specified, a new subnet will later be created by using the wallet,
+		// and the wallet will obtain the tx info at that moment
+		if chainSpec.subnetId != nil {
+			subnetID, err := ids.FromString(*chainSpec.subnetId)
+			if err != nil {
+				return nil, err
+			}
+			subnetTxBytes, err := platformCli.GetTx(ctx, subnetID)
+			if err != nil {
+				return nil, fmt.Errorf("tx not found for subnet %q: %w", subnetID.String(), err)
+			}
+			var subnetTx platformvm.Tx
+			if _, err := platformvm.Codec.Unmarshal(subnetTxBytes, &subnetTx); err != nil {
+				return nil, fmt.Errorf("couldn not unmarshal tx for subnet %q: %w", subnetID.String(), err)
+			}
+			pTXs[subnetID] = &subnetTx
+		}
+	}
+
+	baseWallet, avaxAssetID, testKeyAddr, err := setupWallet(ctx, clientURI, pTXs)
 	if err != nil {
-		return err
+		return nil, err
 	}
-	validatorIDs, err := lc.checkValidators(ctx, platformCli, baseWallet, testKeyAddr)
+
+	// get number of subnets to create
+	// for the list of requested blockchains, we count those that have undefined subnet id
+	// that number of subnets will be created and later assigned to those blockchain requests
+	var numSubnets uint32
+	for _, chainSpec := range chainSpecs {
+		if chainSpec.subnetId == nil {
+			numSubnets++
+		}
+	}
+
+	if err := addPrimaryValidators(ctx, lc.nodeInfos, platformCli, baseWallet, testKeyAddr); err != nil {
+		return nil, err
+	}
+
+	if numSubnets > 0 {
+		// add missing subnets, restarting network and waiting for subnet validation to start
+		addedSubnetIDs, err := lc.installSubnets(ctx, numSubnets, baseWallet, testKeyAddr)
+		if err != nil {
+			return nil, err
+		}
+
+		// assign created subnets to blockchain requests with undefined subnet id
+		j := 0
+		for i := range chainSpecs {
+			if chainSpecs[i].subnetId == nil {
+				subnetIDStr := addedSubnetIDs[j].String()
+				chainSpecs[i].subnetId = &subnetIDStr
+				j++
+			}
+		}
+	}
+
+	subnetIDs := []ids.ID{}
+	for _, chainSpec := range chainSpecs {
+		subnetID, err := ids.FromString(*chainSpec.subnetId)
+		if err != nil {
+			return nil, err
+		}
+		subnetIDs = append(subnetIDs, subnetID)
+	}
+	clientURI = lc.nodeInfos[lc.nodeNames[0]].Uri
+	platformCli = platformvm.NewClient(clientURI)
+	if err = addSubnetValidators(ctx, lc.nodeInfos, platformCli, baseWallet, subnetIDs); err != nil {
+		return nil, err
+	}
+
+	blockchainIDs, err := createBlockchains(ctx, chainSpecs, baseWallet, testKeyAddr)
 	if err != nil {
-		return err
-	}
-	if err = lc.createSubnets(ctx, baseWallet, testKeyAddr); err != nil {
-		return err
-	}
-	if err = lc.restartNodesWithWhitelistedSubnets(ctx); err != nil {
-		return err
+		return nil, err
 	}
 
-	println()
-	color.Outf("{{green}}refreshing the wallet with the new URIs after restarts{{/}}\n")
-	uris = make([]string, 0, len(lc.nodeInfos))
-	for _, i := range lc.nodeInfos {
-		uris = append(uris, i.Uri)
-	}
-	httpRPCEp = uris[0]
-	baseWallet.refresh(httpRPCEp)
-	zap.L().Info("set up base wallet with pre-funded test key",
-		zap.String("http-rpc-endpoint", httpRPCEp),
-		zap.String("address", testKeyAddr.String()),
-	)
-
-	if err = lc.addSubnetValidators(ctx, baseWallet, validatorIDs); err != nil {
-		return err
-	}
-	if err = lc.createBlockchains(ctx, baseWallet, testKeyAddr); err != nil {
-		return err
+	chainInfos := make([]vmInfo, len(chainSpecs))
+	for i, chainSpec := range chainSpecs {
+		vmID, err := utils.VMID(chainSpec.vmName)
+		if err != nil {
+			return nil, err
+		}
+		subnetID, err := ids.FromString(*chainSpec.subnetId)
+		if err != nil {
+			return nil, err
+		}
+		chainInfos[i] = vmInfo{
+			info: &rpcpb.CustomVmInfo{
+				VmName:       chainSpec.vmName,
+				VmId:         vmID.String(),
+				SubnetId:     subnetID.String(),
+				BlockchainId: blockchainIDs[i].String(),
+			},
+			subnetID:     subnetID,
+			blockchainID: blockchainIDs[i],
+		}
 	}
 
 	println()
 	color.Outf("{{green}}checking the remaining balance of the base wallet{{/}}\n")
 	balances, err := baseWallet.P().Builder().GetBalance()
 	if err != nil {
-		return err
+		return nil, err
 	}
 	zap.L().Info("base wallet AVAX balance",
 		zap.String("address", testKeyAddr.String()),
 		zap.Uint64("balance", balances[avaxAssetID]),
 	)
 
-	return nil
+	return chainInfos, nil
 }
 
-func (lc *localNetwork) waitForCustomVMsReady(ctx context.Context) error {
+func (lc *localNetwork) setupWalletAndInstallSubnets(
+	ctx context.Context,
+	numSubnets uint32,
+) ([]ids.ID, error) {
+	println()
+	color.Outf("{{blue}}{{bold}}create and install custom VMs{{/}}\n")
+
+	clientURI := lc.nodeInfos[lc.nodeNames[0]].Uri
+	platformCli := platformvm.NewClient(clientURI)
+
+	pTXs := make(map[ids.ID]*platformvm.Tx)
+	baseWallet, avaxAssetID, testKeyAddr, err := setupWallet(ctx, clientURI, pTXs)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := addPrimaryValidators(ctx, lc.nodeInfos, platformCli, baseWallet, testKeyAddr); err != nil {
+		return nil, err
+	}
+
+	// add subnets restarting network if necessary
+	subnetIDs, err := lc.installSubnets(ctx, numSubnets, baseWallet, testKeyAddr)
+	if err != nil {
+		return nil, err
+	}
+
+	clientURI = lc.nodeInfos[lc.nodeNames[0]].Uri
+	platformCli = platformvm.NewClient(clientURI)
+	if err = addSubnetValidators(ctx, lc.nodeInfos, platformCli, baseWallet, subnetIDs); err != nil {
+		return nil, err
+	}
+
+	if err = waitSubnetValidators(ctx, lc.nodeInfos, platformCli, subnetIDs, lc.stopCh); err != nil {
+		return nil, err
+	}
+
+	println()
+	color.Outf("{{green}}checking the remaining balance of the base wallet{{/}}\n")
+	balances, err := baseWallet.P().Builder().GetBalance()
+	if err != nil {
+		return nil, err
+	}
+	zap.L().Info("base wallet AVAX balance",
+		zap.String("address", testKeyAddr.String()),
+		zap.Uint64("balance", balances[avaxAssetID]),
+	)
+
+	return subnetIDs, nil
+}
+
+func (lc *localNetwork) installSubnets(
+	ctx context.Context,
+	numSubnets uint32,
+	baseWallet *refreshableWallet,
+	testKeyAddr ids.ShortID,
+) ([]ids.ID, error) {
+	println()
+	color.Outf("{{blue}}{{bold}}add subnets{{/}}\n")
+
+	subnetIDs, err := createSubnets(ctx, numSubnets, baseWallet, testKeyAddr)
+	if err != nil {
+		return nil, err
+	}
+	if numSubnets > 0 {
+		if err = lc.restartNodesWithWhitelistedSubnets(ctx, subnetIDs); err != nil {
+			return nil, err
+		}
+		println()
+		color.Outf("{{green}}reconnecting the wallet client after restart{{/}}\n")
+		clientURI := lc.nodeInfos[lc.nodeNames[0]].Uri
+		baseWallet.refresh(clientURI)
+		zap.L().Info("set up base wallet with pre-funded test key",
+			zap.String("http-rpc-endpoint", clientURI),
+			zap.String("address", testKeyAddr.String()),
+		)
+	}
+	return subnetIDs, nil
+}
+
+func (lc *localNetwork) waitForCustomVMsReady(
+	ctx context.Context,
+	chainInfos []vmInfo,
+) error {
 	println()
 	color.Outf("{{blue}}{{bold}}waiting for custom VMs to report healthy...{{/}}\n")
 
-	hc := lc.nw.Healthy(ctx)
-	select {
-	case <-lc.stopc:
-		return errAborted
-	case <-ctx.Done():
-		return ctx.Err()
-	case err := <-hc:
+	if err := lc.nw.Healthy(ctx); err != nil {
+		return err
+	}
+
+	subnetIDs := []ids.ID{}
+	for _, chainInfo := range chainInfos {
+		subnetID, err := ids.FromString(chainInfo.info.SubnetId)
 		if err != nil {
 			return err
 		}
+		subnetIDs = append(subnetIDs, subnetID)
+	}
+	clientURI := lc.nodeInfos[lc.nodeNames[0]].Uri
+	platformCli := platformvm.NewClient(clientURI)
+	if err := waitSubnetValidators(ctx, lc.nodeInfos, platformCli, subnetIDs, lc.stopCh); err != nil {
+		return err
 	}
 
 	for nodeName, nodeInfo := range lc.nodeInfos {
@@ -108,7 +285,7 @@ func (lc *localNetwork) waitForCustomVMsReady(ctx context.Context) error {
 			zap.String("node-name", nodeName),
 			zap.String("log-dir", nodeInfo.LogDir),
 		)
-		for _, vmInfo := range lc.customVMIDToInfo {
+		for _, vmInfo := range chainInfos {
 			p := filepath.Join(nodeInfo.LogDir, vmInfo.info.BlockchainId+".log")
 			zap.L().Info("checking log",
 				zap.String("vm-id", vmInfo.info.VmId),
@@ -131,11 +308,11 @@ func (lc *localNetwork) waitForCustomVMsReady(ctx context.Context) error {
 					zap.Error(err),
 				)
 				select {
-				case <-lc.stopc:
+				case <-lc.stopCh:
 					return errAborted
 				case <-ctx.Done():
 					return ctx.Err()
-				case <-time.After(10 * time.Second):
+				case <-time.After(blockchainLogPullFrequency):
 				}
 			}
 		}
@@ -143,21 +320,18 @@ func (lc *localNetwork) waitForCustomVMsReady(ctx context.Context) error {
 
 	println()
 	color.Outf("{{green}}{{bold}}all custom VMs are running!!!{{/}}\n")
-	for _, i := range lc.nodeInfos {
-		for vmID, vmInfo := range lc.customVMIDToInfo {
-			color.Outf("{{blue}}{{bold}}[blockchain RPC for %q] \"%s/ext/bc/%s\"{{/}}\n", vmID, i.GetUri(), vmInfo.blockchainID.String())
-		}
-	}
 
-	lc.customVMsReadycCloseOnce.Do(func() {
-		println()
-		color.Outf("{{green}}{{bold}}all custom VMs are ready!!!{{/}}\n")
-		close(lc.customVMsReadyc)
-	})
+	println()
+	color.Outf("{{green}}{{bold}}all custom VMs are ready on RPC server-side -- network-runner RPC client can poll and query the cluster status{{/}}\n")
+
 	return nil
 }
 
-func (lc *localNetwork) setupWallet(ctx context.Context, httpRPCEp string) (baseWallet *refreshableWallet, avaxAssetID ids.ID, testKeyAddr ids.ShortID, err error) {
+func setupWallet(
+	ctx context.Context,
+	clientURI string,
+	pTXs map[ids.ID]*platformvm.Tx,
+) (baseWallet *refreshableWallet, avaxAssetID ids.ID, testKeyAddr ids.ShortID, err error) {
 	// "local/default/genesis.json" pre-funds "ewoq" key
 	testKey := genesis.EWOQKey
 	testKeyAddr = testKey.PublicKey().Address()
@@ -165,12 +339,12 @@ func (lc *localNetwork) setupWallet(ctx context.Context, httpRPCEp string) (base
 
 	println()
 	color.Outf("{{green}}setting up the base wallet with the seed test key{{/}}\n")
-	baseWallet, err = createRefreshableWallet(ctx, httpRPCEp, testKeychain)
+	baseWallet, err = createRefreshableWallet(ctx, clientURI, testKeychain, pTXs)
 	if err != nil {
 		return nil, ids.Empty, ids.ShortEmpty, err
 	}
 	zap.L().Info("set up base wallet with pre-funded test key",
-		zap.String("http-rpc-endpoint", httpRPCEp),
+		zap.String("http-rpc-endpoint", clientURI),
 		zap.String("address", testKeyAddr.String()),
 	)
 
@@ -186,7 +360,7 @@ func (lc *localNetwork) setupWallet(ctx context.Context, httpRPCEp string) (base
 		return nil, ids.Empty, ids.ShortEmpty, fmt.Errorf("not enough AVAX balance %v in the address %q", bal, testKeyAddr)
 	}
 	zap.L().Info("fetched base wallet AVAX balance",
-		zap.String("http-rpc-endpoint", httpRPCEp),
+		zap.String("http-rpc-endpoint", clientURI),
 		zap.String("address", testKeyAddr.String()),
 		zap.Uint64("balance", bal),
 	)
@@ -194,59 +368,45 @@ func (lc *localNetwork) setupWallet(ctx context.Context, httpRPCEp string) (base
 	return baseWallet, avaxAssetID, testKeyAddr, nil
 }
 
-func (lc *localNetwork) checkValidators(ctx context.Context, platformCli platformvm.Client, baseWallet *refreshableWallet, testKeyAddr ids.ShortID) (validatorIDs []ids.ShortID, err error) {
-	println()
-	color.Outf("{{green}}fetching all nodes from the existing cluster to make sure all nodes are validating the primary network/subnet{{/}}\n")
+// add the nodes in [nodeInfos] as validators of the primary network, in case they are not
+// the validation starts as soon as possible and its duration is as long as possible, that is,
+// it is set to max accepted duration by avalanchego
+func addPrimaryValidators(
+	ctx context.Context,
+	nodeInfos map[string]*rpcpb.NodeInfo,
+	platformCli platformvm.Client,
+	baseWallet *refreshableWallet,
+	testKeyAddr ids.ShortID,
+) error {
+	color.Outf("{{green}}adding the nodes as primary network validators{{/}}\n")
 	// ref. https://docs.avax.network/build/avalanchego-apis/p-chain/#platformgetcurrentvalidators
 	cctx, cancel := createDefaultCtx(ctx)
 	vs, err := platformCli.GetCurrentValidators(cctx, constants.PrimaryNetworkID, nil)
 	cancel()
 	if err != nil {
-		return nil, err
+		return err
 	}
-	curValidators := make(map[string]struct{})
+	curValidators := make(map[ids.NodeID]struct{})
 	for _, v := range vs {
-		va, ok := v.(map[string]interface{})
-		if !ok {
-			return nil, fmt.Errorf("failed to parse validator data: %T %+v", v, v)
-		}
-		nodeID, ok := va["nodeID"].(string)
-		if !ok {
-			return nil, fmt.Errorf("failed to parse validator data: %T %+v", va, va)
-		}
-		curValidators[nodeID] = struct{}{}
-		zap.L().Info("current validator", zap.String("node-id", nodeID))
+		curValidators[v.NodeID] = struct{}{}
 	}
-
-	println()
-	color.Outf("{{green}}adding all nodes as validator for the primary subnet{{/}}\n")
-	validatorIDs = make([]ids.ShortID, 0, len(lc.nodeInfos))
-	for nodeName, nodeInfo := range lc.nodeInfos {
-		nodeID, err := ids.ShortFromPrefixedString(nodeInfo.Id, constants.NodeIDPrefix)
+	for nodeName, nodeInfo := range nodeInfos {
+		nodeID, err := ids.NodeIDFromString(nodeInfo.Id)
 		if err != nil {
-			return nil, err
+			return err
 		}
-		validatorIDs = append(validatorIDs, nodeID)
 
-		_, isValidator := curValidators[nodeInfo.Id]
+		_, isValidator := curValidators[nodeID]
 		if isValidator {
-			zap.L().Info("the node is already validating the primary subnet; skipping",
-				zap.String("node-name", nodeName),
-				zap.String("node-id", nodeInfo.Id),
-			)
 			continue
 		}
 
-		zap.L().Info("adding a node as a validator to the primary subnet",
-			zap.String("node-name", nodeName),
-			zap.String("node-id", nodeID.String()),
-		)
 		cctx, cancel = createDefaultCtx(ctx)
 		txID, err := baseWallet.P().IssueAddValidatorTx(
-			&platformvm.Validator{
+			&validator.Validator{
 				NodeID: nodeID,
-				Start:  uint64(time.Now().Add(10 * time.Second).Unix()),
-				End:    uint64(time.Now().Add(300 * time.Hour).Unix()),
+				Start:  uint64(time.Now().Add(validationStartOffset).Unix()),
+				End:    uint64(time.Now().Add(validationDuration).Unix()),
 				Wght:   1 * units.Avax,
 			},
 			&secp256k1fx.OutputOwners{
@@ -259,7 +419,7 @@ func (lc *localNetwork) checkValidators(ctx context.Context, platformCli platfor
 		)
 		cancel()
 		if err != nil {
-			return nil, err
+			return err
 		}
 		zap.L().Info("added the node as primary subnet validator",
 			zap.String("node-name", nodeName),
@@ -267,21 +427,21 @@ func (lc *localNetwork) checkValidators(ctx context.Context, platformCli platfor
 			zap.String("tx-id", txID.String()),
 		)
 	}
-	return validatorIDs, nil
+	return nil
 }
 
-func (lc *localNetwork) createSubnets(ctx context.Context, baseWallet *refreshableWallet, testKeyAddr ids.ShortID) error {
+func createSubnets(
+	ctx context.Context,
+	numSubnets uint32,
+	baseWallet *refreshableWallet,
+	testKeyAddr ids.ShortID,
+) ([]ids.ID, error) {
 	println()
-	color.Outf("{{green}}creating subnet for each custom VM{{/}}\n")
-	for vmName := range lc.customVMNameToGenesis {
-		vmID, err := utils.VMID(vmName)
-		if err != nil {
-			return err
-		}
-		zap.L().Info("creating subnet tx",
-			zap.String("vm-name", vmName),
-			zap.String("vm-id", vmID.String()),
-		)
+	color.Outf("{{green}}creating %d subnets VM{{/}}\n", numSubnets)
+	subnetIDs := make([]ids.ID, numSubnets)
+	var i uint32
+	for i = 0; i < numSubnets; i++ {
+		zap.L().Info("creating subnet tx")
 		cctx, cancel := createDefaultCtx(ctx)
 		subnetID, err := baseWallet.P().IssueCreateSubnetTx(
 			&secp256k1fx.OutputOwners{
@@ -293,67 +453,50 @@ func (lc *localNetwork) createSubnets(ctx context.Context, baseWallet *refreshab
 		)
 		cancel()
 		if err != nil {
-			return err
-		}
-		zap.L().Info("created subnet tx",
-			zap.String("vm-name", vmName),
-			zap.String("vm-id", vmID.String()),
-			zap.String("subnet-id", subnetID.String()),
-		)
-		lc.customVMIDToInfo[vmID] = vmInfo{
-			info: &rpcpb.CustomVmInfo{
-				VmName:       vmName,
-				VmId:         vmID.String(),
-				SubnetId:     subnetID.String(),
-				BlockchainId: "",
-			},
-			subnetID: subnetID,
+			return nil, err
 		}
+		zap.L().Info("created subnet tx", zap.String("subnet-id", subnetID.String()))
+		subnetIDs[i] = subnetID
 	}
-	return nil
+	return subnetIDs, nil
 }
 
 // TODO: make this "restart" pattern more generic, so it can be used for "Restart" RPC
-func (lc *localNetwork) restartNodesWithWhitelistedSubnets(ctx context.Context) (err error) {
+func (lc *localNetwork) restartNodesWithWhitelistedSubnets(
+	ctx context.Context,
+	subnetIDs []ids.ID,
+) (err error) {
 	println()
-	color.Outf("{{green}}restarting each node with --whitelisted-subnets{{/}}\n")
-	whitelistedSubnetIDs := make([]string, 0, len(lc.customVMIDToInfo))
-	for _, vmInfo := range lc.customVMIDToInfo {
-		whitelistedSubnetIDs = append(whitelistedSubnetIDs, vmInfo.subnetID.String())
+	color.Outf("{{green}}restarting each node with %s{{/}}\n", config.WhitelistedSubnetsKey)
+	whitelistedSubnetIDsMap := map[string]struct{}{}
+	for _, subnetStr := range lc.subnets {
+		whitelistedSubnetIDsMap[subnetStr] = struct{}{}
+	}
+	for _, subnetID := range subnetIDs {
+		whitelistedSubnetIDsMap[subnetID.String()] = struct{}{}
+	}
+	whitelistedSubnetIDs := []string{}
+	for subnetID := range whitelistedSubnetIDsMap {
+		whitelistedSubnetIDs = append(whitelistedSubnetIDs, subnetID)
 	}
 	sort.Strings(whitelistedSubnetIDs)
 	whitelistedSubnets := strings.Join(whitelistedSubnetIDs, ",")
-	for nodeName, v := range lc.nodeInfos {
-		zap.L().Info("updating node info",
-			zap.String("node-name", nodeName),
-			zap.String("whitelisted-subnets", whitelistedSubnets),
-		)
-		v.WhitelistedSubnets = whitelistedSubnets
-		lc.nodeInfos[nodeName] = v
-	}
-	for i := range lc.cfg.NodeConfigs {
-		nodeName := lc.cfg.NodeConfigs[i].Name
 
-		zap.L().Info("updating node config and info",
-			zap.String("node-name", nodeName),
-			zap.String("whitelisted-subnets", whitelistedSubnets),
-		)
-
-		// replace "whitelisted-subnets" flag
-		lc.cfg.NodeConfigs[i].ConfigFile, err = utils.UpdateJSONKey(lc.cfg.NodeConfigs[i].ConfigFile, "whitelisted-subnets", whitelistedSubnets)
+	zap.L().Info("restarting all nodes to whitelist subnet",
+		zap.Strings("whitelisted-subnets", whitelistedSubnetIDs),
+	)
+	for _, nodeName := range lc.nodeNames {
+		node, err := lc.nw.GetNode(nodeName)
 		if err != nil {
 			return err
 		}
 
-		v := lc.nodeInfos[nodeName]
-		v.Config = []byte(lc.cfg.NodeConfigs[i].ConfigFile)
-		lc.nodeInfos[nodeName] = v
-	}
-	zap.L().Info("restarting all nodes to whitelist subnet",
-		zap.Strings("whitelisted-subnets", whitelistedSubnetIDs),
-	)
-	for _, nodeConfig := range lc.cfg.NodeConfigs {
-		nodeName := nodeConfig.Name
+		// replace WhitelistedSubnetsKey flag
+		nodeConfig := node.GetConfig()
+		nodeConfig.ConfigFile, err = utils.SetJSONKey(nodeConfig.ConfigFile, config.WhitelistedSubnetsKey, whitelistedSubnets)
+		if err != nil {
+			return err
+		}
 
 		lc.customVMRestartMu.Lock()
 		zap.L().Info("removing and adding back the node for whitelisted subnets", zap.String("node-name", nodeName))
@@ -361,6 +504,7 @@ func (lc *localNetwork) restartNodesWithWhitelistedSubnets(ctx context.Context)
 			lc.customVMRestartMu.Unlock()
 			return err
 		}
+
 		if _, err := lc.nw.AddNode(nodeConfig); err != nil {
 			lc.customVMRestartMu.Unlock()
 			return err
@@ -373,31 +517,64 @@ func (lc *localNetwork) restartNodesWithWhitelistedSubnets(ctx context.Context)
 		}
 		lc.customVMRestartMu.Unlock()
 	}
+	if err := lc.updateNodeInfo(); err != nil {
+		return err
+	}
 	return nil
 }
 
-func (lc *localNetwork) addSubnetValidators(ctx context.Context, baseWallet *refreshableWallet, validatorIDs []ids.ShortID) error {
-	println()
-	color.Outf("{{green}}adding all nodes as subnet validator for each subnet{{/}}\n")
-	for vmID, vmInfo := range lc.customVMIDToInfo {
-		zap.L().Info("adding all nodes as subnet validator",
-			zap.String("vm-name", vmInfo.info.VmName),
-			zap.String("vm-id", vmID.String()),
-			zap.String("subnet-id", vmInfo.subnetID.String()),
-		)
-		for _, validatorID := range validatorIDs {
+// add the nodes in [nodeInfos] as validators of the given subnets, in case they are not
+// the validation starts as soon as possible and its duration is as long as possible, that is,
+// it ends at the time the primary network validation ends for the node
+func addSubnetValidators(
+	ctx context.Context,
+	nodeInfos map[string]*rpcpb.NodeInfo,
+	platformCli platformvm.Client,
+	baseWallet *refreshableWallet,
+	subnetIDs []ids.ID,
+) error {
+	color.Outf("{{green}}adding the nodes as subnet validators{{/}}\n")
+	for _, subnetID := range subnetIDs {
+		cctx, cancel := createDefaultCtx(ctx)
+		vs, err := platformCli.GetCurrentValidators(cctx, constants.PrimaryNetworkID, nil)
+		cancel()
+		if err != nil {
+			return err
+		}
+		primaryValidatorsEndtime := make(map[ids.NodeID]time.Time)
+		for _, v := range vs {
+			primaryValidatorsEndtime[v.NodeID] = time.Unix(int64(v.EndTime), 0)
+		}
+		cctx, cancel = createDefaultCtx(ctx)
+		vs, err = platformCli.GetCurrentValidators(cctx, subnetID, nil)
+		cancel()
+		if err != nil {
+			return err
+		}
+		subnetValidators := ids.NodeIDSet{}
+		for _, v := range vs {
+			subnetValidators.Add(v.NodeID)
+		}
+		for nodeName, nodeInfo := range nodeInfos {
+			nodeID, err := ids.NodeIDFromString(nodeInfo.Id)
+			if err != nil {
+				return err
+			}
+			isValidator := subnetValidators.Contains(nodeID)
+			if isValidator {
+				continue
+			}
 			cctx, cancel := createDefaultCtx(ctx)
 			txID, err := baseWallet.P().IssueAddSubnetValidatorTx(
-				&platformvm.SubnetValidator{
-					Validator: platformvm.Validator{
-						NodeID: validatorID,
-
+				&validator.SubnetValidator{
+					Validator: validator.Validator{
+						NodeID: nodeID,
 						// reasonable delay in most/slow test environments
-						Start: uint64(time.Now().Add(time.Minute).Unix()),
-						End:   uint64(time.Now().Add(100 * time.Hour).Unix()),
-						Wght:  1000,
+						Start: uint64(time.Now().Add(validationStartOffset).Unix()),
+						End:   uint64(primaryValidatorsEndtime[nodeID].Unix()),
+						Wght:  subnetValidatorsWeight,
 					},
-					Subnet: vmInfo.subnetID,
+					Subnet: subnetID,
 				},
 				common.WithContext(cctx),
 				defaultPoll,
@@ -407,10 +584,9 @@ func (lc *localNetwork) addSubnetValidators(ctx context.Context, baseWallet *ref
 				return err
 			}
 			zap.L().Info("added the node as a subnet validator",
-				zap.String("vm-name", vmInfo.info.VmName),
-				zap.String("vm-id", vmID.String()),
-				zap.String("subnet-id", vmInfo.subnetID.String()),
-				zap.String("node-id", validatorID.String()),
+				zap.String("subnet-id", subnetID.String()),
+				zap.String("node-name", nodeName),
+				zap.String("node-id", nodeID.String()),
 				zap.String("tx-id", txID.String()),
 			)
 		}
@@ -418,12 +594,67 @@ func (lc *localNetwork) addSubnetValidators(ctx context.Context, baseWallet *ref
 	return nil
 }
 
-func (lc *localNetwork) createBlockchains(ctx context.Context, baseWallet *refreshableWallet, testKeyAddr ids.ShortID) error {
+// waits until all nodes in [nodeInfos] start validating the given [subnetIDs]
+func waitSubnetValidators(
+	ctx context.Context,
+	nodeInfos map[string]*rpcpb.NodeInfo,
+	platformCli platformvm.Client,
+	subnetIDs []ids.ID,
+	stopCh chan struct{},
+) error {
+	color.Outf("{{green}}waiting for the nodes to become subnet validators{{/}}\n")
+	for {
+		ready := true
+		for _, subnetID := range subnetIDs {
+			cctx, cancel := createDefaultCtx(ctx)
+			vs, err := platformCli.GetCurrentValidators(cctx, subnetID, nil)
+			cancel()
+			if err != nil {
+				return err
+			}
+			subnetValidators := ids.NodeIDSet{}
+			for _, v := range vs {
+				subnetValidators.Add(v.NodeID)
+			}
+			for _, nodeInfo := range nodeInfos {
+				nodeID, err := ids.NodeIDFromString(nodeInfo.Id)
+				if err != nil {
+					return err
+				}
+				if isValidator := subnetValidators.Contains(nodeID); !isValidator {
+					ready = false
+				}
+			}
+		}
+		if ready {
+			return nil
+		}
+		select {
+		case <-stopCh:
+			return errAborted
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(waitForValidatorsPullFrequency):
+		}
+	}
+}
+
+func createBlockchains(
+	ctx context.Context,
+	chainSpecs []blockchainSpec,
+	baseWallet *refreshableWallet,
+	testKeyAddr ids.ShortID,
+) ([]ids.ID, error) {
 	println()
 	color.Outf("{{green}}creating blockchain for each custom VM{{/}}\n")
-	for vmID, vmInfo := range lc.customVMIDToInfo {
-		vmName := vmInfo.info.VmName
-		vmGenesisBytes := lc.customVMNameToGenesis[vmName]
+	blockchainIDs := make([]ids.ID, len(chainSpecs))
+	for i, chainSpec := range chainSpecs {
+		vmName := chainSpec.vmName
+		vmID, err := utils.VMID(vmName)
+		if err != nil {
+			return nil, err
+		}
+		vmGenesisBytes := chainSpec.genesis
 
 		zap.L().Info("creating blockchain tx",
 			zap.String("vm-name", vmName),
@@ -431,8 +662,12 @@ func (lc *localNetwork) createBlockchains(ctx context.Context, baseWallet *refre
 			zap.Int("genesis-bytes", len(vmGenesisBytes)),
 		)
 		cctx, cancel := createDefaultCtx(ctx)
+		subnetID, err := ids.FromString(*chainSpec.subnetId)
+		if err != nil {
+			return nil, err
+		}
 		blockchainID, err := baseWallet.P().IssueCreateChainTx(
-			vmInfo.subnetID,
+			subnetID,
 			vmGenesisBytes,
 			vmID,
 			nil,
@@ -442,12 +677,10 @@ func (lc *localNetwork) createBlockchains(ctx context.Context, baseWallet *refre
 		)
 		cancel()
 		if err != nil {
-			return err
+			return nil, fmt.Errorf("failure creating blockchain: %w", err)
 		}
 
-		vmInfo.info.BlockchainId = blockchainID.String()
-		vmInfo.blockchainID = blockchainID
-		lc.customVMIDToInfo[vmID] = vmInfo
+		blockchainIDs[i] = blockchainID
 
 		zap.L().Info("created a new blockchain",
 			zap.String("vm-name", vmName),
@@ -455,7 +688,6 @@ func (lc *localNetwork) createBlockchains(ctx context.Context, baseWallet *refre
 			zap.String("blockchain-id", blockchainID.String()),
 		)
 	}
-	return nil
-}
 
-var defaultPoll = common.WithPollFrequency(5 * time.Second)
+	return blockchainIDs, nil
+}

server/network.go

@@ -8,55 +8,72 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"os"
 	"path/filepath"
-	"strings"
+	"sort"
 	"sync"
 
-	"github.com/ava-labs/avalanche-network-runner/api"
 	"github.com/ava-labs/avalanche-network-runner/local"
 	"github.com/ava-labs/avalanche-network-runner/network"
 	"github.com/ava-labs/avalanche-network-runner/pkg/color"
 	"github.com/ava-labs/avalanche-network-runner/rpcpb"
+	"github.com/ava-labs/avalanchego/config"
 	"github.com/ava-labs/avalanchego/ids"
 	"github.com/ava-labs/avalanchego/network/peer"
 	"github.com/ava-labs/avalanchego/utils/constants"
 	"github.com/ava-labs/avalanchego/utils/logging"
+	"go.uber.org/zap"
 )
 
+const (
+	defaultNodeConfig = `{
+		"network-peer-list-gossip-frequency":"250ms",
+		"network-max-reconnect-delay":"1s",
+		"public-ip":"127.0.0.1",
+		"health-check-frequency":"2s",
+		"api-admin-enabled":true,
+		"api-ipcs-enabled":true,
+		"index-enabled":true
+  }`
+)
+
+var ignoreFields = map[string]struct{}{
+	"public-ip":    {},
+	"http-port":    {},
+	"staking-port": {},
+}
+
 type localNetwork struct {
 	logger logging.Logger
 
-	binPath string
-	cfg     network.Config
+	execPath  string
+	pluginDir string
+
+	cfg network.Config
 
 	nw network.Network
 
 	nodeNames []string
+
 	nodeInfos map[string]*rpcpb.NodeInfo
 
+	options localNetworkOptions
+
 	// maps from node name to peer ID to peer object
 	attachedPeers map[string]map[string]peer.Peer
 
-	apiClis map[string]api.Client
+	// map from blockchain ID to blockchain info
+	customVMBlockchainIDToInfo map[ids.ID]vmInfo
 
-	localClusterReadyc          chan struct{} // closed when local network is ready/healthy
-	localClusterReadycCloseOnce sync.Once
+	customVMRestartMu *sync.RWMutex
 
-	// map from VM name to genesis bytes
-	customVMNameToGenesis map[string][]byte
-	// map from VM ID to VM info
-	customVMIDToInfo map[ids.ID]vmInfo
-
-	customVMsReadyc          chan struct{} // closed when subnet installations are complete
-	customVMsReadycCloseOnce sync.Once
-	customVMRestartMu        *sync.RWMutex
-
-	stopc      chan struct{}
-	startDonec chan struct{}
-	startErrc  chan error
+	stopCh         chan struct{}
+	startDoneCh    chan struct{}
+	startErrCh     chan error
+	startCtxCancel context.CancelFunc // allow the Start context to be cancelled
 
 	stopOnce sync.Once
+
+	subnets []string
 }
 
 type vmInfo struct {
@@ -66,22 +83,28 @@ type vmInfo struct {
 }
 
 type localNetworkOptions struct {
-	execPath           string
-	rootDataDir        string
-	numNodes           uint32
-	whitelistedSubnets string
-	logLevel           string
+	execPath            string
+	rootDataDir         string
+	numNodes            uint32
+	whitelistedSubnets  string
+	redirectNodesOutput bool
+	globalNodeConfig    string
 
-	pluginDir    string
-	customVMs    map[string][]byte
+	pluginDir         string
+	customNodeConfigs map[string]string
 	chainConfigs map[string]string
 
 	// to block racey restart while installing custom VMs
 	restartMu *sync.RWMutex
+
+	snapshotsDir string
 }
 
 func newLocalNetwork(opts localNetworkOptions) (*localNetwork, error) {
-	lcfg := logging.DefaultConfig
+	lcfg := logging.Config{
+		DisplayLevel: logging.Info,
+		LogLevel:     logging.Debug,
+	}
 	lcfg.Directory = opts.rootDataDir
 	logFactory := logging.NewFactory(lcfg)
 	logger, err := logFactory.Make("main")
@@ -89,128 +112,366 @@ func newLocalNetwork(opts localNetworkOptions) (*localNetwork, error) {
 		return nil, err
 	}
 
-	logLevel := opts.logLevel
-	if logLevel == "" {
-		logLevel = "INFO"
-	}
-
-	nodeInfos := make(map[string]*rpcpb.NodeInfo)
-	cfg, err := local.NewDefaultConfigNNodes(opts.execPath, opts.numNodes)
-	if err != nil {
-		return nil, err
-	}
-	nodeNames := make([]string, len(cfg.NodeConfigs))
-	for i := range cfg.NodeConfigs {
-		nodeName := fmt.Sprintf("node%d", i+1)
-		logDir := filepath.Join(opts.rootDataDir, nodeName, "log")
-		dbDir := filepath.Join(opts.rootDataDir, nodeName, "db-dir")
-
-		nodeNames[i] = nodeName
-		cfg.NodeConfigs[i].Name = nodeName
-		cfg.NodeConfigs[i].ConfigFile = createConfigFileString(logLevel, logDir, dbDir, opts.pluginDir, opts.whitelistedSubnets)
-		cfg.NodeConfigs[i].ImplSpecificConfig = json.RawMessage(fmt.Sprintf(`{"binaryPath":"%s","redirectStdout":true,"redirectStderr":true}`, opts.execPath))
-		for chain, config := range opts.chainConfigs {
-			if chain != "C" {
-				// TODO: add support for other chains
-				continue
-			}
-			cfg.NodeConfigs[i].CChainConfigFile = config
-		}
-
-		nodeInfos[nodeName] = &rpcpb.NodeInfo{
-			Name:               nodeName,
-			ExecPath:           opts.execPath,
-			Uri:                "",
-			Id:                 "",
-			LogDir:             logDir,
-			DbDir:              dbDir,
-			PluginDir:          opts.pluginDir,
-			WhitelistedSubnets: opts.whitelistedSubnets,
-			Config:             []byte(cfg.NodeConfigs[i].ConfigFile),
-		}
-	}
-
 	return &localNetwork{
 		logger: logger,
 
-		binPath: opts.execPath,
-		cfg:     cfg,
+		execPath: opts.execPath,
+
+		pluginDir: opts.pluginDir,
+
+		options: opts,
 
-		nodeNames:     nodeNames,
-		nodeInfos:     nodeInfos,
-		apiClis:       make(map[string]api.Client),
 		attachedPeers: make(map[string]map[string]peer.Peer),
 
-		localClusterReadyc: make(chan struct{}),
+		customVMBlockchainIDToInfo: make(map[ids.ID]vmInfo),
+		customVMRestartMu:          opts.restartMu,
 
-		customVMNameToGenesis: opts.customVMs,
-		customVMIDToInfo:      make(map[ids.ID]vmInfo),
-		customVMsReadyc:       make(chan struct{}),
-		customVMRestartMu:     opts.restartMu,
+		stopCh:      make(chan struct{}),
+		startDoneCh: make(chan struct{}),
+		startErrCh:  make(chan error, 1),
 
-		stopc:      make(chan struct{}),
-		startDonec: make(chan struct{}),
-		startErrc:  make(chan error, 1),
+		nodeInfos: make(map[string]*rpcpb.NodeInfo),
+		nodeNames: []string{},
 	}, nil
 }
 
-func createConfigFileString(logLevel string, logDir string, dbDir string, pluginDir string, whitelistedSubnets string) string {
-	// need to whitelist subnet ID to create custom VM chain
-	// ref. vms/platformvm/createChain
-	return fmt.Sprintf(`{
-	"network-peer-list-gossip-frequency":"250ms",
-	"network-max-reconnect-delay":"1s",
-	"public-ip":"127.0.0.1",
-	"health-check-frequency":"2s",
-	"api-admin-enabled":true,
-	"api-ipcs-enabled":true,
-	"index-enabled":true,
-	"log-display-level":"%s",
-	"log-level":"%s",
-	"log-dir":"%s",
-	"db-dir":"%s",
-	"plugin-dir":"%s",
-	"whitelisted-subnets":"%s"
-}`,
-		strings.ToUpper(logLevel),
-		strings.ToUpper(logLevel),
-		logDir,
-		dbDir,
-		pluginDir,
-		whitelistedSubnets,
-	)
+func (lc *localNetwork) createConfig() error {
+	cfg, err := local.NewDefaultConfigNNodes(lc.options.execPath, lc.options.numNodes)
+	if err != nil {
+		return err
+	}
+
+	var defaultConfig, globalConfig map[string]interface{}
+	if err := json.Unmarshal([]byte(defaultNodeConfig), &defaultConfig); err != nil {
+		return err
+	}
+
+	if lc.options.globalNodeConfig != "" {
+		if err := json.Unmarshal([]byte(lc.options.globalNodeConfig), &globalConfig); err != nil {
+			return err
+		}
+	}
+
+	for i := range cfg.NodeConfigs {
+		// NOTE: Naming convention for node names is currently `node` + number, i.e. `node1,node2,node3,...node101`
+		nodeName := fmt.Sprintf("node%d", i+1)
+		logDir := filepath.Join(lc.options.rootDataDir, nodeName, "log")
+		dbDir := filepath.Join(lc.options.rootDataDir, nodeName, "db-dir")
+
+		lc.nodeNames = append(lc.nodeNames, nodeName)
+		cfg.NodeConfigs[i].Name = nodeName
+        cfg.NodeConfigs[i].ChainConfigFiles = opts.chainConfigs
+
+		mergedConfig, err := mergeNodeConfig(defaultConfig, globalConfig, lc.options.customNodeConfigs[nodeName])
+		if err != nil {
+			return fmt.Errorf("failed merging provided configs: %w", err)
+		}
+
+		// avalanchego expects buildDir (parent dir of pluginDir) to be provided at cmdline
+		buildDir, err := getBuildDir(lc.execPath, lc.pluginDir)
+		if err != nil {
+			return err
+		}
+		cfg.NodeConfigs[i].ConfigFile, err = createConfigFileString(mergedConfig, logDir, dbDir, buildDir, lc.options.whitelistedSubnets)
+		if err != nil {
+			return err
+		}
+
+		cfg.NodeConfigs[i].BinaryPath = lc.options.execPath
+		cfg.NodeConfigs[i].RedirectStdout = lc.options.redirectNodesOutput
+		cfg.NodeConfigs[i].RedirectStderr = lc.options.redirectNodesOutput
+	}
+
+	lc.cfg = cfg
+	return nil
 }
 
-func (lc *localNetwork) start(ctx context.Context) {
-	defer func() {
-		close(lc.startDonec)
-	}()
+// mergeAndCheckForIgnores takes two maps, merging the two and overriding the first with the second
+// if common entries are found.
+// It also skips some entries which are internal to the runner
+func mergeAndCheckForIgnores(base, override map[string]interface{}) {
+	for k, v := range override {
+		if _, ok := ignoreFields[k]; ok {
+			continue
+		}
+		base[k] = v
+	}
+}
+
+// mergeNodeConfig evaluates the final node config.
+// defaultConfig: map of base config to be applied
+// globalConfig: map of global config provided to be applied to all nodes. Overrides defaultConfig
+// customConfig: a custom config provided to be applied to this node. Overrides globalConfig and defaultConfig
+// returns final map of node config entries
+func mergeNodeConfig(baseConfig map[string]interface{}, globalConfig map[string]interface{}, customConfig string) (map[string]interface{}, error) {
+	mergeAndCheckForIgnores(baseConfig, globalConfig)
+
+	var jsonCustom map[string]interface{}
+	// merge, overwriting entries in default with the global ones
+	if customConfig != "" {
+		if err := json.Unmarshal([]byte(customConfig), &jsonCustom); err != nil {
+			return nil, err
+		}
+		// merge, overwriting entries in default with the custom ones
+		mergeAndCheckForIgnores(baseConfig, jsonCustom)
+	}
+
+	return baseConfig, nil
+}
+
+// generates buildDir from pluginDir, and if not available, from execPath
+// returns error if pluginDir is non empty and invalid
+func getBuildDir(execPath string, pluginDir string) (string, error) {
+	buildDir := ""
+	if execPath != "" {
+		buildDir = filepath.Dir(execPath)
+	}
+	if pluginDir != "" {
+		pluginDir := filepath.Clean(pluginDir)
+		if filepath.Base(pluginDir) != "plugins" {
+			return "", fmt.Errorf("plugin dir %q is not named plugins", pluginDir)
+		}
+		buildDir = filepath.Dir(pluginDir)
+	}
+	return buildDir, nil
+}
+
+// createConfigFileString finalizes the config setup and returns the node config JSON string
+func createConfigFileString(configFileMap map[string]interface{}, logDir string, dbDir string, buildDir string, whitelistedSubnets string) (string, error) {
+	// add (or overwrite, if given) the following entries
+	if configFileMap[config.LogsDirKey] != "" {
+		zap.L().Warn("ignoring config file entry provided; the network runner needs to set its own", zap.String("entry", config.LogsDirKey))
+	}
+	configFileMap[config.LogsDirKey] = logDir
+	if configFileMap[config.DBPathKey] != "" {
+		zap.L().Warn("ignoring config file entry provided; the network runner needs to set its own", zap.String("entry", config.DBPathKey))
+	}
+	configFileMap[config.DBPathKey] = dbDir
+	if buildDir != "" {
+		configFileMap[config.BuildDirKey] = buildDir
+	}
+	// need to whitelist subnet ID to create custom VM chain
+	// ref. vms/platformvm/createChain
+	if whitelistedSubnets != "" {
+		configFileMap[config.WhitelistedSubnetsKey] = whitelistedSubnets
+	}
+
+	finalJSON, err := json.Marshal(configFileMap)
+	if err != nil {
+		return "", err
+	}
+	return string(finalJSON), nil
+}
+
+func (lc *localNetwork) start() error {
+	if err := lc.createConfig(); err != nil {
+		return err
+	}
 
 	color.Outf("{{blue}}{{bold}}create and run local network{{/}}\n")
-	nw, err := local.NewNetwork(lc.logger, lc.cfg, os.TempDir())
+	nw, err := local.NewNetwork(lc.logger, lc.cfg, lc.options.rootDataDir, lc.options.snapshotsDir)
 	if err != nil {
-		lc.startErrc <- err
-		return
+		return err
 	}
 	lc.nw = nw
 
+	// node info is already available
+	if err := lc.updateNodeInfo(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (lc *localNetwork) startWait(
+	argCtx context.Context,
+	chainSpecs []blockchainSpec, // VM name + genesis bytes
+	readyCh chan struct{}, // messaged when initial network is healthy, closed when subnet installations are complete
+) {
+	defer close(lc.startDoneCh)
+
+	// start triggers a series of different time consuming actions
+	// (in case of subnets: create a wallet, create subnets, issue txs, etc.)
+	// We may need to cancel the context, for example if the client hits Ctrl-C
+	var ctx context.Context
+	ctx, lc.startCtxCancel = context.WithCancel(argCtx)
+
 	if err := lc.waitForLocalClusterReady(ctx); err != nil {
-		lc.startErrc <- err
+		lc.startErrCh <- err
 		return
 	}
 
-	if len(lc.customVMNameToGenesis) == 0 {
+	readyCh <- struct{}{}
+
+	lc.createBlockchains(ctx, chainSpecs, readyCh)
+}
+
+func (lc *localNetwork) createBlockchains(
+	argCtx context.Context,
+	chainSpecs []blockchainSpec, // VM name + genesis bytes
+	createBlockchainsReadyCh chan struct{}, // closed when subnet installations are complete
+) {
+	// createBlockchains triggers a series of different time consuming actions
+	// (in case of subnets: create a wallet, create subnets, issue txs, etc.)
+	// We may need to cancel the context, for example if the client hits Ctrl-C
+	var ctx context.Context
+	ctx, lc.startCtxCancel = context.WithCancel(argCtx)
+
+	if len(chainSpecs) == 0 {
 		color.Outf("{{orange}}{{bold}}custom VM not specified, skipping installation and its health checks...{{/}}\n")
 		return
 	}
-	if err := lc.installCustomVMs(ctx); err != nil {
-		lc.startErrc <- err
+
+	if err := lc.waitForLocalClusterReady(ctx); err != nil {
+		lc.startErrCh <- err
 		return
 	}
-	if err := lc.waitForCustomVMsReady(ctx); err != nil {
-		lc.startErrc <- err
+
+	chainInfos, err := lc.installCustomVMs(ctx, chainSpecs)
+	if err != nil {
+		lc.startErrCh <- err
 		return
 	}
+
+	if err := lc.waitForCustomVMsReady(ctx, chainInfos); err != nil {
+		lc.startErrCh <- err
+		return
+	}
+
+	if err := lc.updateSubnetInfo(ctx); err != nil {
+		lc.startErrCh <- err
+		return
+	}
+
+	close(createBlockchainsReadyCh)
+}
+
+func (lc *localNetwork) createSubnets(
+	argCtx context.Context,
+	numSubnets uint32,
+	createSubnetsReadyCh chan struct{}, // closed when subnet installations are complete
+) {
+	// start triggers a series of different time consuming actions
+	// (in case of subnets: create a wallet, create subnets, issue txs, etc.)
+	// We may need to cancel the context, for example if the client hits Ctrl-C
+	var ctx context.Context
+	ctx, lc.startCtxCancel = context.WithCancel(argCtx)
+
+	if numSubnets == 0 {
+		color.Outf("{{orange}}{{bold}}no subnets specified...{{/}}\n")
+		return
+	}
+
+	if err := lc.waitForLocalClusterReady(ctx); err != nil {
+		lc.startErrCh <- err
+		return
+	}
+
+	_, err := lc.setupWalletAndInstallSubnets(ctx, numSubnets)
+	if err != nil {
+		lc.startErrCh <- err
+		return
+	}
+
+	if err := lc.waitForLocalClusterReady(ctx); err != nil {
+		lc.startErrCh <- err
+		return
+	}
+
+	if err := lc.updateSubnetInfo(ctx); err != nil {
+		lc.startErrCh <- err
+	}
+
+	color.Outf("{{green}}{{bold}}finish adding subnets{{/}}\n")
+
+	close(createSubnetsReadyCh)
+}
+
+func (lc *localNetwork) loadSnapshot(
+	ctx context.Context,
+	snapshotName string,
+) error {
+	color.Outf("{{blue}}{{bold}}create and run local network from snapshot{{/}}\n")
+
+	buildDir, err := getBuildDir(lc.execPath, lc.pluginDir)
+	if err != nil {
+		return err
+	}
+
+	nw, err := local.NewNetworkFromSnapshot(
+		lc.logger,
+		snapshotName,
+		lc.options.rootDataDir,
+		lc.options.snapshotsDir,
+		lc.execPath,
+		buildDir,
+	)
+	if err != nil {
+		return err
+	}
+	lc.nw = nw
+
+	// node info is already available
+	if err := lc.updateNodeInfo(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (lc *localNetwork) loadSnapshotWait(ctx context.Context, loadSnapshotReadyCh chan struct{}) {
+	defer close(lc.startDoneCh)
+	if err := lc.waitForLocalClusterReady(ctx); err != nil {
+		lc.startErrCh <- err
+		return
+	}
+	if err := lc.updateSubnetInfo(ctx); err != nil {
+		lc.startErrCh <- err
+		return
+	}
+	close(loadSnapshotReadyCh)
+}
+
+func (lc *localNetwork) updateSubnetInfo(ctx context.Context) error {
+	node, err := lc.nw.GetNode(lc.nodeNames[0])
+	if err != nil {
+		return err
+	}
+	blockchains, err := node.GetAPIClient().PChainAPI().GetBlockchains(ctx)
+	if err != nil {
+		return err
+	}
+	for _, blockchain := range blockchains {
+		if blockchain.Name != "C-Chain" && blockchain.Name != "X-Chain" {
+			lc.customVMBlockchainIDToInfo[blockchain.ID] = vmInfo{
+				info: &rpcpb.CustomVmInfo{
+					VmName:       blockchain.Name,
+					VmId:         blockchain.VMID.String(),
+					SubnetId:     blockchain.SubnetID.String(),
+					BlockchainId: blockchain.ID.String(),
+				},
+				subnetID:     blockchain.SubnetID,
+				blockchainID: blockchain.ID,
+			}
+		}
+	}
+	subnets, err := node.GetAPIClient().PChainAPI().GetSubnets(ctx, nil)
+	if err != nil {
+		return err
+	}
+	lc.subnets = []string{}
+	for _, subnet := range subnets {
+		if subnet.ID != constants.PlatformChainID {
+			lc.subnets = append(lc.subnets, subnet.ID.String())
+		}
+	}
+	for _, nodeName := range lc.nodeNames {
+		nodeInfo := lc.nodeInfos[nodeName]
+		for blockchainID, vmInfo := range lc.customVMBlockchainIDToInfo {
+			color.Outf("{{blue}}{{bold}}[blockchain RPC for %q] \"%s/ext/bc/%s\"{{/}}\n", vmInfo.info.VmId, nodeInfo.GetUri(), blockchainID)
+		}
+	}
+	return nil
 }
 
 var errAborted = errors.New("aborted")
@@ -218,44 +479,83 @@ var errAborted = errors.New("aborted")
 func (lc *localNetwork) waitForLocalClusterReady(ctx context.Context) error {
 	color.Outf("{{blue}}{{bold}}waiting for all nodes to report healthy...{{/}}\n")
 
-	hc := lc.nw.Healthy(ctx)
-	select {
-	case <-lc.stopc:
-		return errAborted
-	case <-ctx.Done():
-		return ctx.Err()
-	case err := <-hc:
-		if err != nil {
-			return err
-		}
+	if err := lc.nw.Healthy(ctx); err != nil {
+		return err
 	}
 
+	for _, name := range lc.nodeNames {
+		nodeInfo := lc.nodeInfos[name]
+		color.Outf("{{cyan}}%s: node ID %q, URI %q{{/}}\n", name, nodeInfo.Id, nodeInfo.Uri)
+	}
+	return nil
+}
+
+func (lc *localNetwork) updateNodeInfo() error {
 	nodes, err := lc.nw.GetAllNodes()
 	if err != nil {
 		return err
 	}
-	for name, node := range nodes {
-		uri := fmt.Sprintf("http://%s:%d", node.GetURL(), node.GetAPIPort())
-		nodeID := node.GetNodeID().PrefixedString(constants.NodeIDPrefix)
-
-		lc.nodeInfos[name].Uri = uri
-		lc.nodeInfos[name].Id = nodeID
-
-		lc.apiClis[name] = node.GetAPIClient()
-		color.Outf("{{cyan}}%s: node ID %q, URI %q{{/}}\n", name, nodeID, uri)
+	nodeNames := []string{}
+	for name := range nodes {
+		nodeNames = append(nodeNames, name)
 	}
+	sort.Strings(nodeNames)
+	lc.nodeNames = nodeNames
+	lc.nodeInfos = make(map[string]*rpcpb.NodeInfo)
+	for _, name := range lc.nodeNames {
+		node := nodes[name]
+		configFile := []byte(node.GetConfigFile())
+		var pluginDir string
+		var whitelistedSubnets string
+		var configFileMap map[string]interface{}
+		if err := json.Unmarshal(configFile, &configFileMap); err != nil {
+			return err
+		}
+		whitelistedSubnetsIntf, ok := configFileMap[config.WhitelistedSubnetsKey]
+		if ok {
+			whitelistedSubnets, ok = whitelistedSubnetsIntf.(string)
+			if !ok {
+				return fmt.Errorf("unexpected type for %q expected string got %T", config.WhitelistedSubnetsKey, whitelistedSubnetsIntf)
+			}
+		}
+		buildDir := node.GetBuildDir()
+		if buildDir != "" {
+			pluginDir = filepath.Join(buildDir, "plugins")
+		}
 
-	lc.localClusterReadycCloseOnce.Do(func() {
-		close(lc.localClusterReadyc)
-	})
+		lc.nodeInfos[name] = &rpcpb.NodeInfo{
+			Name:               node.GetName(),
+			Uri:                fmt.Sprintf("http://%s:%d", node.GetURL(), node.GetAPIPort()),
+			Id:                 node.GetNodeID().String(),
+			ExecPath:           node.GetBinaryPath(),
+			LogDir:             node.GetLogsDir(),
+			DbDir:              node.GetDbDir(),
+			Config:             []byte(node.GetConfigFile()),
+			PluginDir:          pluginDir,
+			WhitelistedSubnets: whitelistedSubnets,
+		}
+
+		// update default exec and pluginDir if empty (snapshots started without this params)
+		if lc.execPath == "" {
+			lc.execPath = node.GetBinaryPath()
+		}
+		if lc.pluginDir == "" {
+			lc.pluginDir = pluginDir
+		}
+
+	}
 	return nil
 }
 
 func (lc *localNetwork) stop(ctx context.Context) {
 	lc.stopOnce.Do(func() {
-		close(lc.stopc)
+		close(lc.stopCh)
+		// cancel possible concurrent still running start
+		if lc.startCtxCancel != nil {
+			lc.startCtxCancel()
+		}
 		serr := lc.nw.Stop(ctx)
-		<-lc.startDonec
+		<-lc.startDoneCh
 		color.Outf("{{red}}{{bold}}terminated network{{/}} (error %v)\n", serr)
 	})
 }

server/network_test.go

@@ -0,0 +1,154 @@
+package server
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEvalConfig(t *testing.T) {
+	assert := assert.New(t)
+
+	// test using the default config...
+	tLogDir := "/tmp/log"
+	tDbDir := "/tmp/db"
+	tWhitelistedSubnets := "someSubnet"
+	tPluginDir := "/tmp/plugins"
+
+	var defaultConfig, globalConfig map[string]interface{}
+
+	err := json.Unmarshal([]byte(defaultNodeConfig), &defaultConfig)
+	assert.NoError(err)
+
+	config, err := mergeNodeConfig(defaultConfig, globalConfig, "")
+	assert.NoError(err)
+	finalJSON, err := createConfigFileString(config, tLogDir, tDbDir, tPluginDir, tWhitelistedSubnets)
+	assert.NoError(err)
+
+	var controlMap map[string]interface{}
+	err = json.Unmarshal([]byte(finalJSON), &controlMap)
+	assert.NoError(err)
+
+	var test1Map map[string]interface{}
+	err = json.Unmarshal([]byte(defaultNodeConfig), &test1Map)
+	assert.NoError(err)
+	// ...all default config entries should still be there...
+	for k, v := range test1Map {
+		assert.Equal(controlMap[k], v)
+	}
+	// ...as well as additional ones
+	assert.Equal(controlMap["log-level"], defaultConfig["log-level"])
+	assert.Equal(controlMap["log-dir"], tLogDir)
+	assert.Equal(controlMap["db-dir"], tDbDir)
+	assert.Equal(controlMap["whitelisted-subnets"], tWhitelistedSubnets)
+
+	// now test a global provided config
+	globalConfigJSON := `{
+		"log-dir":"/home/user/logs",
+		"db-dir":"/home/user/db",
+		"plugin-dir":"/home/user/plugins",
+		"log-display-level":"debug",
+		"log-level":"debug",
+		"whitelisted-subnets":"otherSubNets",
+		"index-enabled":false,
+		"public-ip":"192.168.0.1",
+		"network-id":999,
+		"http-port":777,
+		"staking-port":555,
+		"tx-fee":9999999
+		}`
+
+	err = json.Unmarshal([]byte(globalConfigJSON), &globalConfig)
+	assert.NoError(err)
+
+	config, err = mergeNodeConfig(defaultConfig, globalConfig, "")
+	assert.NoError(err)
+	finalJSON, err = createConfigFileString(config, tLogDir, tDbDir, tPluginDir, tWhitelistedSubnets)
+	assert.NoError(err)
+
+	err = json.Unmarshal([]byte(finalJSON), &controlMap)
+	assert.NoError(err)
+
+	// the custom ones should be there...
+	assert.Equal(controlMap["index-enabled"], false)
+	assert.Equal(controlMap["network-id"], float64(999))
+	assert.Equal(controlMap["tx-fee"], float64(9999999))
+	// ...as well as the common additional ones
+	assert.Equal(controlMap["log-level"], "debug")
+	assert.Equal(controlMap["log-dir"], tLogDir)
+	assert.Equal(controlMap["db-dir"], tDbDir)
+	assert.Equal(controlMap["whitelisted-subnets"], tWhitelistedSubnets)
+
+	// these ones should be ignored as they are hard-set by the code and required by the runner
+	assert.Equal(controlMap["public-ip"], "127.0.0.1")
+	assert.NotEqual(controlMap["http-port"], 777)
+	assert.NotEqual(controlMap["staking-port"], 555)
+	// same test but as custom only - should have same effect
+	customConfigJSON := globalConfigJSON
+	config, err = mergeNodeConfig(defaultConfig, map[string]interface{}{}, customConfigJSON)
+	assert.NoError(err)
+	finalJSON, err = createConfigFileString(config, tLogDir, tDbDir, tPluginDir, tWhitelistedSubnets)
+	assert.NoError(err)
+
+	err = json.Unmarshal([]byte(finalJSON), &controlMap)
+	assert.NoError(err)
+
+	// the custom ones should be there...
+	assert.Equal(controlMap["index-enabled"], false)
+	assert.Equal(controlMap["network-id"], float64(999))
+	assert.Equal(controlMap["tx-fee"], float64(9999999))
+	// ...as well as the common additional ones
+	assert.Equal(controlMap["log-level"], "debug")
+	assert.Equal(controlMap["log-dir"], tLogDir)
+	assert.Equal(controlMap["db-dir"], tDbDir)
+	assert.Equal(controlMap["whitelisted-subnets"], tWhitelistedSubnets)
+	// these ones should be ignored as they are hard-set by the code and required by the runner
+	assert.Equal(controlMap["public-ip"], "127.0.0.1")
+	assert.NotEqual(controlMap["http-port"], 777)
+	assert.NotEqual(controlMap["staking-port"], 555)
+
+	// finally a combined one with custom and global
+	// newGlobalConfigJSON represents the global config, globalConfigJSON the custom. custom should override global
+	newGlobalConfigJSON := `{
+		"log-dir":"/home/user/logs",
+		"db-dir":"/home/user/db",
+		"plugin-dir":"/home/user/plugins",
+		"log-display-level":"debug",
+		"log-level":"info",
+		"whitelisted-subnets":"otherSubNets",
+		"index-enabled":false,
+		"public-ip":"192.168.2.111",
+		"network-id":888,
+		"tx-fee":9999999,
+		"staking-port":11111,
+		"http-port":5555,
+		"uptime-requirement":98.5
+		}`
+	err = json.Unmarshal([]byte(newGlobalConfigJSON), &globalConfig)
+	assert.NoError(err)
+
+	config, err = mergeNodeConfig(defaultConfig, globalConfig, customConfigJSON)
+	assert.NoError(err)
+	finalJSON, err = createConfigFileString(config, tLogDir, tDbDir, tPluginDir, tWhitelistedSubnets)
+	assert.NoError(err)
+
+	err = json.Unmarshal([]byte(finalJSON), &controlMap)
+	assert.NoError(err)
+
+	// the custom ones should be there...
+	assert.Equal(controlMap["index-enabled"], false)
+	assert.Equal(controlMap["network-id"], float64(999))
+	assert.Equal(controlMap["tx-fee"], float64(9999999))
+	// ...as well as the common additional ones
+	assert.Equal(controlMap["log-level"], "debug")
+	assert.Equal(controlMap["log-dir"], tLogDir)
+	assert.Equal(controlMap["db-dir"], tDbDir)
+	assert.Equal(controlMap["whitelisted-subnets"], tWhitelistedSubnets)
+	// ...as well as the ones only in the global config
+	assert.Equal(controlMap["uptime-requirement"], float64(98.5))
+	// these ones should be ignored as they are hard-set by the code and required by the runner
+	assert.Equal(controlMap["public-ip"], "127.0.0.1")
+	assert.NotEqual(controlMap["staking-port"], float64(11111))
+	assert.NotEqual(controlMap["http-port"], float64(5555))
+}

server/server.go

@@ -11,7 +11,6 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
-	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
@@ -37,9 +36,13 @@ import (
 )
 
 type Config struct {
-	Port        string
-	GwPort      string
-	DialTimeout time.Duration
+	Port   string
+	GwPort string
+	// true to disable grpc-gateway server
+	GwDisabled          bool
+	DialTimeout         time.Duration
+	RedirectNodesOutput bool
+	SnapshotsDir        string
 }
 
 type Server interface {
@@ -84,8 +87,11 @@ var (
 )
 
 const (
-	MinNodes     uint32 = 1
-	DefaultNodes uint32 = 5
+	MinNodes            uint32 = 1
+	DefaultNodes        uint32 = 5
+	StopOnSignalTimeout        = 2 * time.Second
+
+	rootDataDirPrefix = "network-runner-root-data"
 )
 
 func New(cfg Config) (Server, error) {
@@ -97,8 +103,7 @@ func New(cfg Config) (Server, error) {
 	if err != nil {
 		return nil, err
 	}
-	gwMux := runtime.NewServeMux()
-	return &server{
+	srv := &server{
 		cfg: cfg,
 
 		closed: make(chan struct{}),
@@ -106,16 +111,20 @@ func New(cfg Config) (Server, error) {
 		ln:         ln,
 		gRPCServer: grpc.NewServer(),
 
-		gwMux: gwMux,
-		gwServer: &http.Server{
-			Addr:    cfg.GwPort,
-			Handler: gwMux,
-		},
-
 		mu: new(sync.RWMutex),
-	}, nil
+	}
+	if !cfg.GwDisabled {
+		srv.gwMux = runtime.NewServeMux()
+		srv.gwServer = &http.Server{
+			Addr:    cfg.GwPort,
+			Handler: srv.gwMux,
+		}
+	}
+
+	return srv, nil
 }
 
+// Blocking call until server listeners return.
 func (s *server) Run(rootCtx context.Context) (err error) {
 	s.rootCtx = rootCtx
 	s.gRPCRegisterOnce.Do(func() {
@@ -130,58 +139,74 @@ func (s *server) Run(rootCtx context.Context) (err error) {
 	}()
 
 	gwErrc := make(chan error)
-	go func() {
-		zap.L().Info("dialing gRPC server", zap.String("port", s.cfg.Port))
-		ctx, cancel := context.WithTimeout(rootCtx, s.cfg.DialTimeout)
-		gwConn, err := grpc.DialContext(
-			ctx,
-			"0.0.0.0"+s.cfg.Port,
-			grpc.WithBlock(),
-			grpc.WithTransportCredentials(insecure.NewCredentials()),
-		)
-		cancel()
-		if err != nil {
-			gwErrc <- err
-			return
-		}
-		defer gwConn.Close()
+	if s.cfg.GwDisabled {
+		zap.L().Info("gRPC gateway server is disabled")
+	} else {
+		go func() {
+			zap.L().Info("dialing gRPC server for gRPC gateway", zap.String("port", s.cfg.Port))
+			ctx, cancel := context.WithTimeout(rootCtx, s.cfg.DialTimeout)
+			gwConn, err := grpc.DialContext(
+				ctx,
+				"0.0.0.0"+s.cfg.Port,
+				grpc.WithBlock(),
+				grpc.WithTransportCredentials(insecure.NewCredentials()),
+			)
+			cancel()
+			if err != nil {
+				gwErrc <- err
+				return
+			}
+			defer gwConn.Close()
 
-		if err := rpcpb.RegisterPingServiceHandler(rootCtx, s.gwMux, gwConn); err != nil {
-			gwErrc <- err
-			return
-		}
-		if err := rpcpb.RegisterControlServiceHandler(rootCtx, s.gwMux, gwConn); err != nil {
-			gwErrc <- err
-			return
-		}
+			if err := rpcpb.RegisterPingServiceHandler(rootCtx, s.gwMux, gwConn); err != nil {
+				gwErrc <- err
+				return
+			}
+			if err := rpcpb.RegisterControlServiceHandler(rootCtx, s.gwMux, gwConn); err != nil {
+				gwErrc <- err
+				return
+			}
 
-		zap.L().Info("serving gRPC gateway", zap.String("port", s.cfg.GwPort))
-		gwErrc <- s.gwServer.ListenAndServe()
-	}()
+			zap.L().Info("serving gRPC gateway", zap.String("port", s.cfg.GwPort))
+			gwErrc <- s.gwServer.ListenAndServe()
+		}()
+	}
 
 	select {
 	case <-rootCtx.Done():
 		zap.L().Warn("root context is done")
 
-		zap.L().Warn("closed gRPC gateway server", zap.Error(s.gwServer.Close()))
-		<-gwErrc
+		if !s.cfg.GwDisabled {
+			zap.L().Warn("closed gRPC gateway server", zap.Error(s.gwServer.Close()))
+			<-gwErrc
+		}
 
 		s.gRPCServer.Stop()
 		zap.L().Warn("closed gRPC server")
 		<-gRPCErrc
+		zap.L().Warn("gRPC terminated")
 
 	case err = <-gRPCErrc:
 		zap.L().Warn("gRPC server failed", zap.Error(err))
-		zap.L().Warn("closed gRPC gateway server", zap.Error(s.gwServer.Close()))
-		<-gwErrc
+		if !s.cfg.GwDisabled {
+			zap.L().Warn("closed gRPC gateway server", zap.Error(s.gwServer.Close()))
+			<-gwErrc
+		}
 
-	case err = <-gwErrc:
+	case err = <-gwErrc: // if disabled, this will never be selected
 		zap.L().Warn("gRPC gateway server failed", zap.Error(err))
 		s.gRPCServer.Stop()
 		zap.L().Warn("closed gRPC server")
 		<-gRPCErrc
 	}
 
+	if s.network != nil {
+		stopCtx, stopCtxCancel := context.WithTimeout(context.Background(), StopOnSignalTimeout)
+		defer stopCtxCancel()
+		s.network.stop(stopCtx)
+		zap.L().Warn("network stopped")
+	}
+
 	s.closeOnce.Do(func() {
 		close(s.closed)
 	})
@@ -213,19 +238,19 @@ func (s *server) Start(ctx context.Context, req *rpcpb.StartRequest) (*rpcpb.Sta
 	if *req.NumNodes < MinNodes {
 		return nil, ErrNotEnoughNodesForStart
 	}
-	customVMs := make(map[string][]byte)
-	if req.GetPluginDir() == "" {
-		if len(req.GetCustomVms()) > 0 {
-			return nil, ErrPluginDirEmptyButCustomVMsNotEmpty
-		}
-		if err := utils.CheckExecPluginPaths(req.GetExecPath(), "", ""); err != nil {
-			return nil, err
-		}
-	} else {
-		if len(req.GetCustomVms()) == 0 {
-			return nil, ErrPluginDirNonEmptyButCustomVMsEmpty
-		}
-		zap.L().Info("non-empty plugin dir", zap.String("plugin-dir", req.GetPluginDir()))
+	if err := utils.CheckExecPath(req.GetExecPath()); err != nil {
+		return nil, err
+	}
+	pluginDir := ""
+	if req.GetPluginDir() != "" {
+		pluginDir = req.GetPluginDir()
+	}
+	if pluginDir == "" {
+		pluginDir = filepath.Join(filepath.Dir(req.GetExecPath()), "plugins")
+	}
+	chainSpecs := []blockchainSpec{}
+	if len(req.GetCustomVms()) > 0 {
+		zap.L().Info("plugin dir", zap.String("plugin-dir", pluginDir))
 		for vmName, vmGenesisFilePath := range req.GetCustomVms() {
 			zap.L().Info("checking custom VM ID before installation", zap.String("vm-id", vmName))
 			vmID, err := utils.VMID(vmName)
@@ -236,24 +261,22 @@ func (s *server) Start(ctx context.Context, req *rpcpb.StartRequest) (*rpcpb.Sta
 				)
 				return nil, ErrInvalidVMName
 			}
-			if err := utils.CheckExecPluginPaths(
-				req.GetExecPath(),
-				filepath.Join(req.GetPluginDir(), vmID.String()),
+			if err := utils.CheckPluginPaths(
+				filepath.Join(pluginDir, vmID.String()),
 				vmGenesisFilePath,
 			); err != nil {
 				return nil, err
 			}
-			b, err := ioutil.ReadFile(vmGenesisFilePath)
+			b, err := os.ReadFile(vmGenesisFilePath)
 			if err != nil {
 				return nil, err
 			}
-			customVMs[vmName] = b
+			chainSpecs = append(chainSpecs, blockchainSpec{
+				vmName:  vmName,
+				genesis: b,
+			})
 		}
 	}
-	pluginDir := ""
-	if req.GetPluginDir() != "" {
-		pluginDir = req.GetPluginDir()
-	}
 
 	s.mu.Lock()
 	defer s.mu.Unlock()
@@ -269,14 +292,18 @@ func (s *server) Start(ctx context.Context, req *rpcpb.StartRequest) (*rpcpb.Sta
 		whitelistedSubnets = req.GetWhitelistedSubnets()
 		rootDataDir        = req.GetRootDataDir()
 		pid                = int32(os.Getpid())
-		logLevel           = req.GetLogLevel()
+		globalNodeConfig   = req.GetGlobalNodeConfig()
+		customNodeConfigs  = req.GetCustomNodeConfigs()
 		err                error
 	)
+
 	if len(rootDataDir) == 0 {
-		rootDataDir, err = ioutil.TempDir(os.TempDir(), "network-runner-root-data")
-		if err != nil {
-			return nil, err
-		}
+		rootDataDir = os.TempDir()
+	}
+	rootDataDir = filepath.Join(rootDataDir, rootDataDirPrefix)
+	rootDataDir, err = utils.MkDirWithTimestamp(rootDataDir)
+	if err != nil {
+		return nil, err
 	}
 
 	s.clusterInfo = &rpcpb.ClusterInfo{
@@ -293,20 +320,27 @@ func (s *server) Start(ctx context.Context, req *rpcpb.StartRequest) (*rpcpb.Sta
 		zap.String("rootDataDir", rootDataDir),
 		zap.String("pluginDir", pluginDir),
 		zap.Any("chainConfigs", req.ChainConfigs),
+		zap.String("defaultNodeConfig", globalNodeConfig),
 	)
 
 	if s.network != nil {
 		return nil, ErrAlreadyBootstrapped
 	}
 
+	if len(customNodeConfigs) > 0 {
+		zap.L().Warn("custom node configs have been provided; ignoring the 'number-of-nodes' parameter and setting it to", zap.Int("numNodes", len(customNodeConfigs)))
+		numNodes = uint32(len(customNodeConfigs))
+	}
+
 	s.network, err = newLocalNetwork(localNetworkOptions{
-		execPath:           execPath,
-		rootDataDir:        rootDataDir,
-		numNodes:           numNodes,
-		whitelistedSubnets: whitelistedSubnets,
-		logLevel:           logLevel,
-		pluginDir:          pluginDir,
-		customVMs:          customVMs,
+		execPath:            execPath,
+		rootDataDir:         rootDataDir,
+		numNodes:            numNodes,
+		whitelistedSubnets:  whitelistedSubnets,
+		redirectNodesOutput: s.cfg.RedirectNodesOutput,
+		pluginDir:           pluginDir,
+		globalNodeConfig:    globalNodeConfig,
+		customNodeConfigs:   customNodeConfigs,
 		chainConfigs:       req.ChainConfigs,
 
 		// to block racey restart
@@ -314,64 +348,212 @@ func (s *server) Start(ctx context.Context, req *rpcpb.StartRequest) (*rpcpb.Sta
 		// so it would not deadlock with the acquired lock
 		// in this "Start" method
 		restartMu: s.mu,
+
+		snapshotsDir: s.cfg.SnapshotsDir,
 	})
 	if err != nil {
+		s.network = nil
+		s.clusterInfo = nil
+		return nil, err
+	}
+
+	if err := s.network.start(); err != nil {
+		s.network = nil
+		s.clusterInfo = nil
 		return nil, err
 	}
 
 	// start non-blocking to install local cluster + custom VMs (if applicable)
 	// the user is expected to poll cluster status
-	go s.network.start(ctx)
+	readyCh := make(chan struct{})
+	go s.network.startWait(ctx, chainSpecs, readyCh)
 
 	// update cluster info non-blocking
 	// the user is expected to poll this latest information
 	// to decide cluster/subnet readiness
 	go func() {
-		zap.L().Info("waiting for local cluster readiness")
-		select {
-		case <-s.closed:
-			return
-		case <-s.network.stopc:
-			// TODO: fix race from shutdown
-			return
-		case serr := <-s.network.startErrc:
-			zap.L().Warn("start failed to complete", zap.Error(serr))
-			panic(serr)
-		case <-s.network.localClusterReadyc:
-			s.mu.Lock()
-			s.clusterInfo.NodeNames = s.network.nodeNames
-			s.clusterInfo.NodeInfos = s.network.nodeInfos
-			s.clusterInfo.Healthy = true
-			s.mu.Unlock()
-		}
-
+		s.waitChAndUpdateClusterInfo("waiting for local cluster readiness", readyCh, false)
 		if len(req.GetCustomVms()) == 0 {
 			zap.L().Info("no custom VM installation request, skipping its readiness check")
 		} else {
-			zap.L().Info("waiting for custom VMs readiness")
-			select {
-			case <-s.closed:
-				return
-			case <-s.network.stopc:
-				return
-			case serr := <-s.network.startErrc:
-				zap.L().Warn("start custom VMs failed to complete", zap.Error(serr))
-				panic(serr)
-			case <-s.network.customVMsReadyc:
-				s.mu.Lock()
-				s.clusterInfo.CustomVmsHealthy = true
-				s.clusterInfo.CustomVms = make(map[string]*rpcpb.CustomVmInfo)
-				for vmID, vmInfo := range s.network.customVMIDToInfo {
-					s.clusterInfo.CustomVms[vmID.String()] = vmInfo.info
-				}
-				s.mu.Unlock()
-			}
+			s.waitChAndUpdateClusterInfo("waiting for custom VMs readiness", readyCh, true)
 		}
 	}()
 
 	return &rpcpb.StartResponse{ClusterInfo: s.clusterInfo}, nil
 }
 
+func (s *server) waitChAndUpdateClusterInfo(waitMsg string, readyCh chan struct{}, updateCustomVmsInfo bool) {
+	zap.L().Info(waitMsg)
+	select {
+	case <-s.closed:
+		return
+	case <-s.network.stopCh:
+		return
+	case serr := <-s.network.startErrCh:
+		// TODO: decide what to do here, general failure cause network stop()?
+		// maybe try decide if operation was partial (undesired network, fail)
+		// or was not stated (preconditions check, continue)
+		zap.L().Warn("async call failed to complete", zap.String("op", waitMsg), zap.Error(serr))
+		panic(serr)
+	case <-readyCh:
+		s.mu.Lock()
+		s.clusterInfo.Healthy = true
+		s.clusterInfo.NodeNames = s.network.nodeNames
+		s.clusterInfo.NodeInfos = s.network.nodeInfos
+		if updateCustomVmsInfo {
+			s.clusterInfo.CustomVmsHealthy = true
+			s.clusterInfo.CustomVms = make(map[string]*rpcpb.CustomVmInfo)
+			for blockchainID, vmInfo := range s.network.customVMBlockchainIDToInfo {
+				s.clusterInfo.CustomVms[blockchainID.String()] = vmInfo.info
+			}
+			s.clusterInfo.Subnets = s.network.subnets
+		}
+		s.mu.Unlock()
+	}
+}
+
+func (s *server) CreateBlockchains(ctx context.Context, req *rpcpb.CreateBlockchainsRequest) (*rpcpb.CreateBlockchainsResponse, error) {
+	// if timeout is too small or not set, default to 5-min
+	if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) < DefaultStartTimeout {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(context.Background(), DefaultStartTimeout)
+		_ = cancel // don't call since "start" is async, "curl" may not specify timeout
+		zap.L().Info("received start request with default timeout", zap.String("timeout", DefaultStartTimeout.String()))
+	} else {
+		zap.L().Info("received start request with existing timeout", zap.String("deadline", deadline.String()))
+	}
+
+	zap.L().Debug("CreateBlockchains")
+	if info := s.getClusterInfo(); info == nil {
+		return nil, ErrNotBootstrapped
+	}
+
+	if len(req.GetBlockchainSpecs()) == 0 {
+		return nil, errors.New("no blockchain spec was provided")
+	}
+
+	chainSpecs := []blockchainSpec{}
+	for i := range req.GetBlockchainSpecs() {
+		vmName := req.GetBlockchainSpecs()[i].VmName
+		vmGenesisFilePath := req.GetBlockchainSpecs()[i].Genesis
+		zap.L().Info("checking custom VM ID before installation", zap.String("vm-id", vmName))
+		vmID, err := utils.VMID(vmName)
+		if err != nil {
+			zap.L().Warn("failed to convert VM name to VM ID",
+				zap.String("vm-name", vmName),
+				zap.Error(err),
+			)
+			return nil, ErrInvalidVMName
+		}
+		if err := utils.CheckPluginPaths(
+			filepath.Join(s.network.pluginDir, vmID.String()),
+			vmGenesisFilePath,
+		); err != nil {
+			return nil, err
+		}
+		b, err := os.ReadFile(vmGenesisFilePath)
+		if err != nil {
+			return nil, err
+		}
+		chainSpecs = append(chainSpecs, blockchainSpec{
+			vmName:   vmName,
+			genesis:  b,
+			subnetId: req.GetBlockchainSpecs()[i].SubnetId,
+		})
+	}
+
+	// check that defined subnets exist
+	subnetsMap := map[string]struct{}{}
+	for _, subnet := range s.clusterInfo.Subnets {
+		subnetsMap[subnet] = struct{}{}
+	}
+	for _, chainSpec := range chainSpecs {
+		if chainSpec.subnetId != nil {
+			_, ok := subnetsMap[*chainSpec.subnetId]
+			if !ok {
+				return nil, fmt.Errorf("subnet id %q does not exits", *chainSpec.subnetId)
+			}
+		}
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	// if there will be a restart, network will not be healthy
+	// until finishing
+	for _, chainSpec := range chainSpecs {
+		if chainSpec.subnetId == nil {
+			s.clusterInfo.Healthy = false
+		}
+	}
+
+	s.clusterInfo.CustomVmsHealthy = false
+
+	// start non-blocking to install custom VMs (if applicable)
+	// the user is expected to poll cluster status
+	readyCh := make(chan struct{})
+	go s.network.createBlockchains(ctx, chainSpecs, readyCh)
+
+	// update cluster info non-blocking
+	// the user is expected to poll this latest information
+	// to decide cluster/subnet readiness
+	go func() {
+		s.waitChAndUpdateClusterInfo("waiting for custom VMs readiness", readyCh, true)
+	}()
+
+	return &rpcpb.CreateBlockchainsResponse{ClusterInfo: s.clusterInfo}, nil
+}
+
+func (s *server) CreateSubnets(ctx context.Context, req *rpcpb.CreateSubnetsRequest) (*rpcpb.CreateSubnetsResponse, error) {
+	// if timeout is too small or not set, default to 5-min
+	if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) < DefaultStartTimeout {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(context.Background(), DefaultStartTimeout)
+		_ = cancel // don't call since "start" is async, "curl" may not specify timeout
+		zap.L().Info("received start request with default timeout", zap.String("timeout", DefaultStartTimeout.String()))
+	} else {
+		zap.L().Info("received start request with existing timeout", zap.String("deadline", deadline.String()))
+	}
+
+	zap.L().Debug("CreateSubnets", zap.Uint32("num-subnets", req.GetNumSubnets()))
+
+	if info := s.getClusterInfo(); info == nil {
+		return nil, ErrNotBootstrapped
+	}
+
+	// default behaviour without args is to create one subnet
+	numSubnets := req.GetNumSubnets()
+	if numSubnets == 0 {
+		numSubnets = 1
+	}
+
+	zap.L().Info("waiting for local cluster readiness")
+	if err := s.network.waitForLocalClusterReady(ctx); err != nil {
+		return nil, err
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	s.clusterInfo.Healthy = false
+	s.clusterInfo.CustomVmsHealthy = false
+
+	// start non-blocking to add subnets
+	// the user is expected to poll cluster status
+	readyCh := make(chan struct{})
+	go s.network.createSubnets(ctx, numSubnets, readyCh)
+
+	// update cluster info non-blocking
+	// the user is expected to poll this latest information
+	// to decide cluster/subnet readiness
+	go func() {
+		s.waitChAndUpdateClusterInfo("waiting for custom VMs readiness", readyCh, true)
+	}()
+
+	return &rpcpb.CreateSubnetsResponse{ClusterInfo: s.clusterInfo}, nil
+}
+
 func (s *server) Health(ctx context.Context, req *rpcpb.HealthRequest) (*rpcpb.HealthResponse, error) {
 	zap.L().Debug("health")
 	if info := s.getClusterInfo(); info == nil {
@@ -386,10 +568,6 @@ func (s *server) Health(ctx context.Context, req *rpcpb.HealthRequest) (*rpcpb.H
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
-	s.network.nodeNames = make([]string, 0)
-	for name := range s.network.nodeInfos {
-		s.network.nodeNames = append(s.network.nodeNames, name)
-	}
 	s.clusterInfo.NodeNames = s.network.nodeNames
 	s.clusterInfo.NodeInfos = s.network.nodeInfos
 
@@ -528,15 +706,20 @@ func (s *server) AddNode(ctx context.Context, req *rpcpb.AddNodeRequest) (*rpcpb
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
-	nodeName := req.Name
-	if _, exists := s.network.nodeInfos[nodeName]; exists {
-		return nil, fmt.Errorf("node with name %s already exists", nodeName)
+	var whitelistedSubnets string
+
+	if _, exists := s.network.nodeInfos[req.Name]; exists {
+		return nil, fmt.Errorf("node with name %s already exists", req.Name)
+	}
+	// fix if not given
+	if req.StartRequest == nil {
+		req.StartRequest = &rpcpb.StartRequest{}
 	}
 	// user can override bin path for this node...
 	execPath := req.StartRequest.ExecPath
 	if execPath == "" {
 		// ...or use the same binary as the rest of the network
-		execPath = s.network.binPath
+		execPath = s.network.execPath
 	}
 	_, err := os.Stat(execPath)
 	if err != nil {
@@ -545,44 +728,57 @@ func (s *server) AddNode(ctx context.Context, req *rpcpb.AddNodeRequest) (*rpcpb
 		}
 		return nil, fmt.Errorf("failed to stat exec %q (%w)", execPath, err)
 	}
-	logLevel := *req.StartRequest.LogLevel
-	whitelistedSubnets := *req.StartRequest.WhitelistedSubnets
-	pluginDir := *req.StartRequest.PluginDir
+
+	// use same configs from other nodes
+	whitelistedSubnets = s.network.options.whitelistedSubnets
+	buildDir, err := getBuildDir(execPath, s.network.pluginDir)
+	if err != nil {
+		return nil, err
+	}
 
 	rootDataDir := s.clusterInfo.RootDataDir
 
-	logDir := filepath.Join(rootDataDir, nodeName, "log")
-	dbDir := filepath.Join(rootDataDir, nodeName, "db-dir")
+	logDir := filepath.Join(rootDataDir, req.Name, "log")
+	dbDir := filepath.Join(rootDataDir, req.Name, "db-dir")
 
-	configFile := createConfigFileString(logLevel, logDir, dbDir, pluginDir, whitelistedSubnets)
+	var defaultConfig, globalConfig map[string]interface{}
+	if err := json.Unmarshal([]byte(defaultNodeConfig), &defaultConfig); err != nil {
+		return nil, err
+	}
+	if req.StartRequest.GetGlobalNodeConfig() != "" {
+		if err := json.Unmarshal([]byte(req.StartRequest.GetGlobalNodeConfig()), &globalConfig); err != nil {
+			return nil, err
+		}
+	}
+
+	var mergedConfig map[string]interface{}
+	// we only need to merge from the default node config here, as we are only adding one node
+	mergedConfig, err = mergeNodeConfig(defaultConfig, globalConfig, "")
+	if err != nil {
+		return nil, fmt.Errorf("failed merging provided configs: %w", err)
+	}
+	configFile, err := createConfigFileString(mergedConfig, logDir, dbDir, buildDir, whitelistedSubnets)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate json node config string: %w", err)
+	}
 	stakingCert, stakingKey, err := staking.NewCertAndKeyBytes()
 	if err != nil {
 		return nil, fmt.Errorf("couldn't generate staking Cert/Key: %w", err)
 	}
 	nodeConfig := node.Config{
-		Name:               req.Name,
-		ConfigFile:         configFile,
-		ImplSpecificConfig: json.RawMessage(fmt.Sprintf(`{"binaryPath":"%s","redirectStdout":true,"redirectStderr":true}`, execPath)),
-		StakingKey:         string(stakingKey),
-		StakingCert:        string(stakingCert),
-		CChainConfigFile:   req.StartRequest.ChainConfigs["C"], // TODO: add support for other chains
+		Name:           req.Name,
+		ConfigFile:     configFile,
+		StakingKey:     string(stakingKey),
+		StakingCert:    string(stakingCert),
+		BinaryPath:     execPath,
+		RedirectStdout: s.cfg.RedirectNodesOutput,
+		RedirectStderr: s.cfg.RedirectNodesOutput,
+		ChainConfigFiles: req.StartRequest.ChainConfigs,
 	}
-	s.network.nodeNames = append(s.network.nodeNames, req.Name)
 	_, err = s.network.nw.AddNode(nodeConfig)
 	if err != nil {
 		return nil, err
 	}
-	info := &rpcpb.NodeInfo{
-		Name:               nodeName,
-		ExecPath:           execPath,
-		Uri:                "",
-		Id:                 "",
-		LogDir:             logDir,
-		DbDir:              dbDir,
-		WhitelistedSubnets: whitelistedSubnets,
-		Config:             []byte(configFile),
-	}
-	s.network.nodeInfos[req.Name] = info
 
 	return &rpcpb.AddNodeResponse{ClusterInfo: s.clusterInfo}, nil
 }
@@ -603,19 +799,15 @@ func (s *server) RemoveNode(ctx context.Context, req *rpcpb.RemoveNodeRequest) (
 	if err := s.network.nw.RemoveNode(req.Name); err != nil {
 		return nil, err
 	}
-	delete(s.network.nodeInfos, req.Name)
-	s.network.nodeNames = make([]string, 0)
-	for name := range s.network.nodeInfos {
-		s.network.nodeNames = append(s.network.nodeNames, name)
-	}
-	s.clusterInfo.NodeNames = s.network.nodeNames
-	s.clusterInfo.NodeInfos = s.network.nodeInfos
 
 	zap.L().Info("waiting for local cluster readiness")
 	if err := s.network.waitForLocalClusterReady(ctx); err != nil {
 		return nil, err
 	}
 
+	s.clusterInfo.NodeNames = s.network.nodeNames
+	s.clusterInfo.NodeInfos = s.network.nodeInfos
+
 	return &rpcpb.RemoveNodeResponse{ClusterInfo: s.clusterInfo}, nil
 }
 
@@ -633,20 +825,11 @@ func (s *server) RestartNode(ctx context.Context, req *rpcpb.RestartNodeRequest)
 		return nil, ErrNodeNotFound
 	}
 
-	found, idx := false, 0
-	oldNodeConfig := node.Config{}
-	for i, cfg := range s.network.cfg.NodeConfigs {
-		if cfg.Name == req.Name {
-			oldNodeConfig = cfg
-			found = true
-			idx = i
-			break
-		}
-	}
-	if !found {
+	node, err := s.network.nw.GetNode(req.Name)
+	if err != nil {
 		return nil, ErrNodeNotFound
 	}
-	nodeConfig := oldNodeConfig
+	nodeConfig := node.GetConfig()
 
 	// use existing value if not specified
 	if req.GetExecPath() != "" {
@@ -658,34 +841,31 @@ func (s *server) RestartNode(ctx context.Context, req *rpcpb.RestartNodeRequest)
 	if req.GetRootDataDir() != "" {
 		nodeInfo.DbDir = filepath.Join(req.GetRootDataDir(), req.Name, "db-dir")
 	}
-	logLevel := "INFO"
-	if req.GetLogLevel() != "" {
-		logLevel = strings.ToUpper(req.GetLogLevel())
+
+	var defaultConfig map[string]interface{}
+	if err := json.Unmarshal([]byte(defaultNodeConfig), &defaultConfig); err != nil {
+		return nil, err
 	}
 
-	nodeConfig.ConfigFile = fmt.Sprintf(`{
-	"network-peer-list-gossip-frequency":"250ms",
-	"network-max-reconnect-delay":"1s",
-	"public-ip":"127.0.0.1",
-	"health-check-frequency":"2s",
-	"api-admin-enabled":true,
-	"api-ipcs-enabled":true,
-	"index-enabled":true,
-	"log-display-level":"%s",
-	"log-level":"%s",
-	"log-dir":"%s",
-	"db-dir":"%s",
-	"plugin-dir":"%s",
-	"whitelisted-subnets":"%s"
-}`,
-		logLevel,
-		logLevel,
+	buildDir, err := getBuildDir(nodeInfo.ExecPath, nodeInfo.PluginDir)
+	if err != nil {
+		return nil, err
+	}
+
+	nodeConfig.ConfigFile, err = createConfigFileString(
+		defaultConfig,
 		nodeInfo.LogDir,
 		nodeInfo.DbDir,
-		nodeInfo.PluginDir,
+		buildDir,
 		nodeInfo.WhitelistedSubnets,
 	)
-	nodeConfig.ImplSpecificConfig = json.RawMessage(fmt.Sprintf(`{"binaryPath":"%s","redirectStdout":true,"redirectStderr":true}`, nodeInfo.ExecPath))
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate json node config string: %w", err)
+	}
+
+	nodeConfig.BinaryPath = nodeInfo.ExecPath
+	nodeConfig.RedirectStdout = s.cfg.RedirectNodesOutput
+	nodeConfig.RedirectStderr = s.cfg.RedirectNodesOutput
 
 	// now remove the node before restart
 	zap.L().Info("removing the node")
@@ -705,7 +885,7 @@ func (s *server) RestartNode(ctx context.Context, req *rpcpb.RestartNodeRequest)
 	}
 
 	// update with the new config
-	s.network.cfg.NodeConfigs[idx] = nodeConfig
+	s.clusterInfo.NodeNames = s.network.nodeNames
 	s.clusterInfo.NodeInfos = s.network.nodeInfos
 
 	return &rpcpb.RestartNodeResponse{ClusterInfo: s.clusterInfo}, nil
@@ -713,19 +893,24 @@ func (s *server) RestartNode(ctx context.Context, req *rpcpb.RestartNodeRequest)
 
 func (s *server) Stop(ctx context.Context, req *rpcpb.StopRequest) (*rpcpb.StopResponse, error) {
 	zap.L().Debug("received stop request")
-	info := s.getClusterInfo()
-	if info == nil {
-		return nil, ErrNotBootstrapped
-	}
 
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
+	if s.network == nil {
+		return nil, ErrNotBootstrapped
+	}
+
+	info := s.clusterInfo
+	if info == nil {
+		info = &rpcpb.ClusterInfo{}
+	}
+
 	s.network.stop(ctx)
 	s.network = nil
-	info.Healthy = false
 	s.clusterInfo = nil
 
+	info.Healthy = false
 	return &rpcpb.StopResponse{ClusterInfo: info}, nil
 }
 
@@ -817,10 +1002,148 @@ func (s *server) SendOutboundMessage(ctx context.Context, req *rpcpb.SendOutboun
 	}
 
 	msg := message.NewTestMsg(message.Op(req.Op), req.Bytes, false)
-	sent := attachedPeer.Send(msg)
+	sent := attachedPeer.Send(ctx, msg)
 	return &rpcpb.SendOutboundMessageResponse{Sent: sent}, nil
 }
 
+func (s *server) LoadSnapshot(ctx context.Context, req *rpcpb.LoadSnapshotRequest) (*rpcpb.LoadSnapshotResponse, error) {
+	// if timeout is too small or not set, default to 5-min
+	if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) < DefaultStartTimeout {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(context.Background(), DefaultStartTimeout)
+		_ = cancel // don't call since "start" is async, "curl" may not specify timeout
+		zap.L().Info("received start request with default timeout", zap.String("timeout", DefaultStartTimeout.String()))
+	} else {
+		zap.L().Info("received start request with existing timeout", zap.String("deadline", deadline.String()))
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	// If [clusterInfo] is already populated, the server has already been started.
+	if s.clusterInfo != nil {
+		return nil, ErrAlreadyBootstrapped
+	}
+
+	var (
+		pid = int32(os.Getpid())
+		err error
+	)
+
+	rootDataDir := req.GetRootDataDir()
+	if len(rootDataDir) == 0 {
+		rootDataDir = os.TempDir()
+	}
+	rootDataDir = filepath.Join(rootDataDir, rootDataDirPrefix)
+	rootDataDir, err = utils.MkDirWithTimestamp(rootDataDir)
+	if err != nil {
+		return nil, err
+	}
+
+	s.clusterInfo = &rpcpb.ClusterInfo{
+		Pid:         pid,
+		RootDataDir: rootDataDir,
+		Healthy:     false,
+	}
+
+	zap.L().Info("starting",
+		zap.Int32("pid", pid),
+		zap.String("rootDataDir", rootDataDir),
+	)
+
+	if s.network != nil {
+		return nil, ErrAlreadyBootstrapped
+	}
+
+	s.network, err = newLocalNetwork(localNetworkOptions{
+		execPath:    req.GetExecPath(),
+		pluginDir:   req.GetPluginDir(),
+		rootDataDir: rootDataDir,
+
+		// to block racey restart
+		// "s.network.start" runs asynchronously
+		// so it would not deadlock with the acquired lock
+		// in this "Start" method
+		restartMu: s.mu,
+
+		snapshotsDir: s.cfg.SnapshotsDir,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// blocking load snapshot to soon get not found snapshot errors
+	if err := s.network.loadSnapshot(ctx, req.SnapshotName); err != nil {
+		zap.L().Warn("snapshot load failed to complete", zap.Error(err))
+		s.network = nil
+		s.clusterInfo = nil
+		return nil, err
+	}
+
+	// start non-blocking wait to load snapshot results
+	// the user is expected to poll cluster status
+	readyCh := make(chan struct{})
+	go s.network.loadSnapshotWait(ctx, readyCh)
+
+	// update cluster info non-blocking
+	// the user is expected to poll this latest information
+	// to decide cluster/subnet readiness
+	go func() {
+		s.waitChAndUpdateClusterInfo("waiting for local cluster readiness", readyCh, true)
+	}()
+
+	return &rpcpb.LoadSnapshotResponse{ClusterInfo: s.clusterInfo}, nil
+}
+
+func (s *server) SaveSnapshot(ctx context.Context, req *rpcpb.SaveSnapshotRequest) (*rpcpb.SaveSnapshotResponse, error) {
+	zap.L().Info("received save snapshot request", zap.String("snapshot-name", req.SnapshotName))
+	info := s.getClusterInfo()
+	if info == nil {
+		return nil, ErrNotBootstrapped
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	snapshotPath, err := s.network.nw.SaveSnapshot(ctx, req.SnapshotName)
+	if err != nil {
+		zap.L().Warn("snapshot save failed to complete", zap.Error(err))
+		return nil, err
+	}
+	s.network = nil
+	s.clusterInfo = nil
+
+	return &rpcpb.SaveSnapshotResponse{SnapshotPath: snapshotPath}, nil
+}
+
+func (s *server) RemoveSnapshot(ctx context.Context, req *rpcpb.RemoveSnapshotRequest) (*rpcpb.RemoveSnapshotResponse, error) {
+	zap.L().Info("received remove snapshot request", zap.String("snapshot-name", req.SnapshotName))
+	info := s.getClusterInfo()
+	if info == nil {
+		return nil, ErrNotBootstrapped
+	}
+
+	if err := s.network.nw.RemoveSnapshot(req.SnapshotName); err != nil {
+		zap.L().Warn("snapshot remove failed to complete", zap.Error(err))
+		return nil, err
+	}
+	return &rpcpb.RemoveSnapshotResponse{}, nil
+}
+
+func (s *server) GetSnapshotNames(ctx context.Context, req *rpcpb.GetSnapshotNamesRequest) (*rpcpb.GetSnapshotNamesResponse, error) {
+	zap.L().Info("get snapshot names")
+	info := s.getClusterInfo()
+	if info == nil {
+		return nil, ErrNotBootstrapped
+	}
+
+	snapshotNames, err := s.network.nw.GetSnapshotNames()
+	if err != nil {
+		return nil, err
+	}
+	return &rpcpb.GetSnapshotNamesResponse{SnapshotNames: snapshotNames}, nil
+}
+
 func (s *server) getClusterInfo() *rpcpb.ClusterInfo {
 	s.mu.RLock()
 	info := s.clusterInfo

server/wallet.go

@@ -17,7 +17,7 @@ import (
 	"github.com/ava-labs/avalanchego/wallet/subnet/primary"
 )
 
-const defaultTimeout = 30 * time.Second
+const defaultTimeout = time.Minute
 
 func createDefaultCtx(ctx context.Context) (context.Context, context.CancelFunc) {
 	if ctx == nil {
@@ -44,7 +44,12 @@ type refreshableWallet struct {
 // Creates a new wallet to work around the case where the new wallet object
 // is not able to find previous transactions in the cache.
 // TODO: support tx backfilling in upstream wallet SDK.
-func createRefreshableWallet(ctx context.Context, httpRPCEp string, kc *secp256k1fx.Keychain) (*refreshableWallet, error) {
+func createRefreshableWallet(
+	ctx context.Context,
+	httpRPCEp string,
+	kc *secp256k1fx.Keychain,
+	pTXs map[ids.ID]*platformvm.Tx,
+) (*refreshableWallet, error) {
 	cctx, cancel := createDefaultCtx(ctx)
 	pCTX, xCTX, utxos, err := primary.FetchState(cctx, httpRPCEp, kc.Addrs)
 	cancel()
@@ -53,7 +58,6 @@ func createRefreshableWallet(ctx context.Context, httpRPCEp string, kc *secp256k
 	}
 
 	pUTXOs := primary.NewChainUTXOs(constants.PlatformChainID, utxos)
-	pTXs := make(map[ids.ID]*platformvm.Tx)
 	pBackend := p.NewBackend(pCTX, pUTXOs, pTXs)
 	pBuilder := p.NewBuilder(kc.Addrs, pBackend)
 	pSigner := p.NewSigner(kc, pBackend)

tests/e2e/e2e_test.go

@@ -8,7 +8,6 @@ import (
 	"context"
 	"flag"
 	"io/fs"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -20,6 +19,8 @@ import (
 	"github.com/ava-labs/avalanche-network-runner/pkg/logutil"
 	"github.com/ava-labs/avalanche-network-runner/server"
 	"github.com/ava-labs/avalanche-network-runner/utils"
+
+	"github.com/ava-labs/avalanchego/api/admin"
 	"github.com/ava-labs/avalanchego/ids"
 	"github.com/ava-labs/avalanchego/message"
 	"github.com/ava-labs/avalanchego/utils/constants"
@@ -42,6 +43,18 @@ var (
 	gRPCGatewayEp string
 	execPath1     string
 	execPath2     string
+
+	newNodeName       = "test-add-node"
+	customNodeConfigs = map[string]string{
+		"node1": `{"api-admin-enabled":true}`,
+		"node2": `{"api-admin-enabled":true}`,
+		"node3": `{"api-admin-enabled":true}`,
+		"node4": `{"api-admin-enabled":false}`,
+		"node5": `{"api-admin-enabled":false}`,
+		"node6": `{"api-admin-enabled":false}`,
+		"node7": `{"api-admin-enabled":false}`,
+	}
+	numNodes = uint32(5)
 )
 
 func init() {
@@ -101,7 +114,7 @@ var _ = ginkgo.AfterSuite(func() {
 	gomega.Ω(err).Should(gomega.BeNil())
 })
 
-var _ = ginkgo.Describe("[Start/Remove/Restart/Stop]", func() {
+var _ = ginkgo.Describe("[Start/Remove/Restart/Add/Stop]", func() {
 	ginkgo.It("can start", func() {
 		ginkgo.By("start request with invalid exec path should fail", func() {
 			ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
@@ -144,36 +157,11 @@ var _ = ginkgo.Describe("[Start/Remove/Restart/Stop]", func() {
 			os.RemoveAll(filePath)
 		})
 
-		ginkgo.By("start request with missing plugin dir should fail", func() {
-			ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
-			_, err := cli.Start(ctx, execPath1,
-				client.WithCustomVMs(map[string]string{"test": "{0}"}),
-			)
-			cancel()
-			gomega.Ω(err.Error()).Should(gomega.ContainSubstring(server.ErrPluginDirEmptyButCustomVMsNotEmpty.Error()))
-		})
-
-		ginkgo.By("start request with missing custom VMs should fail", func() {
-			f, err := os.CreateTemp(os.TempDir(), strings.Repeat("a", 33))
-			gomega.Ω(err).Should(gomega.BeNil())
-			filePath := f.Name()
-			gomega.Ω(f.Close()).Should(gomega.BeNil())
-
-			ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
-			_, err = cli.Start(ctx, execPath1,
-				client.WithPluginDir(filepath.Dir(filePath)),
-			)
-			cancel()
-			gomega.Ω(err.Error()).Should(gomega.ContainSubstring(server.ErrPluginDirNonEmptyButCustomVMsEmpty.Error()))
-
-			os.RemoveAll(filePath)
-		})
-
 		ginkgo.By("start request with invalid custom VM genesis path should fail", func() {
 			vmID, err := utils.VMID("hello")
 			gomega.Ω(err).Should(gomega.BeNil())
 			filePath := filepath.Join(os.TempDir(), vmID.String())
-			gomega.Ω(ioutil.WriteFile(filePath, []byte{0}, fs.ModePerm)).Should(gomega.BeNil())
+			gomega.Ω(os.WriteFile(filePath, []byte{0}, fs.ModePerm)).Should(gomega.BeNil())
 
 			ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
 			_, err = cli.Start(ctx, execPath1,
@@ -197,9 +185,6 @@ var _ = ginkgo.Describe("[Start/Remove/Restart/Stop]", func() {
 	})
 
 	ginkgo.It("can wait for health", func() {
-		// start is async, so wait some time for cluster health
-		time.Sleep(2 * time.Minute)
-
 		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
 		_, err := cli.Health(ctx)
 		cancel()
@@ -293,4 +278,274 @@ var _ = ginkgo.Describe("[Start/Remove/Restart/Stop]", func() {
 			gomega.Ω(sresp.Sent).Should(gomega.BeTrue())
 		})
 	})
+
+	time.Sleep(10 * time.Second)
+	ginkgo.It("can add a node", func() {
+		ginkgo.By("calling AddNode", func() {
+			color.Outf("{{green}}calling 'add-node' with the valid binary path:{{/}} %q\n", execPath1)
+			ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+			resp, err := cli.AddNode(ctx, newNodeName, execPath1)
+			cancel()
+			gomega.Ω(err).Should(gomega.BeNil())
+			color.Outf("{{green}}successfully started:{{/}} %+v\n", resp.ClusterInfo.NodeNames)
+		})
+
+		ginkgo.By("calling AddNode with existing node name, should fail", func() {
+			color.Outf("{{green}}calling 'add-node' with the valid binary path:{{/}} %q\n", execPath1)
+			ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+			resp, err := cli.AddNode(ctx, newNodeName, execPath1)
+			cancel()
+			gomega.Ω(err.Error()).Should(gomega.ContainSubstring("repeated node name"))
+			gomega.Ω(resp).Should(gomega.BeNil())
+			color.Outf("{{green}}add-node failed as expected")
+		})
+	})
+
+	ginkgo.It("can start with custom config", func() {
+		ginkgo.By("stopping network first", func() {
+			color.Outf("{{red}}shutting down cluster{{/}}\n")
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+			_, err := cli.Stop(ctx)
+			cancel()
+			gomega.Ω(err).Should(gomega.BeNil())
+
+			color.Outf("{{red}}shutting down client{{/}}\n")
+			gomega.Ω(err).Should(gomega.BeNil())
+		})
+		ginkgo.By("calling start API with custom config", func() {
+			color.Outf("{{green}}sending 'start' with the valid binary path:{{/}} %q\n", execPath1)
+			ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+			opts := []client.OpOption{
+				client.WithNumNodes(numNodes),
+				client.WithCustomNodeConfigs(customNodeConfigs),
+			}
+			resp, err := cli.Start(ctx, execPath1, opts...)
+			cancel()
+			gomega.Ω(err).Should(gomega.BeNil())
+			color.Outf("{{green}}successfully started:{{/}} %+v\n", resp.ClusterInfo.NodeNames)
+		})
+		ginkgo.By("can wait for health", func() {
+			// start is async, so wait some time for cluster health
+			time.Sleep(30 * time.Second)
+
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+			_, err := cli.Health(ctx)
+			cancel()
+			gomega.Ω(err).Should(gomega.BeNil())
+		})
+		ginkgo.By("overrides num-nodes", func() {
+			color.Outf("{{green}}checking that given num-nodes %d have been overriden by custom configs with %d:\n", numNodes, len(customNodeConfigs))
+			ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+			uris, err := cli.URIs(ctx)
+			cancel()
+			gomega.Ω(err).Should(gomega.BeNil())
+			gomega.Ω(uris).Should(gomega.HaveLen(len(customNodeConfigs)))
+			color.Outf("{{green}}expected number of nodes up:{{/}} %q\n", len(customNodeConfigs))
+
+			color.Outf("{{green}}checking correct admin APIs are enabled resp. disabled")
+			// we have 7 nodes, 3 have the admin API enabled, the other 4 disabled
+			// therefore we expect exactly 4 calls to fail and exactly 3 to succeed.
+			ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
+			errCnt := 0
+			for i := 0; i < len(uris); i++ {
+				cli := admin.NewClient(uris[i])
+				_, err := cli.LockProfile(ctx)
+				if err != nil {
+					errCnt++
+				}
+			}
+			cancel()
+			gomega.Ω(errCnt).Should(gomega.Equal(4))
+		})
+	})
+
+	ginkgo.It("start with default network, for subsecuent steps", func() {
+		ginkgo.By("stopping network first", func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+			_, err := cli.Stop(ctx)
+			cancel()
+			gomega.Ω(err).Should(gomega.BeNil())
+		})
+		ginkgo.By("starting", func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+			_, err := cli.Start(ctx, execPath1)
+			cancel()
+			gomega.Ω(err).Should(gomega.BeNil())
+		})
+		ginkgo.By("wait for health", func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+			_, err := cli.Health(ctx)
+			cancel()
+			gomega.Ω(err).Should(gomega.BeNil())
+		})
+	})
+
+	ginkgo.It("subnet creation", func() {
+		ginkgo.By("check subnets are 0", func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+			status, err := cli.Status(ctx)
+			cancel()
+			gomega.Ω(err).Should(gomega.BeNil())
+			numSubnets := len(status.ClusterInfo.Subnets)
+			gomega.Ω(numSubnets).Should(gomega.Equal(0))
+		})
+		ginkgo.By("add 1 subnet", func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+			_, err := cli.CreateSubnets(ctx)
+			cancel()
+			gomega.Ω(err).Should(gomega.BeNil())
+		})
+		ginkgo.By("wait for network to be healthy", func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+			var created bool
+			continueLoop := true
+			for continueLoop {
+				select {
+				case <-ctx.Done():
+					continueLoop = false
+				case <-time.After(5 * time.Second):
+					ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+					status, err := cli.Status(ctx)
+					cancel()
+					gomega.Ω(err).Should(gomega.BeNil())
+					created = status.ClusterInfo.CustomVmsHealthy
+					if created {
+						continueLoop = false
+					}
+				}
+			}
+			cancel()
+			gomega.Ω(created).Should(gomega.Equal(true))
+		})
+		ginkgo.By("check subnets are 1", func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+			status, err := cli.Status(ctx)
+			cancel()
+			gomega.Ω(err).Should(gomega.BeNil())
+			numSubnets := len(status.ClusterInfo.Subnets)
+			gomega.Ω(numSubnets).Should(gomega.Equal(1))
+		})
+	})
+
+	ginkgo.It("snapshots + blockchain creation", func() {
+		var originalUris []string
+		var originalSubnets []string
+		ginkgo.By("get original URIs", func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+			var err error
+			originalUris, err = cli.URIs(ctx)
+			cancel()
+			gomega.Ω(err).Should(gomega.BeNil())
+			gomega.Ω(len(originalUris)).Should(gomega.Equal(5))
+		})
+		ginkgo.By("get original subnets", func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+			status, err := cli.Status(ctx)
+			cancel()
+			gomega.Ω(err).Should(gomega.BeNil())
+			numSubnets := len(status.ClusterInfo.Subnets)
+			gomega.Ω(numSubnets).Should(gomega.Equal(1))
+			originalSubnets = status.ClusterInfo.Subnets
+		})
+		ginkgo.By("check there are no snapshots", func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+			snapshotNames, err := cli.GetSnapshotNames(ctx)
+			cancel()
+			gomega.Ω(err).Should(gomega.BeNil())
+			gomega.Ω(snapshotNames).Should(gomega.Equal([]string(nil)))
+		})
+		ginkgo.By("can save snapshot", func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+			_, err := cli.SaveSnapshot(ctx, "pepe")
+			cancel()
+			gomega.Ω(err).Should(gomega.BeNil())
+		})
+		ginkgo.By("wait fail for stopped network", func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+			_, err := cli.Health(ctx)
+			cancel()
+			gomega.Ω(err.Error()).Should(gomega.ContainSubstring("not bootstrapped"))
+		})
+		ginkgo.By("load fail for unknown snapshot", func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+			_, err := cli.LoadSnapshot(ctx, "papa")
+			cancel()
+			gomega.Ω(err.Error()).Should(gomega.ContainSubstring("snapshot \"papa\" does not exists"))
+		})
+		ginkgo.By("can load snapshot", func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+			_, err := cli.LoadSnapshot(ctx, "pepe")
+			cancel()
+			gomega.Ω(err).Should(gomega.BeNil())
+		})
+		ginkgo.By("wait for network to be healthy", func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+			var created bool
+			continueLoop := true
+			for continueLoop {
+				select {
+				case <-ctx.Done():
+					continueLoop = false
+				case <-time.After(5 * time.Second):
+					ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+					status, err := cli.Status(ctx)
+					cancel()
+					gomega.Ω(err).Should(gomega.BeNil())
+					created = status.ClusterInfo.CustomVmsHealthy
+					if created {
+						continueLoop = false
+					}
+				}
+			}
+			cancel()
+			gomega.Ω(created).Should(gomega.Equal(true))
+		})
+		ginkgo.By("check URIs", func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+			var err error
+			uris, err := cli.URIs(ctx)
+			cancel()
+			gomega.Ω(err).Should(gomega.BeNil())
+			gomega.Ω(uris).Should(gomega.Equal(originalUris))
+		})
+		ginkgo.By("check subnets", func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+			status, err := cli.Status(ctx)
+			cancel()
+			gomega.Ω(err).Should(gomega.BeNil())
+			gomega.Ω(status.ClusterInfo.Subnets).Should(gomega.Equal(originalSubnets))
+		})
+		ginkgo.By("save fail for already saved snapshot", func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+			_, err := cli.SaveSnapshot(ctx, "pepe")
+			cancel()
+			gomega.Ω(err.Error()).Should(gomega.ContainSubstring("snapshot \"pepe\" already exists"))
+		})
+		ginkgo.By("check there is a snapshot", func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+			snapshotNames, err := cli.GetSnapshotNames(ctx)
+			cancel()
+			gomega.Ω(err).Should(gomega.BeNil())
+			gomega.Ω(snapshotNames).Should(gomega.Equal([]string{"pepe"}))
+		})
+		ginkgo.By("can remove snapshot", func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+			_, err := cli.RemoveSnapshot(ctx, "pepe")
+			cancel()
+			gomega.Ω(err).Should(gomega.BeNil())
+		})
+		ginkgo.By("check there are no snapshots", func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+			snapshotNames, err := cli.GetSnapshotNames(ctx)
+			cancel()
+			gomega.Ω(err).Should(gomega.BeNil())
+			gomega.Ω(snapshotNames).Should(gomega.Equal([]string(nil)))
+		})
+		ginkgo.By("remove fail for unknown snapshot", func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+			_, err := cli.RemoveSnapshot(ctx, "pepe")
+			cancel()
+			gomega.Ω(err.Error()).Should(gomega.ContainSubstring("snapshot \"pepe\" does not exists"))
+		})
+	})
 })

utils/beacon/beacon.go

@@ -1,33 +0,0 @@
-// Copyright (C) 2019-2021, Ava Labs, Inc. All rights reserved.
-// See the file LICENSE for licensing terms.
-
-package beacon
-
-import (
-	"github.com/ava-labs/avalanchego/ids"
-	"github.com/ava-labs/avalanchego/utils"
-)
-
-// TODO: remove this in favor of an exported utility from avalanchego
-
-var _ Beacon = &beacon{}
-
-type Beacon interface {
-	ID() ids.ShortID
-	IP() utils.IPDesc
-}
-
-type beacon struct {
-	id ids.ShortID
-	ip utils.IPDesc
-}
-
-func New(id ids.ShortID, ip utils.IPDesc) Beacon {
-	return &beacon{
-		id: id,
-		ip: ip,
-	}
-}
-
-func (b *beacon) ID() ids.ShortID  { return b.id }
-func (b *beacon) IP() utils.IPDesc { return b.ip }

utils/beacon/set.go

@@ -1,133 +0,0 @@
-// Copyright (C) 2019-2021, Ava Labs, Inc. All rights reserved.
-// See the file LICENSE for licensing terms.
-
-package beacon
-
-import (
-	"errors"
-	"strings"
-
-	"github.com/ava-labs/avalanchego/ids"
-	"github.com/ava-labs/avalanchego/utils"
-	"github.com/ava-labs/avalanchego/utils/constants"
-)
-
-// TODO: remove this in favor of an exported utility from avalanchego
-
-var (
-	_ Set = &set{}
-
-	errDuplicateID = errors.New("duplicated ID")
-	errDuplicateIP = errors.New("duplicated IP")
-
-	errUnknownID = errors.New("unknown ID")
-	errUnknownIP = errors.New("unknown IP")
-)
-
-type Set interface {
-	Add(Beacon) error
-
-	RemoveByID(ids.ShortID) error
-	RemoveByIP(utils.IPDesc) error
-
-	Len() int
-
-	IDsArg() string
-	IPsArg() string
-}
-
-type set struct {
-	ids     map[ids.ShortID]int
-	ips     map[string]int
-	beacons []Beacon
-}
-
-func NewSet() Set {
-	return &set{
-		ids: make(map[ids.ShortID]int),
-		ips: make(map[string]int),
-	}
-}
-
-func (s *set) Add(b Beacon) error {
-	id := b.ID()
-	_, duplicateID := s.ids[id]
-	if duplicateID {
-		return errDuplicateID
-	}
-
-	ipStr := b.IP().String()
-	_, duplicateIP := s.ips[ipStr]
-	if duplicateIP {
-		return errDuplicateIP
-	}
-
-	s.ids[id] = len(s.beacons)
-	s.ips[ipStr] = len(s.beacons)
-	s.beacons = append(s.beacons, b)
-	return nil
-}
-
-func (s *set) RemoveByID(idToRemove ids.ShortID) error {
-	indexToRemove, exists := s.ids[idToRemove]
-	if !exists {
-		return errUnknownID
-	}
-	toRemove := s.beacons[indexToRemove]
-	ipToRemove := toRemove.IP().String()
-
-	indexToMove := len(s.beacons) - 1
-	toMove := s.beacons[indexToMove]
-	idToMove := toMove.ID()
-	ipToMove := toMove.IP().String()
-
-	s.ids[idToMove] = indexToRemove
-	s.ips[ipToMove] = indexToRemove
-	s.beacons[indexToRemove] = toMove
-
-	delete(s.ids, idToRemove)
-	delete(s.ips, ipToRemove)
-	s.beacons[indexToMove] = nil
-	s.beacons = s.beacons[:indexToMove]
-	return nil
-}
-
-func (s *set) RemoveByIP(ip utils.IPDesc) error {
-	indexToRemove, exists := s.ips[ip.String()]
-	if !exists {
-		return errUnknownIP
-	}
-	toRemove := s.beacons[indexToRemove]
-	idToRemove := toRemove.ID()
-	return s.RemoveByID(idToRemove)
-}
-
-func (s *set) Len() int { return len(s.beacons) }
-
-func (s *set) IDsArg() string {
-	sb := strings.Builder{}
-	if len(s.beacons) == 0 {
-		return ""
-	}
-	b := s.beacons[0]
-	_, _ = sb.WriteString(b.ID().PrefixedString(constants.NodeIDPrefix))
-	for _, b := range s.beacons[1:] {
-		_, _ = sb.WriteString(",")
-		_, _ = sb.WriteString(b.ID().PrefixedString(constants.NodeIDPrefix))
-	}
-	return sb.String()
-}
-
-func (s *set) IPsArg() string {
-	sb := strings.Builder{}
-	if len(s.beacons) == 0 {
-		return ""
-	}
-	b := s.beacons[0]
-	_, _ = sb.WriteString(b.IP().String())
-	for _, b := range s.beacons[1:] {
-		_, _ = sb.WriteString(",")
-		_, _ = sb.WriteString(b.IP().String())
-	}
-	return sb.String()
-}

utils/beacon/set_test.go

@@ -1,108 +0,0 @@
-// Copyright (C) 2019-2021, Ava Labs, Inc. All rights reserved.
-// See the file LICENSE for licensing terms.
-
-package beacon
-
-import (
-	"net"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-
-	"github.com/ava-labs/avalanchego/ids"
-	"github.com/ava-labs/avalanchego/utils"
-)
-
-func TestSet(t *testing.T) {
-	assert := assert.New(t)
-
-	id0 := ids.ShortID{0}
-	id1 := ids.ShortID{1}
-	id2 := ids.ShortID{2}
-
-	ip0 := utils.IPDesc{
-		IP:   net.IPv4zero,
-		Port: 0,
-	}
-	ip1 := utils.IPDesc{
-		IP:   net.IPv4zero,
-		Port: 1,
-	}
-	ip2 := utils.IPDesc{
-		IP:   net.IPv4zero,
-		Port: 2,
-	}
-
-	b0 := New(id0, ip0)
-	b1 := New(id1, ip1)
-	b2 := New(id2, ip2)
-
-	s := NewSet()
-
-	idsArg := s.IDsArg()
-	assert.Equal("", idsArg)
-	ipsArg := s.IPsArg()
-	assert.Equal("", ipsArg)
-	len := s.Len()
-	assert.Equal(0, len)
-
-	err := s.Add(b0)
-	assert.NoError(err)
-
-	idsArg = s.IDsArg()
-	assert.Equal("NodeID-111111111111111111116DBWJs", idsArg)
-	ipsArg = s.IPsArg()
-	assert.Equal("0.0.0.0:0", ipsArg)
-	len = s.Len()
-	assert.Equal(1, len)
-
-	err = s.Add(b0)
-	assert.ErrorIs(err, errDuplicateID)
-
-	idsArg = s.IDsArg()
-	assert.Equal("NodeID-111111111111111111116DBWJs", idsArg)
-	ipsArg = s.IPsArg()
-	assert.Equal("0.0.0.0:0", ipsArg)
-	len = s.Len()
-	assert.Equal(1, len)
-
-	err = s.Add(b1)
-	assert.NoError(err)
-
-	idsArg = s.IDsArg()
-	assert.Equal("NodeID-111111111111111111116DBWJs,NodeID-6HgC8KRBEhXYbF4riJyJFLSHt37UNuRt", idsArg)
-	ipsArg = s.IPsArg()
-	assert.Equal("0.0.0.0:0,0.0.0.0:1", ipsArg)
-	len = s.Len()
-	assert.Equal(2, len)
-
-	err = s.Add(b2)
-	assert.NoError(err)
-
-	idsArg = s.IDsArg()
-	assert.Equal("NodeID-111111111111111111116DBWJs,NodeID-6HgC8KRBEhXYbF4riJyJFLSHt37UNuRt,NodeID-BaMPFdqMUQ46BV8iRcwbVfsam55kMqcp", idsArg)
-	ipsArg = s.IPsArg()
-	assert.Equal("0.0.0.0:0,0.0.0.0:1,0.0.0.0:2", ipsArg)
-	len = s.Len()
-	assert.Equal(3, len)
-
-	err = s.RemoveByID(b0.ID())
-	assert.NoError(err)
-
-	idsArg = s.IDsArg()
-	assert.Equal("NodeID-BaMPFdqMUQ46BV8iRcwbVfsam55kMqcp,NodeID-6HgC8KRBEhXYbF4riJyJFLSHt37UNuRt", idsArg)
-	ipsArg = s.IPsArg()
-	assert.Equal("0.0.0.0:2,0.0.0.0:1", ipsArg)
-	len = s.Len()
-	assert.Equal(2, len)
-
-	err = s.RemoveByIP(b1.IP())
-	assert.NoError(err)
-
-	idsArg = s.IDsArg()
-	assert.Equal("NodeID-BaMPFdqMUQ46BV8iRcwbVfsam55kMqcp", idsArg)
-	ipsArg = s.IPsArg()
-	assert.Equal("0.0.0.0:2", ipsArg)
-	len = s.Len()
-	assert.Equal(1, len)
-}

utils/json.go

@@ -3,34 +3,26 @@
 
 package utils
 
-import (
-	"bufio"
-	"fmt"
-	"strings"
-)
+import "encoding/json"
 
-// Update the JSON body if the matching key is found
-// and replace the value.
+// Set k=v in JSON string
 // e.g., "whitelisted-subnets" is the key and value is "a,b,c".
-func UpdateJSONKey(jsonBody string, k string, v string) (string, error) {
-	k = fmt.Sprintf(`"%s":`, k)
-	lines := make([]string, 0)
-	scanner := bufio.NewScanner(strings.NewReader(jsonBody))
-	for scanner.Scan() {
-		line := scanner.Text()
-		if !strings.Contains(line, k) {
-			lines = append(lines, line)
-			continue
-		}
+func SetJSONKey(jsonBody string, k string, v string) (string, error) {
+	var config map[string]interface{}
 
-		idx := strings.Index(line, k)
-		if idx == -1 {
-			// should never happen...
-			return "", fmt.Errorf("line %q is missing the key %q", line, k)
-		}
-		line = line[:idx]
-		line += fmt.Sprintf(`%s"%s"`, k, v)
-		lines = append(lines, line)
+	if err := json.Unmarshal([]byte(jsonBody), &config); err != nil {
+		return "", err
 	}
-	return strings.Join(lines, "\n"), scanner.Err()
+
+	if v == "" {
+		delete(config, k)
+	} else {
+		config[k] = v
+	}
+
+	updatedJSON, err := json.Marshal(config)
+	if err != nil {
+		return "", err
+	}
+	return string(updatedJSON), nil
 }

utils/json_test.go

@@ -4,12 +4,13 @@
 package utils
 
 import (
+	"encoding/json"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestUpdateJSONKey(t *testing.T) {
+func TestSetJSONKey(t *testing.T) {
 	b := `{
 		"network-peer-list-gossip-frequency":"250ms",
 		"network-max-reconnect-delay":"1s",
@@ -22,10 +23,22 @@ func TestUpdateJSONKey(t *testing.T) {
 		"log-level":"INFO",
 		"log-dir":"INFO",
 		"db-dir":"INFO",
-		"plugin-dir":"INFO",
-		"whitelisted-subnets":"a,b,c"
+		"whitelisted-subnets":"a,b,c",
+		"plugin-dir":"INFO"
 	}`
-	s, err := UpdateJSONKey(b, "whitelisted-subnets", "d,e,f")
+	s, err := SetJSONKey(b, "whitelisted-subnets", "d,e,f")
 	assert.NoError(t, err)
 	assert.Contains(t, s, `"whitelisted-subnets":"d,e,f"`)
+	// now check it's actual correct JSON
+	var m map[string]interface{}
+	err = json.Unmarshal([]byte(s), &m)
+	assert.NoError(t, err)
+	// check if one-liner also works
+	bb := `{"api-admin-enabled":true,"api-ipcs-enabled":true,"db-dir":"/tmp/network-runner-root-data3856302950/node5/db-dir","health-check-frequency":"2s","index-enabled":true,"log-dir":"/tmp/network-runner-root-data3856302950/node5/log","log-display-level":"INFO","log-level":"INFO","network-max-reconnect-delay":"1s","network-peer-list-gossip-frequency":"250ms","plugin-dir":"/home/fabio/go/src/github.com/ava-labs/avalanchego/build/plugins","public-ip":"127.0.0.1","whitelisted-subnets":""}`
+	ss, err := SetJSONKey(bb, "whitelisted-subnets", "d,e,f")
+	assert.NoError(t, err)
+	assert.Contains(t, s, `"whitelisted-subnets":"d,e,f"`)
+	// also check here it's correct JSON
+	err = json.Unmarshal([]byte(ss), &m)
+	assert.NoError(t, err)
 }

utils/utils.go

@@ -6,20 +6,23 @@ import (
 	"fmt"
 	"io/fs"
 	"os"
+	"time"
 
 	"github.com/ava-labs/avalanchego/ids"
-	"github.com/ava-labs/avalanchego/network/peer"
 	"github.com/ava-labs/avalanchego/staking"
 )
 
-const genesisNetworkIDKey = "networkID"
+const (
+	genesisNetworkIDKey = "networkID"
+	dirTimestampFormat  = "20060102_150405"
+)
 
-func ToNodeID(stakingKey, stakingCert []byte) (ids.ShortID, error) {
+func ToNodeID(stakingKey, stakingCert []byte) (ids.NodeID, error) {
 	cert, err := staking.LoadTLSCertFromBytes(stakingKey, stakingCert)
 	if err != nil {
-		return ids.ShortID{}, err
+		return ids.EmptyNodeID, err
 	}
-	nodeID := peer.CertToID(cert.Leaf)
+	nodeID := ids.NodeIDFromCert(cert.Leaf)
 	return nodeID, nil
 }
 
@@ -40,29 +43,6 @@ func NetworkIDFromGenesis(genesis []byte) (uint32, error) {
 	return uint32(networkID), nil
 }
 
-// NewLocalNodeConfigJsonRaw returns a JSON formatted string as json.RawMessage for a
-// local node config object (ImplSpecificConfig)
-func NewLocalNodeConfigJsonRaw(binaryPath string) json.RawMessage {
-	return json.RawMessage(fmt.Sprintf(`{"binaryPath":"%s"}`, binaryPath))
-}
-
-// NewK8sNodeConfigJsonRaw returns a JSON formatted string as json.RawMessage for a
-// kubernetes node config object (ImplSpecificConfig)
-func NewK8sNodeConfigJsonRaw(
-	api,
-	id,
-	image,
-	kind,
-	namespace,
-	tag string,
-) json.RawMessage {
-	return json.RawMessage(
-		fmt.Sprintf(`{"apiVersion":"%s","identifier":"%s","image":"%s","kind":"%s","namespace":"%s","tag":"%s"}`,
-			api, id, image, kind, namespace, tag,
-		),
-	)
-}
-
 var (
 	ErrInvalidExecPath        = errors.New("avalanche exec is invalid")
 	ErrNotExists              = errors.New("avalanche exec not exists")
@@ -70,7 +50,7 @@ var (
 	ErrNotExistsPluginGenesis = errors.New("plugin genesis not exists")
 )
 
-func CheckExecPluginPaths(exec string, pluginExec string, pluginGenesisPath string) error {
+func CheckExecPath(exec string) error {
 	if exec == "" {
 		return ErrInvalidExecPath
 	}
@@ -81,14 +61,11 @@ func CheckExecPluginPaths(exec string, pluginExec string, pluginGenesisPath stri
 		}
 		return fmt.Errorf("failed to stat exec %q (%w)", exec, err)
 	}
+	return nil
+}
 
-	// no custom VM is specified
-	// no need to check further
-	// (subnet installation is optional)
-	if pluginExec == "" {
-		return nil
-	}
-
+func CheckPluginPaths(pluginExec string, pluginGenesisPath string) error {
+	var err error
 	if _, err = os.Stat(pluginExec); err != nil {
 		if errors.Is(err, fs.ErrNotExist) {
 			return ErrNotExistsPlugin
@@ -113,3 +90,9 @@ func VMID(vmName string) (ids.ID, error) {
 	copy(b, []byte(vmName))
 	return ids.ToID(b)
 }
+
+func MkDirWithTimestamp(dirPrefix string) (string, error) {
+	currentTime := time.Now().Format(dirTimestampFormat)
+	dirName := dirPrefix + "_" + currentTime
+	return dirName, os.MkdirAll(dirName, os.ModePerm)
+}

utils/utils_test.go

@@ -50,12 +50,40 @@ func TestExtractNetworkID(t *testing.T) {
 	assert.EqualValues(t, netID, 1337)
 }
 
-func TestCheckExecPluginPaths(t *testing.T) {
+func TestCheckExecPath(t *testing.T) {
 	execF, err := os.CreateTemp(os.TempDir(), "test-check-exec")
 	assert.NoError(t, err)
 	execPath := execF.Name()
 	assert.NoError(t, execF.Close())
 
+	t.Cleanup(func() {
+		os.RemoveAll(execPath)
+	})
+
+	tt := []struct {
+		execPath    string
+		expectedErr error
+	}{
+		{
+			execPath:    execPath,
+			expectedErr: nil,
+		},
+		{
+			execPath:    "",
+			expectedErr: ErrInvalidExecPath,
+		},
+		{
+			execPath:    "invalid",
+			expectedErr: ErrNotExists,
+		},
+	}
+	for i, tv := range tt {
+		err := CheckExecPath(tv.execPath)
+		assert.Equal(t, tv.expectedErr, err, fmt.Sprintf("[%d] unexpected error", i))
+	}
+}
+
+func TestCheckPluginPaths(t *testing.T) {
 	pluginF, err := os.CreateTemp(os.TempDir(), "test-check-exec-plugin")
 	assert.NoError(t, err)
 	pluginPath := pluginF.Name()
@@ -67,56 +95,33 @@ func TestCheckExecPluginPaths(t *testing.T) {
 	assert.NoError(t, genesisF.Close())
 
 	t.Cleanup(func() {
-		os.RemoveAll(execPath)
 		os.RemoveAll(pluginPath)
 		os.RemoveAll(genesisPath)
 	})
 
 	tt := []struct {
-		execPath    string
 		pluginPath  string
 		genesisPath string
 		expectedErr error
 	}{
 		{
-			execPath:    execPath,
-			pluginPath:  "",
-			genesisPath: "",
-			expectedErr: nil,
-		},
-		{
-			execPath:    execPath,
 			pluginPath:  pluginPath,
 			genesisPath: genesisPath,
 			expectedErr: nil,
 		},
 		{
-			execPath:    "",
-			pluginPath:  "",
-			genesisPath: "",
-			expectedErr: ErrInvalidExecPath,
-		},
-		{
-			execPath:    "invalid",
-			pluginPath:  "",
-			genesisPath: "",
-			expectedErr: ErrNotExists,
-		},
-		{
-			execPath:    execPath,
 			pluginPath:  "invalid",
 			genesisPath: "",
 			expectedErr: ErrNotExistsPlugin,
 		},
 		{
-			execPath:    execPath,
 			pluginPath:  pluginPath,
 			genesisPath: "invalid",
 			expectedErr: ErrNotExistsPluginGenesis,
 		},
 	}
 	for i, tv := range tt {
-		err := CheckExecPluginPaths(tv.execPath, tv.pluginPath, tv.genesisPath)
+		err := CheckPluginPaths(tv.pluginPath, tv.genesisPath)
 		assert.Equal(t, tv.expectedErr, err, fmt.Sprintf("[%d] unexpected error", i))
 	}
 }