diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 032795b6436e..f6afe6f1776f 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -42,7 +42,7 @@ Define any flags/configurations in [`e2e.go`](./e2e.go). Create a new package to implement feature-specific tests, or add tests to an existing package. For example: ``` -. +tests └── e2e ├── README.md ├── e2e.go diff --git a/tests/fixture/e2e/env.go b/tests/fixture/e2e/env.go index 1ea9dc69f2b9..107589cd7b44 100644 --- a/tests/fixture/e2e/env.go +++ b/tests/fixture/e2e/env.go @@ -69,7 +69,7 @@ func NewTestEnvironment(flagVars *FlagVars) *TestEnvironment { network = StartNetwork(flagVars.AvalancheGoExecPath(), DefaultNetworkDir) } - uris := tmpnet.GetNodeURIs(network.Nodes) + uris := network.GetNodeURIs() require.NotEmpty(uris, "network contains no nodes") tests.Outf("{{green}}network URIs: {{/}} %+v\n", uris) @@ -132,5 +132,5 @@ func (te *TestEnvironment) NewPrivateNetwork() *tmpnet.Network { privateNetworksDir := filepath.Join(sharedNetwork.Dir, PrivateNetworksDirName) te.require.NoError(os.MkdirAll(privateNetworksDir, perms.ReadWriteExecute)) - return StartNetwork(sharedNetwork.AvalancheGoPath, privateNetworksDir) + return StartNetwork(sharedNetwork.DefaultRuntimeConfig.AvalancheGoPath, privateNetworksDir) } diff --git a/tests/fixture/e2e/helpers.go b/tests/fixture/e2e/helpers.go index ddebfbc8b81c..6ec1717c1390 100644 --- a/tests/fixture/e2e/helpers.go +++ b/tests/fixture/e2e/helpers.go @@ -201,22 +201,14 @@ func CheckBootstrapIsPossible(network *tmpnet.Network) { } // Start a temporary network with the provided avalanchego binary. -func StartNetwork(avalancheGoExecPath string, networkDir string) *tmpnet.Network { +func StartNetwork(avalancheGoExecPath string, rootNetworkDir string) *tmpnet.Network { require := require.New(ginkgo.GinkgoT()) - network, err := tmpnet.StartNetwork( - DefaultContext(), - ginkgo.GinkgoWriter, - networkDir, - &tmpnet.Network{ - NodeRuntimeConfig: tmpnet.NodeRuntimeConfig{ - AvalancheGoPath: avalancheGoExecPath, - }, - }, - tmpnet.DefaultNodeCount, - tmpnet.DefaultPreFundedKeyCount, - ) + network, err := tmpnet.NewDefaultNetwork(ginkgo.GinkgoWriter, avalancheGoExecPath, tmpnet.DefaultNodeCount) require.NoError(err) + require.NoError(network.Create(rootNetworkDir)) + require.NoError(network.Start(DefaultContext(), ginkgo.GinkgoWriter)) + ginkgo.DeferCleanup(func() { tests.Outf("Shutting down network\n") ctx, cancel := context.WithTimeout(context.Background(), DefaultTimeout) diff --git a/tests/fixture/tmpnet/README.md b/tests/fixture/tmpnet/README.md index 90ec7f678b2f..abccbf52cf79 100644 --- a/tests/fixture/tmpnet/README.md +++ b/tests/fixture/tmpnet/README.md @@ -30,6 +30,7 @@ the following non-test files: | flags.go | FlagsMap | Simplifies configuration of avalanchego flags | | genesis.go | | Creates test genesis | | network.go | Network | Orchestrates and configures temporary networks | +| network_config.go | Network | Reads and writes network configuration | | node.go | Node | Orchestrates and configures nodes | | node_config.go | Node | Reads and writes node configuration | | node_process.go | NodeProcess | Orchestrates node processes | @@ -73,60 +74,25 @@ network. A temporary network can be managed in code: ```golang -network, _ := tmpnet.StartNetwork( - ctx, // Context used to limit duration of waiting for network health - ginkgo.GinkgoWriter, // Writer to report progress of network start - "", // Use default root dir (~/.tmpnet) - &tmpnet.Network{ - NodeRuntimeConfig: tmpnet.NodeRuntimeConfig{ - ExecPath: "/path/to/avalanchego", // Defining the avalanchego exec path is required - }, - }, - 5, // Number of initial validating nodes - 50, // Number of pre-funded keys to create +network, _ := tmpnet.NewDefaultNetwork( + ginkgo.GinkgoWriter, // Writer to report progress of initialization + "/path/to/avalanchego", // The path to the binary that nodes will execute + 5, // Number of initial validating nodes +) +_ = network.Create("") // Finalize network configuration and write to disk +_ = network.Start( // Start the nodes of the network and wait until they report healthy + ctx, // Context used to limit duration of waiting for network health + ginkgo.GinkgoWriter, // Writer to report progress of network start ) -uris := network.GetURIs() +uris := network.GetNodeURIs() // Use URIs to interact with the network // Stop all nodes in the network -network.Stop() +network.Stop(context.Background()) ``` -If non-default node behavior is required, the `Network` instance -supplied to `StartNetwork()` can be initialized with explicit node -configuration and by supplying a nodeCount argument of `0`: - -```golang -network, _ := tmpnet.StartNetwork( - ctx, - ginkgo.GinkgoWriter, - "", - &tmpnet.Network{ - NodeRuntimeConfig: tmpnet.NodeRuntimeConfig{ - ExecPath: "/path/to/avalanchego", - }, - Nodes: []*Node{ - { // node1 configuration is customized - Flags: FlagsMap{ // Any and all node flags can be configured here - config.DataDirKey: "/custom/path/to/node/data", - } - }, - }, - {}, // node2 uses default configuration - {}, // node3 uses default configuration - {}, // node4 uses default configuration - {}, // node5 uses default configuration - }, - 0, // Node count must be zero when setting node config - 50, -) -``` - -Further examples of code-based usage are located in the [e2e -tests](../../e2e/e2e_test.go). - ## Networking configuration By default, nodes in a temporary network will be started with staking and @@ -161,18 +127,18 @@ HOME ├── chains │ └── C │ └── config.json // C-Chain config for all nodes - ├── defaults.json // Default flags and configuration for network + ├── config.json // Common configuration (including defaults and pre-funded keys) ├── genesis.json // Genesis for all nodes └── network.env // Sets network dir env var to simplify network usage ``` -### Default flags and configuration +### Common networking configuration -The default avalanchego node flags (e.g. `--staking-port=`) and -default configuration like the avalanchego path are stored at -`[network-dir]/defaults.json`. The value for a given defaulted flag -will be set on initial and subsequently added nodes that do not supply -values for a given defaulted flag. +Network configuration such as default flags (e.g. `--log-level=`), +runtime defaults (e.g. avalanchego path) and pre-funded private keys +are stored at `[network-dir]/config.json`. A given default will only +be applied to a new node on its addition to the network if the node +does not explicitly set a given value. ### Genesis diff --git a/tests/fixture/tmpnet/cmd/main.go b/tests/fixture/tmpnet/cmd/main.go index 9baf4557bca4..a36856185d38 100644 --- a/tests/fixture/tmpnet/cmd/main.go +++ b/tests/fixture/tmpnet/cmd/main.go @@ -10,6 +10,7 @@ import ( "io/fs" "os" "path/filepath" + "time" "github.com/spf13/cobra" @@ -45,10 +46,9 @@ func main() { rootCmd.AddCommand(versionCmd) var ( - rootDir string - execPath string - nodeCount uint8 - preFundedKeyCount uint8 + rootDir string + execPath string + nodeCount uint8 ) startNetworkCmd := &cobra.Command{ Use: "start-network", @@ -60,15 +60,21 @@ func main() { // Root dir will be defaulted on start if not provided - network := &tmpnet.Network{ - NodeRuntimeConfig: tmpnet.NodeRuntimeConfig{ - AvalancheGoPath: execPath, - }, + network, err := tmpnet.NewDefaultNetwork(os.Stdout, execPath, int(nodeCount)) + if err != nil { + return err } - ctx, cancel := context.WithTimeout(context.Background(), tmpnet.DefaultNetworkTimeout) + + if err := network.Create(rootDir); err != nil { + return err + } + + // Extreme upper bound, should never take this long + networkStartTimeout := 2 * time.Minute + + ctx, cancel := context.WithTimeout(context.Background(), networkStartTimeout) defer cancel() - network, err := tmpnet.StartNetwork(ctx, os.Stdout, rootDir, network, int(nodeCount), int(preFundedKeyCount)) - if err != nil { + if err := network.Start(ctx, os.Stdout); err != nil { return err } @@ -94,7 +100,6 @@ func main() { startNetworkCmd.PersistentFlags().StringVar(&rootDir, "root-dir", os.Getenv(tmpnet.RootDirEnvName), "The path to the root directory for temporary networks") startNetworkCmd.PersistentFlags().StringVar(&execPath, "avalanchego-path", os.Getenv(tmpnet.AvalancheGoPathEnvName), "The path to an avalanchego binary") startNetworkCmd.PersistentFlags().Uint8Var(&nodeCount, "node-count", tmpnet.DefaultNodeCount, "Number of nodes the network should initially consist of") - startNetworkCmd.PersistentFlags().Uint8Var(&preFundedKeyCount, "pre-funded-key-count", tmpnet.DefaultPreFundedKeyCount, "Number of pre-funded keys the network should start with") rootCmd.AddCommand(startNetworkCmd) var networkDir string diff --git a/tests/fixture/tmpnet/defaults.go b/tests/fixture/tmpnet/defaults.go index 11e9ab72787d..1112e3345307 100644 --- a/tests/fixture/tmpnet/defaults.go +++ b/tests/fixture/tmpnet/defaults.go @@ -10,11 +10,6 @@ import ( ) const ( - // Constants defining the names of shell variables whose value can - // configure temporary network orchestration. - NetworkDirEnvName = "TMPNET_NETWORK_DIR" - RootDirEnvName = "TMPNET_ROOT_DIR" - DefaultNetworkTimeout = 2 * time.Minute // Minimum required to ensure connectivity-based health checks will pass @@ -48,12 +43,14 @@ func DefaultFlags() FlagsMap { } } -// C-Chain config for testing. -func DefaultCChainConfig() FlagsMap { - // Supply only non-default configuration to ensure that default - // values will be used. Available C-Chain configuration options are - // defined in the `github.com/ava-labs/coreth/evm` package. - return FlagsMap{ - "log-level": "trace", +// A set of chain configurations appropriate for testing. +func DefaultChainConfigs() map[string]FlagsMap { + return map[string]FlagsMap{ + // Supply only non-default configuration to ensure that default + // values will be used. Available C-Chain configuration options are + // defined in the `github.com/ava-labs/coreth/evm` package. + "C": { + "log-level": "trace", + }, } } diff --git a/tests/fixture/tmpnet/network.go b/tests/fixture/tmpnet/network.go index 5f1eaf5a4513..266ad922087d 100644 --- a/tests/fixture/tmpnet/network.go +++ b/tests/fixture/tmpnet/network.go @@ -5,7 +5,6 @@ package tmpnet import ( "context" - "encoding/json" "errors" "fmt" "io" @@ -24,114 +23,121 @@ import ( "github.com/ava-labs/avalanchego/utils/set" ) +// The Network type is defined in this file (network.go - orchestration) and +// network.go (reading/writing configuration). + const ( + // Constants defining the names of shell variables whose value can + // configure network orchestration. + NetworkDirEnvName = "TMPNET_NETWORK_DIR" + RootDirEnvName = "TMPNET_ROOT_DIR" + // This interval was chosen to avoid spamming node APIs during // startup, as smaller intervals (e.g. 50ms) seemed to noticeably // increase the time for a network's nodes to be seen as healthy. networkHealthCheckInterval = 200 * time.Millisecond - - defaultEphemeralDirName = "ephemeral" ) -var ( - errInvalidNodeCount = errors.New("failed to populate network config: non-zero node count is only valid for a network without nodes") - errInvalidKeyCount = errors.New("failed to populate network config: non-zero key count is only valid for a network without keys") - errNetworkDirNotSet = errors.New("network directory not set - has Create() been called?") - errInvalidNetworkDir = errors.New("failed to write network: invalid network directory") - errMissingBootstrapNodes = errors.New("failed to add node due to missing bootstrap nodes") -) +// Collects the configuration for running a temporary avalanchego network +type Network struct { + // Path where network configuration and data is stored + Dir string -// Default root dir for storing networks and their configuration. -func GetDefaultRootDir() (string, error) { - homeDir, err := os.UserHomeDir() - if err != nil { - return "", err - } - return filepath.Join(homeDir, ".tmpnet", "networks"), nil -} + // Configuration common across nodes + Genesis *genesis.UnparsedConfig + ChainConfigs map[string]FlagsMap -// Find the next available network ID by attempting to create a -// directory numbered from 1000 until creation succeeds. Returns the -// network id and the full path of the created directory. -func FindNextNetworkID(rootDir string) (uint32, string, error) { - var ( - networkID uint32 = 1000 - dirPath string - ) - for { - _, reserved := constants.NetworkIDToNetworkName[networkID] - if reserved { - networkID++ - continue - } + // Default configuration to use when creating new nodes + DefaultFlags FlagsMap + DefaultRuntimeConfig NodeRuntimeConfig - dirPath = filepath.Join(rootDir, strconv.FormatUint(uint64(networkID), 10)) - err := os.Mkdir(dirPath, perms.ReadWriteExecute) - if err == nil { - return networkID, dirPath, nil - } + // Keys pre-funded in the genesis on both the X-Chain and the C-Chain + PreFundedKeys []*secp256k1.PrivateKey - if !errors.Is(err, fs.ErrExist) { - return 0, "", fmt.Errorf("failed to create network directory: %w", err) - } + // Nodes that constitute the network + Nodes []*Node +} - // Directory already exists, keep iterating - networkID++ +// Ensure a real and absolute network dir so that node +// configuration that embeds the network path will continue to +// work regardless of symlink and working directory changes. +func toCanonicalDir(dir string) (string, error) { + absDir, err := filepath.Abs(dir) + if err != nil { + return "", err } + return filepath.EvalSymlinks(absDir) } -// Defines the configuration required for a tempoary network -type Network struct { - NodeRuntimeConfig +// Initializes a new network with default configuration. +func NewDefaultNetwork(w io.Writer, avalancheGoPath string, nodeCount int) (*Network, error) { + if _, err := fmt.Fprintf(w, "Preparing configuration for new network with %s\n", avalancheGoPath); err != nil { + return nil, err + } - Genesis *genesis.UnparsedConfig - CChainConfig FlagsMap - DefaultFlags FlagsMap - PreFundedKeys []*secp256k1.PrivateKey + keys, err := NewPrivateKeys(DefaultPreFundedKeyCount) + if err != nil { + return nil, err + } - // Nodes comprising the network - Nodes []*Node + network := &Network{ + DefaultFlags: DefaultFlags(), + DefaultRuntimeConfig: NodeRuntimeConfig{ + AvalancheGoPath: avalancheGoPath, + }, + PreFundedKeys: keys, + ChainConfigs: DefaultChainConfigs(), + } - // Path where network configuration will be stored - Dir string + network.Nodes = make([]*Node, nodeCount) + for i := range network.Nodes { + network.Nodes[i] = NewNode("") + if err := network.EnsureNodeConfig(network.Nodes[i]); err != nil { + return nil, err + } + } + + return network, nil } -// Adds a backend-agnostic ephemeral node to the network -func (n *Network) AddEphemeralNode(ctx context.Context, w io.Writer, flags FlagsMap) (*Node, error) { - if flags == nil { - flags = FlagsMap{} +// Stops the nodes of the network configured in the provided directory. +func StopNetwork(ctx context.Context, dir string) error { + network, err := ReadNetwork(dir) + if err != nil { + return err } - return n.AddNode(ctx, w, &Node{ - Flags: flags, - }, true /* isEphemeral */) + return network.Stop(ctx) } -// Starts a new network stored under the provided root dir. Required -// configuration will be defaulted if not provided. -func StartNetwork( - ctx context.Context, - w io.Writer, - rootDir string, - network *Network, - nodeCount int, - keyCount int, -) (*Network, error) { - if _, err := fmt.Fprintf(w, "Preparing configuration for new temporary network with %s\n", network.AvalancheGoPath); err != nil { +// Reads a network from the provided directory. +func ReadNetwork(dir string) (*Network, error) { + canonicalDir, err := toCanonicalDir(dir) + if err != nil { return nil, err } + network := &Network{ + Dir: canonicalDir, + } + if err := network.Read(); err != nil { + return nil, fmt.Errorf("failed to read network: %w", err) + } + return network, nil +} +// Creates the network on disk, choosing its network id and generating its genesis in the process. +func (n *Network) Create(rootDir string) error { if len(rootDir) == 0 { // Use the default root dir var err error - rootDir, err = GetDefaultRootDir() + rootDir, err = getDefaultRootDir() if err != nil { - return nil, err + return err } } // Ensure creation of the root dir if err := os.MkdirAll(rootDir, perms.ReadWriteExecute); err != nil { - return nil, fmt.Errorf("failed to create root network dir: %w", err) + return fmt.Errorf("failed to create root network dir: %w", err) } // Determine the network path and ID @@ -139,107 +145,30 @@ func StartNetwork( networkDir string networkID uint32 ) - if network.Genesis != nil && network.Genesis.NetworkID > 0 { + if n.Genesis != nil && n.Genesis.NetworkID > 0 { // Use the network ID defined in the provided genesis - networkID = network.Genesis.NetworkID + networkID = n.Genesis.NetworkID } if networkID > 0 { // Use a directory with a random suffix var err error - networkDir, err = os.MkdirTemp(rootDir, fmt.Sprintf("%d.", network.Genesis.NetworkID)) + networkDir, err = os.MkdirTemp(rootDir, fmt.Sprintf("%d.", n.Genesis.NetworkID)) if err != nil { - return nil, fmt.Errorf("failed to create network dir: %w", err) + return fmt.Errorf("failed to create network dir: %w", err) } } else { // Find the next available network ID based on the contents of the root dir var err error - networkID, networkDir, err = FindNextNetworkID(rootDir) + networkID, networkDir, err = findNextNetworkID(rootDir) if err != nil { - return nil, err + return err } } - - // Setting the network dir before populating config ensures the - // nodes know where to write their configuration. - network.Dir = networkDir - - if err := network.PopulateNetworkConfig(networkID, nodeCount, keyCount); err != nil { - return nil, err - } - - if err := network.WriteAll(); err != nil { - return nil, err - } - if _, err := fmt.Fprintf(w, "Starting network %d @ %s\n", network.Genesis.NetworkID, network.Dir); err != nil { - return nil, err - } - if err := network.Start(w); err != nil { - return nil, err - } - if _, err := fmt.Fprintf(w, "Waiting for all nodes to report healthy...\n\n"); err != nil { - return nil, err - } - if err := network.WaitForHealthy(ctx, w); err != nil { - return nil, err - } - if _, err := fmt.Fprintf(w, "\nStarted network %d @ %s\n", network.Genesis.NetworkID, network.Dir); err != nil { - return nil, err - } - return network, nil -} - -// Read a network from the provided directory. -func ReadNetwork(dir string) (*Network, error) { - network := &Network{Dir: dir} - if err := network.ReadAll(); err != nil { - return nil, fmt.Errorf("failed to read network: %w", err) - } - return network, nil -} - -// Stop the nodes of the network configured in the provided directory. -func StopNetwork(ctx context.Context, dir string) error { - network, err := ReadNetwork(dir) + canonicalDir, err := toCanonicalDir(networkDir) if err != nil { return err } - return network.Stop(ctx) -} - -// Ensure the network has the configuration it needs to start. -func (n *Network) PopulateNetworkConfig(networkID uint32, nodeCount int, keyCount int) error { - if len(n.Nodes) > 0 && nodeCount > 0 { - return errInvalidNodeCount - } - if len(n.PreFundedKeys) > 0 && keyCount > 0 { - return errInvalidKeyCount - } - - if nodeCount > 0 { - // Add the specified number of nodes - nodes := make([]*Node, 0, nodeCount) - for i := 0; i < nodeCount; i++ { - nodes = append(nodes, NewNode("")) - } - n.Nodes = nodes - } - - // Ensure each node has keys and an associated node ID. This - // ensures the availability of node IDs and proofs of possession - // for genesis generation. - for _, node := range n.Nodes { - if err := node.EnsureKeys(); err != nil { - return err - } - } - - if keyCount > 0 { - keys, err := NewPrivateKeys(keyCount) - if err != nil { - return err - } - n.PreFundedKeys = keys - } + n.Dir = canonicalDir if n.Genesis == nil { genesis, err := NewTestGenesis(networkID, n.Nodes, n.PreFundedKeys) @@ -249,117 +178,84 @@ func (n *Network) PopulateNetworkConfig(networkID uint32, nodeCount int, keyCoun n.Genesis = genesis } - if n.CChainConfig == nil { - n.CChainConfig = DefaultCChainConfig() - } - - // Default flags need to be set in advance of node config - // population to ensure correct node configuration. - if n.DefaultFlags == nil { - n.DefaultFlags = DefaultFlags() - } - for _, node := range n.Nodes { // Ensure the node is configured for use with the network and // knows where to write its configuration. - if err := n.PopulateNodeConfig(node, n.Dir); err != nil { - return err + if err := n.EnsureNodeConfig(node); err != nil { + return nil } } - return nil + // Ensure configuration on disk is current + return n.Write() } -// Ensure the provided node has the configuration it needs to start. If the data dir is -// not set, it will be defaulted to [nodeParentDir]/[node ID]. Requires that the -// network has valid genesis data. -func (n *Network) PopulateNodeConfig(node *Node, nodeParentDir string) error { - flags := node.Flags - - // Set values common to all nodes - flags.SetDefaults(n.DefaultFlags) - flags.SetDefaults(FlagsMap{ - config.GenesisFileKey: n.GetGenesisPath(), - config.ChainConfigDirKey: n.GetChainConfigDir(), - }) - - // Convert the network id to a string to ensure consistency in JSON round-tripping. - flags[config.NetworkNameKey] = strconv.FormatUint(uint64(n.Genesis.NetworkID), 10) - - // Ensure keys are added if necessary - if err := node.EnsureKeys(); err != nil { +// Starts all nodes in the network +func (n *Network) Start(ctx context.Context, w io.Writer) error { + if _, err := fmt.Fprintf(w, "Starting network %d @ %s\n", n.Genesis.NetworkID, n.Dir); err != nil { return err } - // Ensure the node is configured with a runtime config - if node.RuntimeConfig == nil { - node.RuntimeConfig = &NodeRuntimeConfig{ - AvalancheGoPath: n.AvalancheGoPath, + // Configure the networking for each node and start + for _, node := range n.Nodes { + if err := n.StartNode(ctx, w, node); err != nil { + return err } } - // Ensure the node's data dir is configured - dataDir := node.getDataDir() - if len(dataDir) == 0 { - // NodeID will have been set by EnsureKeys - dataDir = filepath.Join(nodeParentDir, node.NodeID.String()) - flags[config.DataDirKey] = dataDir + if _, err := fmt.Fprintf(w, "Waiting for all nodes to report healthy...\n\n"); err != nil { + return err + } + if err := n.WaitForHealthy(ctx, w); err != nil { + return err + } + if _, err := fmt.Fprintf(w, "\nStarted network %d @ %s\n", n.Genesis.NetworkID, n.Dir); err != nil { + return err } return nil } -// Starts a network for the first time -func (n *Network) Start(w io.Writer) error { - if len(n.Dir) == 0 { - return errNetworkDirNotSet +func (n *Network) AddEphemeralNode(ctx context.Context, w io.Writer, flags FlagsMap) (*Node, error) { + node := NewNode("") + node.Flags = flags + node.IsEphemeral = true + if err := n.StartNode(ctx, w, node); err != nil { + return nil, err } + return node, nil +} - // Ensure configuration on disk is current - if err := n.WriteAll(); err != nil { +// Starts the provided node after configuring it for the network. +func (n *Network) StartNode(ctx context.Context, w io.Writer, node *Node) error { + if err := n.EnsureNodeConfig(node); err != nil { return err } - // Accumulate bootstrap nodes such that each subsequently started - // node bootstraps from the nodes previously started. - // - // e.g. - // 1st node: no bootstrap nodes - // 2nd node: 1st node - // 3rd node: 1st and 2nd nodes - // ... - // - bootstrapIDs := make([]string, 0, len(n.Nodes)) - bootstrapIPs := make([]string, 0, len(n.Nodes)) - - // Configure networking and start each node - for _, node := range n.Nodes { - // Update network configuration - node.SetNetworkingConfig(bootstrapIDs, bootstrapIPs) + bootstrapIPs, bootstrapIDs, err := n.getBootstrapIPsAndIDs(node) + if err != nil { + return err + } + node.SetNetworkingConfig(bootstrapIDs, bootstrapIPs) - // Write configuration to disk in preparation for node start - if err := node.Write(); err != nil { - return err - } + if err := node.Write(); err != nil { + return err + } - // Start waits for the process context to be written which - // indicates that the node will be accepting connections on - // its staking port. The network will start faster with this - // synchronization due to the avoidance of exponential backoff - // if a node tries to connect to a beacon that is not ready. - if err := node.Start(w); err != nil { - return err + if err := node.Start(w); err != nil { + // Attempt to stop an unhealthy node to provide some assurance to the caller + // that an error condition will not result in a lingering process. + stopErr := node.Stop(ctx) + if stopErr != nil { + err = errors.Join(err, stopErr) } - - // Collect bootstrap nodes for subsequently started nodes to use - bootstrapIDs = append(bootstrapIDs, node.NodeID.String()) - bootstrapIPs = append(bootstrapIPs, node.StakingAddress) + return err } return nil } -// Wait until all nodes in the network are healthy. +// Waits until all nodes in the network are healthy. func (n *Network) WaitForHealthy(ctx context.Context, w io.Writer) error { ticker := time.NewTicker(networkHealthCheckInterval) defer ticker.Stop() @@ -394,303 +290,150 @@ func (n *Network) WaitForHealthy(ctx context.Context, w io.Writer) error { return nil } -// Retrieve API URIs for all running primary validator nodes. URIs for -// ephemeral nodes are not returned. -func (n *Network) GetURIs() []NodeURI { - uris := make([]NodeURI, 0, len(n.Nodes)) - for _, node := range n.Nodes { - // Only append URIs that are not empty. A node may have an - // empty URI if it was not running at the time - // node.ReadProcessContext() was called. - if len(node.URI) > 0 { - uris = append(uris, NodeURI{ - NodeID: node.NodeID, - URI: node.URI, - }) - } +// Stops all nodes in the network. +func (n *Network) Stop(ctx context.Context) error { + // Target all nodes, including the ephemeral ones + nodes, err := ReadNodes(n.Dir, true /* includeEphemeral */) + if err != nil { + return err } - return uris -} -// Stop all nodes in the network. -func (n *Network) Stop(ctx context.Context) error { var errs []error - // Assume the nodes are loaded and the pids are current - for _, node := range n.Nodes { - if err := node.Stop(ctx); err != nil { + + // Initiate stop on all nodes + for _, node := range nodes { + if err := node.InitiateStop(); err != nil { errs = append(errs, fmt.Errorf("failed to stop node %s: %w", node.NodeID, err)) } } - if len(errs) > 0 { - return fmt.Errorf("failed to stop network:\n%w", errors.Join(errs...)) - } - return nil -} - -func (n *Network) GetGenesisPath() string { - return filepath.Join(n.Dir, "genesis.json") -} - -func (n *Network) ReadGenesis() error { - bytes, err := os.ReadFile(n.GetGenesisPath()) - if err != nil { - return fmt.Errorf("failed to read genesis: %w", err) - } - genesis := genesis.UnparsedConfig{} - if err := json.Unmarshal(bytes, &genesis); err != nil { - return fmt.Errorf("failed to unmarshal genesis: %w", err) - } - n.Genesis = &genesis - return nil -} -func (n *Network) WriteGenesis() error { - bytes, err := DefaultJSONMarshal(n.Genesis) - if err != nil { - return fmt.Errorf("failed to marshal genesis: %w", err) - } - if err := os.WriteFile(n.GetGenesisPath(), bytes, perms.ReadWrite); err != nil { - return fmt.Errorf("failed to write genesis: %w", err) + // Wait for stop to complete on all nodes + for _, node := range nodes { + if err := node.WaitForStopped(ctx); err != nil { + errs = append(errs, fmt.Errorf("failed to wait for node %s to stop: %w", node.NodeID, err)) + } } - return nil -} - -func (n *Network) GetChainConfigDir() string { - return filepath.Join(n.Dir, "chains") -} - -func (n *Network) GetCChainConfigPath() string { - return filepath.Join(n.GetChainConfigDir(), "C", "config.json") -} -func (n *Network) ReadCChainConfig() error { - chainConfig, err := ReadFlagsMap(n.GetCChainConfigPath(), "C-Chain config") - if err != nil { - return err + if len(errs) > 0 { + return fmt.Errorf("failed to stop network:\n%w", errors.Join(errs...)) } - n.CChainConfig = *chainConfig return nil } -func (n *Network) WriteCChainConfig() error { - path := n.GetCChainConfigPath() - dir := filepath.Dir(path) - if err := os.MkdirAll(dir, perms.ReadWriteExecute); err != nil { - return fmt.Errorf("failed to create C-Chain config dir: %w", err) - } - return n.CChainConfig.Write(path, "C-Chain config") -} - -// Used to marshal/unmarshal persistent network defaults. -type networkDefaults struct { - Flags FlagsMap - AvalancheGoPath string - PreFundedKeys []*secp256k1.PrivateKey -} - -func (n *Network) GetDefaultsPath() string { - return filepath.Join(n.Dir, "defaults.json") -} +// Ensures the provided node has the configuration it needs to start. If the data dir is not +// set, it will be defaulted to [nodeParentDir]/[node ID]. For a not-yet-created network, +// no action will be taken. +// TODO(marun) Reword or refactor to account for the differing behavior pre- vs post-start +func (n *Network) EnsureNodeConfig(node *Node) error { + flags := node.Flags -func (n *Network) ReadDefaults() error { - bytes, err := os.ReadFile(n.GetDefaultsPath()) - if err != nil { - return fmt.Errorf("failed to read defaults: %w", err) + // Set the network name if available + if n.Genesis != nil && n.Genesis.NetworkID > 0 { + // Convert the network id to a string to ensure consistency in JSON round-tripping. + flags[config.NetworkNameKey] = strconv.FormatUint(uint64(n.Genesis.NetworkID), 10) } - defaults := networkDefaults{} - if err := json.Unmarshal(bytes, &defaults); err != nil { - return fmt.Errorf("failed to unmarshal defaults: %w", err) - } - n.DefaultFlags = defaults.Flags - n.AvalancheGoPath = defaults.AvalancheGoPath - n.PreFundedKeys = defaults.PreFundedKeys - return nil -} -func (n *Network) WriteDefaults() error { - defaults := networkDefaults{ - Flags: n.DefaultFlags, - AvalancheGoPath: n.AvalancheGoPath, - PreFundedKeys: n.PreFundedKeys, - } - bytes, err := DefaultJSONMarshal(defaults) - if err != nil { - return fmt.Errorf("failed to marshal defaults: %w", err) - } - if err := os.WriteFile(n.GetDefaultsPath(), bytes, perms.ReadWrite); err != nil { - return fmt.Errorf("failed to write defaults: %w", err) + if err := node.EnsureKeys(); err != nil { + return err } - return nil -} -func (n *Network) EnvFilePath() string { - return filepath.Join(n.Dir, "network.env") -} - -func (n *Network) EnvFileContents() string { - return fmt.Sprintf("export %s=%s", NetworkDirEnvName, n.Dir) -} + flags.SetDefaults(n.DefaultFlags) -// Write an env file that sets the network dir env when sourced. -func (n *Network) WriteEnvFile() error { - if err := os.WriteFile(n.EnvFilePath(), []byte(n.EnvFileContents()), perms.ReadWrite); err != nil { - return fmt.Errorf("failed to write network env file: %w", err) + // Set fields including the network path + if len(n.Dir) > 0 { + node.Flags.SetDefaults(FlagsMap{ + config.GenesisFileKey: n.getGenesisPath(), + config.ChainConfigDirKey: n.getChainConfigDir(), + }) + + // Ensure the node's data dir is configured + dataDir := node.getDataDir() + if len(dataDir) == 0 { + // NodeID will have been set by EnsureKeys + dataDir = filepath.Join(n.Dir, node.NodeID.String()) + flags[config.DataDirKey] = dataDir + } } - return nil -} -func (n *Network) WriteNodes() error { - for _, node := range n.Nodes { - if err := node.Write(); err != nil { - return err + // Ensure the node runtime is configured + if node.RuntimeConfig == nil { + node.RuntimeConfig = &NodeRuntimeConfig{ + AvalancheGoPath: n.DefaultRuntimeConfig.AvalancheGoPath, } } - return nil -} -// Write network configuration to disk. -func (n *Network) WriteAll() error { - if len(n.Dir) == 0 { - return errInvalidNetworkDir - } - if err := n.WriteGenesis(); err != nil { - return err - } - if err := n.WriteCChainConfig(); err != nil { - return err - } - if err := n.WriteDefaults(); err != nil { - return err - } - if err := n.WriteEnvFile(); err != nil { - return err - } - return n.WriteNodes() + return nil } -// Read network configuration from disk. -func (n *Network) ReadConfig() error { - if err := n.ReadGenesis(); err != nil { - return err - } - if err := n.ReadCChainConfig(); err != nil { - return err - } - return n.ReadDefaults() +func (n *Network) GetNodeURIs() []NodeURI { + return GetNodeURIs(n.Nodes) } -// Read node configuration and process context from disk. -func (n *Network) ReadNodes() error { - nodes := []*Node{} - - // Node configuration / process context is stored in child directories - entries, err := os.ReadDir(n.Dir) +// Retrieves bootstrap IPs and IDs for all nodes except the skipped one (this supports +// collecting the bootstrap details for restarting a node). +func (n *Network) getBootstrapIPsAndIDs(skippedNode *Node) ([]string, []string, error) { + // Collect staking addresses of non-ephemeral nodes for use in bootstrapping a node + nodes, err := ReadNodes(n.Dir, false /* includeEphemeral */) if err != nil { - return fmt.Errorf("failed to read network path: %w", err) + return nil, nil, fmt.Errorf("failed to read network's nodes: %w", err) } - for _, entry := range entries { - if !entry.IsDir() { + var ( + bootstrapIPs = make([]string, 0, len(nodes)) + bootstrapIDs = make([]string, 0, len(nodes)) + ) + for _, node := range nodes { + if skippedNode != nil && node.NodeID == skippedNode.NodeID { continue } - nodeDir := filepath.Join(n.Dir, entry.Name()) - node, err := ReadNode(nodeDir) - if errors.Is(err, os.ErrNotExist) { - // If no config file exists, assume this is not the path of a node + if len(node.StakingAddress) == 0 { + // Node is not running continue - } else if err != nil { - return err } - nodes = append(nodes, node) + bootstrapIPs = append(bootstrapIPs, node.StakingAddress) + bootstrapIDs = append(bootstrapIDs, node.NodeID.String()) } - n.Nodes = nodes - - return nil -} - -// Read network and node configuration from disk. -func (n *Network) ReadAll() error { - if err := n.ReadConfig(); err != nil { - return err - } - return n.ReadNodes() + return bootstrapIPs, bootstrapIDs, nil } -func (n *Network) AddNode(ctx context.Context, w io.Writer, node *Node, isEphemeral bool) (*Node, error) { - // Assume network configuration has been written to disk and is current in memory - - if node == nil { - // Set an empty data dir so that PopulateNodeConfig will know - // to set the default of `[network dir]/[node id]`. - node = NewNode("") - } - - // Default to a data dir of [network-dir]/[node-ID] - nodeParentDir := n.Dir - if isEphemeral { - // For an ephemeral node, default to a data dir of [network-dir]/[ephemeral-dir]/[node-ID] - // to provide a clear separation between nodes that are expected to expose stable API - // endpoints and those that will live for only a short time (e.g. a node started by a test - // and stopped on teardown). - // - // The data for an ephemeral node is still stored in the file tree rooted at the network - // dir to ensure that recursively archiving the network dir in CI will collect all node - // data used for a test run. - nodeParentDir = filepath.Join(n.Dir, defaultEphemeralDirName) - } - - if err := n.PopulateNodeConfig(node, nodeParentDir); err != nil { - return nil, err - } - - bootstrapIPs, bootstrapIDs, err := n.GetBootstrapIPsAndIDs() - if err != nil { - return nil, err - } - node.SetNetworkingConfig(bootstrapIDs, bootstrapIPs) - - if err := node.Write(); err != nil { - return nil, err - } - - err = node.Start(w) +// Retrieves the default root dir for storing networks and their +// configuration. +func getDefaultRootDir() (string, error) { + homeDir, err := os.UserHomeDir() if err != nil { - // Attempt to stop an unhealthy node to provide some assurance to the caller - // that an error condition will not result in a lingering process. - stopErr := node.Stop(ctx) - if stopErr != nil { - err = errors.Join(err, stopErr) - } - return nil, err + return "", err } - - return node, nil + return filepath.Join(homeDir, ".tmpnet", "networks"), nil } -func (n *Network) GetBootstrapIPsAndIDs() ([]string, []string, error) { - // Collect staking addresses of running nodes for use in bootstrapping a node - if err := n.ReadNodes(); err != nil { - return nil, nil, fmt.Errorf("failed to read network nodes: %w", err) - } +// Finds the next available network ID by attempting to create a +// directory numbered from 1000 until creation succeeds. Returns the +// network id and the full path of the created directory. +func findNextNetworkID(rootDir string) (uint32, string, error) { var ( - bootstrapIPs = make([]string, 0, len(n.Nodes)) - bootstrapIDs = make([]string, 0, len(n.Nodes)) + networkID uint32 = 1000 + dirPath string ) - for _, node := range n.Nodes { - if len(node.StakingAddress) == 0 { - // Node is not running + for { + _, reserved := constants.NetworkIDToNetworkName[networkID] + if reserved { + networkID++ continue } - bootstrapIPs = append(bootstrapIPs, node.StakingAddress) - bootstrapIDs = append(bootstrapIDs, node.NodeID.String()) - } + dirPath = filepath.Join(rootDir, strconv.FormatUint(uint64(networkID), 10)) + err := os.Mkdir(dirPath, perms.ReadWriteExecute) + if err == nil { + return networkID, dirPath, nil + } - if len(bootstrapIDs) == 0 { - return nil, nil, errMissingBootstrapNodes - } + if !errors.Is(err, fs.ErrExist) { + return 0, "", fmt.Errorf("failed to create network directory: %w", err) + } - return bootstrapIPs, bootstrapIDs, nil + // Directory already exists, keep iterating + networkID++ + } } diff --git a/tests/fixture/tmpnet/network_config.go b/tests/fixture/tmpnet/network_config.go new file mode 100644 index 000000000000..5d89ddc1716d --- /dev/null +++ b/tests/fixture/tmpnet/network_config.go @@ -0,0 +1,220 @@ +// Copyright (C) 2019-2023, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package tmpnet + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/ava-labs/avalanchego/genesis" + "github.com/ava-labs/avalanchego/utils/crypto/secp256k1" + "github.com/ava-labs/avalanchego/utils/perms" +) + +// The Network type is defined in this file (network_config.go - reading/writing configuration) and +// network.go (network.go - orchestration). + +var errMissingNetworkDir = errors.New("failed to write network: missing network directory") + +// Read network and node configuration from disk. +func (n *Network) Read() error { + if err := n.readNetwork(); err != nil { + return err + } + return n.readNodes() +} + +// Write network configuration to disk. +func (n *Network) Write() error { + if len(n.Dir) == 0 { + return errMissingNetworkDir + } + if err := n.writeGenesis(); err != nil { + return err + } + if err := n.writeChainConfigs(); err != nil { + return err + } + if err := n.writeNetworkConfig(); err != nil { + return err + } + if err := n.writeEnvFile(); err != nil { + return err + } + return n.writeNodes() +} + +// Read network configuration from disk. +func (n *Network) readNetwork() error { + if err := n.readGenesis(); err != nil { + return err + } + if err := n.readChainConfigs(); err != nil { + return err + } + return n.readConfig() +} + +// Read the non-ephemeral nodes associated with the network from disk. +func (n *Network) readNodes() error { + nodes, err := ReadNodes(n.Dir, false /* includeEphemeral */) + if err != nil { + return err + } + n.Nodes = nodes + return nil +} + +func (n *Network) writeNodes() error { + for _, node := range n.Nodes { + if err := node.Write(); err != nil { + return err + } + } + return nil +} + +func (n *Network) getGenesisPath() string { + return filepath.Join(n.Dir, "genesis.json") +} + +func (n *Network) readGenesis() error { + bytes, err := os.ReadFile(n.getGenesisPath()) + if err != nil { + return fmt.Errorf("failed to read genesis: %w", err) + } + genesis := genesis.UnparsedConfig{} + if err := json.Unmarshal(bytes, &genesis); err != nil { + return fmt.Errorf("failed to unmarshal genesis: %w", err) + } + n.Genesis = &genesis + return nil +} + +func (n *Network) writeGenesis() error { + bytes, err := DefaultJSONMarshal(n.Genesis) + if err != nil { + return fmt.Errorf("failed to marshal genesis: %w", err) + } + if err := os.WriteFile(n.getGenesisPath(), bytes, perms.ReadWrite); err != nil { + return fmt.Errorf("failed to write genesis: %w", err) + } + return nil +} + +func (n *Network) getChainConfigDir() string { + return filepath.Join(n.Dir, "chains") +} + +func (n *Network) readChainConfigs() error { + baseChainConfigDir := n.getChainConfigDir() + entries, err := os.ReadDir(baseChainConfigDir) + if err != nil { + return fmt.Errorf("failed to read chain config dir: %w", err) + } + + // Clear the map of data that may end up stale (e.g. if a given + // chain is in the map but no longer exists on disk) + n.ChainConfigs = map[string]FlagsMap{} + + for _, entry := range entries { + if !entry.IsDir() { + // Chain config files are expected to be nested under a + // directory with the name of the chain alias. + continue + } + chainAlias := entry.Name() + configPath := filepath.Join(baseChainConfigDir, chainAlias, defaultConfigFilename) + if _, err := os.Stat(configPath); os.IsNotExist(err) { + // No config file present + continue + } + chainConfig, err := ReadFlagsMap(configPath, fmt.Sprintf("%s chain config", chainAlias)) + if err != nil { + return err + } + n.ChainConfigs[chainAlias] = *chainConfig + } + + return nil +} + +func (n *Network) writeChainConfigs() error { + baseChainConfigDir := n.getChainConfigDir() + + for chainAlias, chainConfig := range n.ChainConfigs { + // Create the directory + chainConfigDir := filepath.Join(baseChainConfigDir, chainAlias) + if err := os.MkdirAll(chainConfigDir, perms.ReadWriteExecute); err != nil { + return fmt.Errorf("failed to create %s chain config dir: %w", chainAlias, err) + } + + // Write the file + path := filepath.Join(chainConfigDir, defaultConfigFilename) + if err := chainConfig.Write(path, fmt.Sprintf("%s chain config", chainAlias)); err != nil { + return err + } + } + + // TODO(marun) Ensure the removal of chain aliases that aren't present in the map + + return nil +} + +func (n *Network) getConfigPath() string { + return filepath.Join(n.Dir, defaultConfigFilename) +} + +func (n *Network) readConfig() error { + bytes, err := os.ReadFile(n.getConfigPath()) + if err != nil { + return fmt.Errorf("failed to read network config: %w", err) + } + if err := json.Unmarshal(bytes, n); err != nil { + return fmt.Errorf("failed to unmarshal network config: %w", err) + } + return nil +} + +// The subset of network fields to store in the network config file. +type serializedNetworkConfig struct { + DefaultFlags FlagsMap + DefaultRuntimeConfig NodeRuntimeConfig + PreFundedKeys []*secp256k1.PrivateKey +} + +func (n *Network) writeNetworkConfig() error { + config := &serializedNetworkConfig{ + DefaultFlags: n.DefaultFlags, + DefaultRuntimeConfig: n.DefaultRuntimeConfig, + PreFundedKeys: n.PreFundedKeys, + } + bytes, err := DefaultJSONMarshal(config) + if err != nil { + return fmt.Errorf("failed to marshal network config: %w", err) + } + if err := os.WriteFile(n.getConfigPath(), bytes, perms.ReadWrite); err != nil { + return fmt.Errorf("failed to write network config: %w", err) + } + return nil +} + +func (n *Network) EnvFilePath() string { + return filepath.Join(n.Dir, "network.env") +} + +func (n *Network) EnvFileContents() string { + return fmt.Sprintf("export %s=%s", NetworkDirEnvName, n.Dir) +} + +// Write an env file that sets the network dir env when sourced. +func (n *Network) writeEnvFile() error { + if err := os.WriteFile(n.EnvFilePath(), []byte(n.EnvFileContents()), perms.ReadWrite); err != nil { + return fmt.Errorf("failed to write network env file: %w", err) + } + return nil +} diff --git a/tests/fixture/tmpnet/network_test.go b/tests/fixture/tmpnet/network_test.go index 1a06a07f4a38..24c9c783992b 100644 --- a/tests/fixture/tmpnet/network_test.go +++ b/tests/fixture/tmpnet/network_test.go @@ -4,6 +4,7 @@ package tmpnet import ( + "bytes" "testing" "github.com/stretchr/testify/require" @@ -14,14 +15,14 @@ func TestNetworkSerialization(t *testing.T) { tmpDir := t.TempDir() - network := &Network{Dir: tmpDir} - require.NoError(network.PopulateNetworkConfig(1337, 1, 1)) - require.NoError(network.WriteAll()) + network, err := NewDefaultNetwork(&bytes.Buffer{}, "/path/to/avalanche/go", 1) + require.NoError(err) + require.NoError(network.Create(tmpDir)) // Ensure node runtime is initialized - require.NoError(network.ReadNodes()) + require.NoError(network.readNodes()) - loadedNetwork, err := ReadNetwork(tmpDir) + loadedNetwork, err := ReadNetwork(network.Dir) require.NoError(err) for _, key := range loadedNetwork.PreFundedKeys { // Address() enables comparison with the original network by diff --git a/tests/fixture/tmpnet/node.go b/tests/fixture/tmpnet/node.go index 955e38dc6995..270cb6cff8ef 100644 --- a/tests/fixture/tmpnet/node.go +++ b/tests/fixture/tmpnet/node.go @@ -89,7 +89,7 @@ func ReadNode(dataDir string) (*Node, error) { } // Reads nodes from the specified network directory. -func ReadNodes(networkDir string) ([]*Node, error) { +func ReadNodes(networkDir string, includeEphemeral bool) ([]*Node, error) { nodes := []*Node{} // Node configuration is stored in child directories @@ -111,6 +111,10 @@ func ReadNodes(networkDir string) ([]*Node, error) { return nil, err } + if !includeEphemeral && node.IsEphemeral { + continue + } + nodes = append(nodes, node) } diff --git a/tests/upgrade/upgrade_test.go b/tests/upgrade/upgrade_test.go index 862b08555b4f..2a9421872b08 100644 --- a/tests/upgrade/upgrade_test.go +++ b/tests/upgrade/upgrade_test.go @@ -6,7 +6,6 @@ package upgrade import ( "flag" "fmt" - "strings" "testing" "github.com/onsi/ginkgo/v2" @@ -15,7 +14,6 @@ import ( "github.com/stretchr/testify/require" - "github.com/ava-labs/avalanchego/config" "github.com/ava-labs/avalanchego/tests/fixture/e2e" ) @@ -55,21 +53,9 @@ var _ = ginkgo.Describe("[Upgrade]", func() { ginkgo.By(fmt.Sprintf("restarting node %q with %q binary", node.NodeID, avalancheGoExecPathToUpgradeTo)) require.NoError(node.Stop(e2e.DefaultContext())) - // A node must start with sufficient bootstrap nodes to represent a quorum. Since the node's current - // bootstrap configuration may not satisfy this requirement (i.e. if on network start the node was one of - // the first validators), updating the node to bootstrap from all running validators maximizes the - // chances of a successful start. - // - // TODO(marun) Refactor node start to do this automatically - bootstrapIPs, bootstrapIDs, err := network.GetBootstrapIPsAndIDs() - require.NoError(err) - require.NotEmpty(bootstrapIDs) - node.Flags[config.BootstrapIDsKey] = strings.Join(bootstrapIDs, ",") - node.Flags[config.BootstrapIPsKey] = strings.Join(bootstrapIPs, ",") - node.RuntimeConfig.AvalancheGoPath = avalancheGoExecPath - require.NoError(node.Write()) - - require.NoError(node.Start(ginkgo.GinkgoWriter)) + node.RuntimeConfig.AvalancheGoPath = avalancheGoExecPathToUpgradeTo + + require.NoError(network.StartNode(e2e.DefaultContext(), ginkgo.GinkgoWriter, node)) ginkgo.By(fmt.Sprintf("waiting for node %q to report healthy after restart", node.NodeID)) e2e.WaitForHealthy(node)