From 26e329a66f2a53e1286a54f9868a214609d99f01 Mon Sep 17 00:00:00 2001 From: marun Date: Wed, 17 Jan 2024 16:49:44 +0100 Subject: [PATCH] `tmpnet`: Add support for subnets (#2492) --- tests/e2e/e2e_test.go | 3 +- tests/fixture/e2e/env.go | 60 ++++- tests/fixture/e2e/flags.go | 19 +- tests/fixture/e2e/helpers.go | 25 +- tests/fixture/tmpnet/README.md | 61 +++-- tests/fixture/tmpnet/cmd/main.go | 50 ++-- tests/fixture/tmpnet/defaults.go | 13 +- tests/fixture/tmpnet/network.go | 314 ++++++++++++++++++++--- tests/fixture/tmpnet/network_config.go | 18 +- tests/fixture/tmpnet/network_test.go | 5 +- tests/fixture/tmpnet/node.go | 9 + tests/fixture/tmpnet/subnet.go | 333 +++++++++++++++++++++++++ tests/upgrade/upgrade_test.go | 4 +- 13 files changed, 825 insertions(+), 89 deletions(-) create mode 100644 tests/fixture/tmpnet/subnet.go diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go index aec783441e80..d363ff775086 100644 --- a/tests/e2e/e2e_test.go +++ b/tests/e2e/e2e_test.go @@ -11,6 +11,7 @@ import ( "github.com/onsi/gomega" "github.com/ava-labs/avalanchego/tests/fixture/e2e" + "github.com/ava-labs/avalanchego/tests/fixture/tmpnet" // ensure test packages are scanned by ginkgo _ "github.com/ava-labs/avalanchego/tests/e2e/banff" @@ -34,7 +35,7 @@ func init() { var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { // Run only once in the first ginkgo process - return e2e.NewTestEnvironment(flagVars).Marshal() + return e2e.NewTestEnvironment(flagVars, &tmpnet.Network{}).Marshal() }, func(envBytes []byte) { // Run in every ginkgo process diff --git a/tests/fixture/e2e/env.go b/tests/fixture/e2e/env.go index d87c0985edd6..9019c9438b9e 100644 --- a/tests/fixture/e2e/env.go +++ b/tests/fixture/e2e/env.go @@ -14,6 +14,8 @@ import ( "github.com/stretchr/testify/require" + "github.com/ava-labs/avalanchego/api/info" + "github.com/ava-labs/avalanchego/config" "github.com/ava-labs/avalanchego/tests" "github.com/ava-labs/avalanchego/tests/fixture" "github.com/ava-labs/avalanchego/tests/fixture/tmpnet" @@ -29,10 +31,9 @@ var Env *TestEnvironment func InitSharedTestEnvironment(envBytes []byte) { require := require.New(ginkgo.GinkgoT()) require.Nil(Env, "env already initialized") - Env = &TestEnvironment{ - require: require, - } + Env = &TestEnvironment{} require.NoError(json.Unmarshal(envBytes, Env)) + Env.require = require } type TestEnvironment struct { @@ -53,7 +54,7 @@ func (te *TestEnvironment) Marshal() []byte { } // Initialize a new test environment with a shared network (either pre-existing or newly created). -func NewTestEnvironment(flagVars *FlagVars) *TestEnvironment { +func NewTestEnvironment(flagVars *FlagVars, desiredNetwork *tmpnet.Network) *TestEnvironment { require := require.New(ginkgo.GinkgoT()) networkDir := flagVars.NetworkDir() @@ -65,10 +66,44 @@ func NewTestEnvironment(flagVars *FlagVars) *TestEnvironment { network, err = tmpnet.ReadNetwork(networkDir) require.NoError(err) tests.Outf("{{yellow}}Using an existing network configured at %s{{/}}\n", network.Dir) + + // Set the desired subnet configuration to ensure subsequent creation. + for _, subnet := range desiredNetwork.Subnets { + if existing := network.GetSubnet(subnet.Name); existing != nil { + // Already present + continue + } + network.Subnets = append(network.Subnets, subnet) + } } else { - network = StartNetwork(flagVars.AvalancheGoExecPath(), DefaultNetworkDir) + network = desiredNetwork + StartNetwork(network, DefaultNetworkDir, flagVars.AvalancheGoExecPath(), flagVars.PluginDir()) } + // A new network will always need subnet creation and an existing + // network will also need subnets to be created the first time it + // is used. + require.NoError(network.CreateSubnets(DefaultContext(), ginkgo.GinkgoWriter)) + + // Wait for chains to have bootstrapped on all nodes + Eventually(func() bool { + for _, subnet := range network.Subnets { + for _, validatorID := range subnet.ValidatorIDs { + uri, err := network.GetURIForNodeID(validatorID) + require.NoError(err) + infoClient := info.NewClient(uri) + for _, chain := range subnet.Chains { + isBootstrapped, err := infoClient.IsBootstrapped(DefaultContext(), chain.ChainID.String()) + // Ignore errors since a chain id that is not yet known will result in a recoverable error. + if err != nil || !isBootstrapped { + return false + } + } + } + } + return true + }, DefaultTimeout, DefaultPollingInterval, "failed to see all chains bootstrap before timeout") + uris := network.GetNodeURIs() require.NotEmpty(uris, "network contains no nodes") tests.Outf("{{green}}network URIs: {{/}} %+v\n", uris) @@ -83,6 +118,7 @@ func NewTestEnvironment(flagVars *FlagVars) *TestEnvironment { NetworkDir: network.Dir, URIs: uris, TestDataServerURI: testDataServerURI, + require: require, } } @@ -127,10 +163,22 @@ func (te *TestEnvironment) NewPrivateNetwork() *tmpnet.Network { sharedNetwork, err := tmpnet.ReadNetwork(te.NetworkDir) te.require.NoError(err) + network := &tmpnet.Network{} + // The private networks dir is under the shared network dir to ensure it // will be included in the artifact uploaded in CI. privateNetworksDir := filepath.Join(sharedNetwork.Dir, PrivateNetworksDirName) te.require.NoError(os.MkdirAll(privateNetworksDir, perms.ReadWriteExecute)) - return StartNetwork(sharedNetwork.DefaultRuntimeConfig.AvalancheGoPath, privateNetworksDir) + pluginDir, err := sharedNetwork.DefaultFlags.GetStringVal(config.PluginDirKey) + te.require.NoError(err) + + StartNetwork( + network, + privateNetworksDir, + sharedNetwork.DefaultRuntimeConfig.AvalancheGoPath, + pluginDir, + ) + + return network } diff --git a/tests/fixture/e2e/flags.go b/tests/fixture/e2e/flags.go index 6cedf003b3a3..2a00df97a885 100644 --- a/tests/fixture/e2e/flags.go +++ b/tests/fixture/e2e/flags.go @@ -13,10 +13,19 @@ import ( type FlagVars struct { avalancheGoExecPath string + pluginDir string networkDir string useExistingNetwork bool } +func (v *FlagVars) AvalancheGoExecPath() string { + return v.avalancheGoExecPath +} + +func (v *FlagVars) PluginDir() string { + return v.pluginDir +} + func (v *FlagVars) NetworkDir() string { if !v.useExistingNetwork { return "" @@ -27,10 +36,6 @@ func (v *FlagVars) NetworkDir() string { return os.Getenv(tmpnet.NetworkDirEnvName) } -func (v *FlagVars) AvalancheGoExecPath() string { - return v.avalancheGoExecPath -} - func (v *FlagVars) UseExistingNetwork() bool { return v.useExistingNetwork } @@ -43,6 +48,12 @@ func RegisterFlags() *FlagVars { os.Getenv(tmpnet.AvalancheGoPathEnvName), fmt.Sprintf("avalanchego executable path (required if not using an existing network). Also possible to configure via the %s env variable.", tmpnet.AvalancheGoPathEnvName), ) + flag.StringVar( + &vars.pluginDir, + "plugin-dir", + os.ExpandEnv("$HOME/.avalanchego/plugins"), + "[optional] the dir containing VM plugins.", + ) flag.StringVar( &vars.networkDir, "network-dir", diff --git a/tests/fixture/e2e/helpers.go b/tests/fixture/e2e/helpers.go index 706af72dde05..c1d87a4beba8 100644 --- a/tests/fixture/e2e/helpers.go +++ b/tests/fixture/e2e/helpers.go @@ -34,15 +34,15 @@ const ( // contention. DefaultTimeout = 2 * time.Minute - // Interval appropriate for network operations that should be - // retried periodically but not too often. - DefaultPollingInterval = 500 * time.Millisecond + DefaultPollingInterval = tmpnet.DefaultPollingInterval // Setting this env will disable post-test bootstrap // checks. Useful for speeding up iteration during test // development. SkipBootstrapChecksEnvName = "E2E_SKIP_BOOTSTRAP_CHECKS" + DefaultValidatorStartTimeDiff = tmpnet.DefaultValidatorStartTimeDiff + DefaultGasLimit = uint64(21000) // Standard gas limit // An empty string prompts the use of the default path which ensures a @@ -217,13 +217,20 @@ func CheckBootstrapIsPossible(network *tmpnet.Network) { } // Start a temporary network with the provided avalanchego binary. -func StartNetwork(avalancheGoExecPath string, rootNetworkDir string) *tmpnet.Network { +func StartNetwork(network *tmpnet.Network, rootNetworkDir string, avalancheGoExecPath string, pluginDir string) { require := require.New(ginkgo.GinkgoT()) - network, err := tmpnet.NewDefaultNetwork(ginkgo.GinkgoWriter, avalancheGoExecPath, tmpnet.DefaultNodeCount) - require.NoError(err) - require.NoError(network.Create(rootNetworkDir)) - require.NoError(network.Start(DefaultContext(), ginkgo.GinkgoWriter)) + require.NoError( + tmpnet.StartNewNetwork( + DefaultContext(), + ginkgo.GinkgoWriter, + network, + rootNetworkDir, + avalancheGoExecPath, + pluginDir, + tmpnet.DefaultNodeCount, + ), + ) ginkgo.DeferCleanup(func() { tests.Outf("Shutting down network\n") @@ -233,6 +240,4 @@ func StartNetwork(avalancheGoExecPath string, rootNetworkDir string) *tmpnet.Net }) tests.Outf("{{green}}Successfully started network{{/}}\n") - - return network } diff --git a/tests/fixture/tmpnet/README.md b/tests/fixture/tmpnet/README.md index abccbf52cf79..909a29c6ee12 100644 --- a/tests/fixture/tmpnet/README.md +++ b/tests/fixture/tmpnet/README.md @@ -34,6 +34,7 @@ the following non-test files: | node.go | Node | Orchestrates and configures nodes | | node_config.go | Node | Reads and writes node configuration | | node_process.go | NodeProcess | Orchestrates node processes | +| subnet.go | Subnet | Orchestrates subnets | | utils.go | | Defines shared utility functions | ## Usage @@ -74,16 +75,33 @@ network. A temporary network can be managed in code: ```golang -network, _ := tmpnet.NewDefaultNetwork( +network := &tmpnet.Network{ // Configure non-default values for the new network + DefaultFlags: tmpnet.FlagsMap{ + config.LogLevelKey: "INFO", // Change one of the network's defaults + }, + Subnets: []*tmpnet.Subnet{ // Subnets to create on the new network once it is running + { + Name: "xsvm-a", // User-defined name used to reference subnet in code and on disk + Chains: []*tmpnet.Chain{ + { + VMName: "xsvm", // Name of the VM the chain will run, will be used to derive the name of the VM binary + Genesis: , // Genesis bytes used to initialize the custom chain + PreFundedKey: , // (Optional) A private key that is funded in the genesis bytes + }, + }, + }, + }, +} + +_ := tmpnet.StartNewNetwork( // Start the network + ctx, // Context used to limit duration of waiting for network health ginkgo.GinkgoWriter, // Writer to report progress of initialization + network, + "", // Empty string uses the default network path (~/tmpnet/networks) "/path/to/avalanchego", // The path to the binary that nodes will execute + "/path/to/plugins", // The path nodes will use for plugin binaries (suggested value ~/.avalanchego/plugins) 5, // Number of initial validating nodes ) -_ = network.Create("") // Finalize network configuration and write to disk -_ = network.Start( // Start the nodes of the network and wait until they report healthy - ctx, // Context used to limit duration of waiting for network health - ginkgo.GinkgoWriter, // Writer to report progress of network start -) uris := network.GetNodeURIs() @@ -125,11 +143,16 @@ HOME │ │ └── ... │ └── process.json // Node process details (PID, API URI, staking address) ├── chains - │ └── C - │ └── config.json // C-Chain config for all nodes + │ ├── C + │ │ └── config.json // C-Chain config for all nodes + │ └── raZ51bwfepaSaZ1MNSRNYNs3ZPfj...U7pa3 + │ └── config.json // Custom chain configuration for all nodes ├── config.json // Common configuration (including defaults and pre-funded keys) ├── genesis.json // Genesis for all nodes - └── network.env // Sets network dir env var to simplify network usage + ├── network.env // Sets network dir env var to simplify network usage + └── subnets // Parent directory for subnet definitions + ├─ subnet-a.json // Configuration for subnet-a and its chain(s) + └─ subnet-b.json // Configuration for subnet-b and its chain(s) ``` ### Common networking configuration @@ -148,17 +171,19 @@ content will be generated with reasonable defaults if not supplied. Each node in the network can override the default by setting an explicit value for `--genesis-file` or `--genesis-file-content`. -### C-Chain config +### Chain configuration -The C-Chain config for a temporary network is stored at -`[network-dir]/chains/C/config.json` and referenced by default by all -nodes in the network. The C-Chain config will be generated with -reasonable defaults if not supplied. Each node in the network can -override the default by setting an explicit value for -`--chain-config-dir` and ensuring the C-Chain config file exists at -`[chain-config-dir]/C/config.json`. +The chain configuration for a temporary network is stored at +`[network-dir]/chains/[chain alias or ID]/config.json` and referenced +by all nodes in the network. The C-Chain config will be generated with +reasonable defaults if not supplied. X-Chain and P-Chain will use +implicit defaults. The configuration for custom chains can be provided +with subnet configuration and will be writen to the appropriate path. -TODO(marun) Enable configuration of X-Chain and P-Chain. +Each node in the network can override network-level chain +configuration by setting `--chain-config-dir` to an explicit value and +ensuring that configuration files for all chains exist at +`[custom-chain-config-dir]/[chain alias or ID]/config.json`. ### Network env diff --git a/tests/fixture/tmpnet/cmd/main.go b/tests/fixture/tmpnet/cmd/main.go index 1ab3f2648325..dd59c300bbb3 100644 --- a/tests/fixture/tmpnet/cmd/main.go +++ b/tests/fixture/tmpnet/cmd/main.go @@ -26,10 +26,12 @@ var ( ) func main() { + var networkDir string rootCmd := &cobra.Command{ Use: "tmpnetctl", Short: "tmpnetctl commands", } + rootCmd.PersistentFlags().StringVar(&networkDir, "network-dir", os.Getenv(tmpnet.NetworkDirEnvName), "The path to the configuration directory of a temporary network") versionCmd := &cobra.Command{ Use: "version", @@ -46,35 +48,38 @@ func main() { rootCmd.AddCommand(versionCmd) var ( - rootDir string - execPath string - nodeCount uint8 + rootDir string + avalancheGoPath string + pluginDir string + nodeCount uint8 ) startNetworkCmd := &cobra.Command{ Use: "start-network", Short: "Start a new temporary network", RunE: func(*cobra.Command, []string) error { - if len(execPath) == 0 { + if len(avalancheGoPath) == 0 { return errAvalancheGoRequired } // Root dir will be defaulted on start if not provided - network, err := tmpnet.NewDefaultNetwork(os.Stdout, execPath, int(nodeCount)) - if err != nil { - return err - } - - if err := network.Create(rootDir); err != nil { - return err - } + network := &tmpnet.Network{} // Extreme upper bound, should never take this long networkStartTimeout := 2 * time.Minute ctx, cancel := context.WithTimeout(context.Background(), networkStartTimeout) defer cancel() - if err := network.Start(ctx, os.Stdout); err != nil { + err := tmpnet.StartNewNetwork( + ctx, + os.Stdout, + network, + rootDir, + avalancheGoPath, + pluginDir, + int(nodeCount), + ) + if err != nil { return err } @@ -98,11 +103,11 @@ func main() { }, } startNetworkCmd.PersistentFlags().StringVar(&rootDir, "root-dir", os.Getenv(tmpnet.RootDirEnvName), "The path to the root directory for temporary networks") - startNetworkCmd.PersistentFlags().StringVar(&execPath, "avalanchego-path", os.Getenv(tmpnet.AvalancheGoPathEnvName), "The path to an avalanchego binary") + startNetworkCmd.PersistentFlags().StringVar(&avalancheGoPath, "avalanchego-path", os.Getenv(tmpnet.AvalancheGoPathEnvName), "The path to an avalanchego binary") + startNetworkCmd.PersistentFlags().StringVar(&pluginDir, "plugin-dir", os.ExpandEnv("$HOME/.avalanchego/plugins"), "[optional] the dir containing VM plugins") startNetworkCmd.PersistentFlags().Uint8Var(&nodeCount, "node-count", tmpnet.DefaultNodeCount, "Number of nodes the network should initially consist of") rootCmd.AddCommand(startNetworkCmd) - var networkDir string stopNetworkCmd := &cobra.Command{ Use: "stop-network", Short: "Stop a temporary network", @@ -119,9 +124,22 @@ func main() { return nil }, } - stopNetworkCmd.PersistentFlags().StringVar(&networkDir, "network-dir", os.Getenv(tmpnet.NetworkDirEnvName), "The path to the configuration directory of a temporary network") rootCmd.AddCommand(stopNetworkCmd) + restartNetworkCmd := &cobra.Command{ + Use: "restart-network", + Short: "Restart a temporary network", + RunE: func(*cobra.Command, []string) error { + if len(networkDir) == 0 { + return errNetworkDirRequired + } + ctx, cancel := context.WithTimeout(context.Background(), tmpnet.DefaultNetworkTimeout) + defer cancel() + return tmpnet.RestartNetwork(ctx, os.Stdout, networkDir) + }, + } + rootCmd.AddCommand(restartNetworkCmd) + if err := rootCmd.Execute(); err != nil { fmt.Fprintf(os.Stderr, "tmpnetctl failed: %v\n", err) os.Exit(1) diff --git a/tests/fixture/tmpnet/defaults.go b/tests/fixture/tmpnet/defaults.go index ce09def2582a..2b88ef49afc1 100644 --- a/tests/fixture/tmpnet/defaults.go +++ b/tests/fixture/tmpnet/defaults.go @@ -7,9 +7,19 @@ import ( "time" "github.com/ava-labs/avalanchego/config" + "github.com/ava-labs/avalanchego/vms/platformvm/txs/executor" ) const ( + // Interval appropriate for network operations that should be + // retried periodically but not too often. + DefaultPollingInterval = 500 * time.Millisecond + + // Validator start time must be a minimum of SyncBound from the + // current time for validator addition to succeed, and adding 20 + // seconds provides a buffer in case of any delay in processing. + DefaultValidatorStartTimeDiff = executor.SyncBound + 20*time.Second + DefaultNetworkTimeout = 2 * time.Minute // Minimum required to ensure connectivity-based health checks will pass @@ -50,7 +60,8 @@ func DefaultChainConfigs() map[string]FlagsMap { // values will be used. Available C-Chain configuration options are // defined in the `github.com/ava-labs/coreth/evm` package. "C": { - "log-level": "trace", + "warp-api-enabled": true, + "log-level": "trace", }, } } diff --git a/tests/fixture/tmpnet/network.go b/tests/fixture/tmpnet/network.go index 3352f7dcb1cf..01829da70da5 100644 --- a/tests/fixture/tmpnet/network.go +++ b/tests/fixture/tmpnet/network.go @@ -5,6 +5,7 @@ package tmpnet import ( "context" + "encoding/hex" "errors" "fmt" "io" @@ -12,6 +13,7 @@ import ( "os" "path/filepath" "strconv" + "strings" "time" "github.com/ava-labs/avalanchego/config" @@ -21,6 +23,7 @@ import ( "github.com/ava-labs/avalanchego/utils/crypto/secp256k1" "github.com/ava-labs/avalanchego/utils/perms" "github.com/ava-labs/avalanchego/utils/set" + "github.com/ava-labs/avalanchego/vms/platformvm" ) // The Network type is defined in this file (orchestration) and @@ -36,8 +39,26 @@ const ( // startup, as smaller intervals (e.g. 50ms) seemed to noticeably // increase the time for a network's nodes to be seen as healthy. networkHealthCheckInterval = 200 * time.Millisecond + + // eth address: 0x8db97C7cEcE249c2b98bDC0226Cc4C2A57BF52FC + HardHatKeyStr = "56289e99c94b6912bfc12adc093c9b51124f0dc54ac7a766b2bc5ccf558d8027" ) +// HardhatKey is a legacy used for hardhat testing in subnet-evm +// TODO(marun) Remove when no longer needed. +var HardhatKey *secp256k1.PrivateKey + +func init() { + hardhatKeyBytes, err := hex.DecodeString(HardHatKeyStr) + if err != nil { + panic(err) + } + HardhatKey, err = secp256k1.ToPrivateKey(hardhatKeyBytes) + if err != nil { + panic(err) + } +} + // Collects the configuration for running a temporary avalanchego network type Network struct { // Path where network configuration and data is stored @@ -56,6 +77,9 @@ type Network struct { // Nodes that constitute the network Nodes []*Node + + // Subnets that have been enabled on the network + Subnets []*Subnet } // Ensure a real and absolute network dir so that node @@ -69,35 +93,22 @@ func toCanonicalDir(dir string) (string, error) { return filepath.EvalSymlinks(absDir) } -// Initializes a new network with default configuration. -func NewDefaultNetwork(w io.Writer, avalancheGoPath string, nodeCount int) (*Network, error) { - if _, err := fmt.Fprintf(w, "Preparing configuration for new network with %s\n", avalancheGoPath); err != nil { - return nil, err - } - - keys, err := NewPrivateKeys(DefaultPreFundedKeyCount) - if err != nil { - return nil, err - } - - network := &Network{ - DefaultFlags: DefaultFlags(), - DefaultRuntimeConfig: NodeRuntimeConfig{ - AvalancheGoPath: avalancheGoPath, - }, - PreFundedKeys: keys, - ChainConfigs: DefaultChainConfigs(), +func StartNewNetwork( + ctx context.Context, + w io.Writer, + network *Network, + rootNetworkDir string, + avalancheGoExecPath string, + pluginDir string, + nodeCount int, +) error { + if err := network.EnsureDefaultConfig(w, avalancheGoExecPath, pluginDir, nodeCount); err != nil { + return err } - - network.Nodes = make([]*Node, nodeCount) - for i := range network.Nodes { - network.Nodes[i] = NewNode("") - if err := network.EnsureNodeConfig(network.Nodes[i]); err != nil { - return nil, err - } + if err := network.Create(rootNetworkDir); err != nil { + return err } - - return network, nil + return network.Start(ctx, w) } // Stops the nodes of the network configured in the provided directory. @@ -109,6 +120,15 @@ func StopNetwork(ctx context.Context, dir string) error { return network.Stop(ctx) } +// Restarts the nodes of the network configured in the provided directory. +func RestartNetwork(ctx context.Context, w io.Writer, dir string) error { + network, err := ReadNetwork(dir) + if err != nil { + return err + } + return network.Restart(ctx, w) +} + // Reads a network from the provided directory. func ReadNetwork(dir string) (*Network, error) { canonicalDir, err := toCanonicalDir(dir) @@ -124,6 +144,68 @@ func ReadNetwork(dir string) (*Network, error) { return network, nil } +// Initializes a new network with default configuration. +func (n *Network) EnsureDefaultConfig(w io.Writer, avalancheGoPath string, pluginDir string, nodeCount int) error { + if _, err := fmt.Fprintf(w, "Preparing configuration for new network with %s\n", avalancheGoPath); err != nil { + return err + } + + // Ensure default flags + if n.DefaultFlags == nil { + n.DefaultFlags = FlagsMap{} + } + n.DefaultFlags.SetDefaults(DefaultFlags()) + + // Only configure the plugin dir with a non-empty value to ensure + // the use of the default value (`[datadir]/plugins`) when + // no plugin dir is configured. + if len(pluginDir) > 0 { + if _, ok := n.DefaultFlags[config.PluginDirKey]; !ok { + n.DefaultFlags[config.PluginDirKey] = pluginDir + } + } + + // Ensure pre-funded keys + if len(n.PreFundedKeys) == 0 { + keys, err := NewPrivateKeys(DefaultPreFundedKeyCount) + if err != nil { + return err + } + n.PreFundedKeys = keys + } + + // Ensure primary chains are configured + if n.ChainConfigs == nil { + n.ChainConfigs = map[string]FlagsMap{} + } + defaultChainConfigs := DefaultChainConfigs() + for alias, chainConfig := range defaultChainConfigs { + if _, ok := n.ChainConfigs[alias]; !ok { + n.ChainConfigs[alias] = FlagsMap{} + } + n.ChainConfigs[alias].SetDefaults(chainConfig) + } + + // Ensure runtime is configured + if len(n.DefaultRuntimeConfig.AvalancheGoPath) == 0 { + n.DefaultRuntimeConfig.AvalancheGoPath = avalancheGoPath + } + + // Ensure nodes are created + if len(n.Nodes) == 0 { + n.Nodes = NewNodes(nodeCount) + } + + // Ensure nodes are configured + for i := range n.Nodes { + if err := n.EnsureNodeConfig(n.Nodes[i]); err != nil { + return err + } + } + + return nil +} + // Creates the network on disk, choosing its network id and generating its genesis in the process. func (n *Network) Create(rootDir string) error { if len(rootDir) == 0 { @@ -170,8 +252,30 @@ func (n *Network) Create(rootDir string) error { } n.Dir = canonicalDir + pluginDir, err := n.DefaultFlags.GetStringVal(config.PluginDirKey) + if err != nil { + return err + } + if len(pluginDir) > 0 { + // Ensure the existence of the plugin directory or nodes won't be able to start. + if err := os.MkdirAll(pluginDir, perms.ReadWriteExecute); err != nil { + return fmt.Errorf("failed to create plugin dir: %w", err) + } + } + if n.Genesis == nil { - genesis, err := NewTestGenesis(networkID, n.Nodes, n.PreFundedKeys) + // Pre-fund known legacy keys to support ad-hoc testing. Usage of a legacy key will + // require knowing the key beforehand rather than retrieving it from the set of pre-funded + // keys exposed by a network. Since allocation will not be exclusive, a test using a + // legacy key is unlikely to be a good candidate for parallel execution. + keysToFund := []*secp256k1.PrivateKey{ + genesis.VMRQKey, + genesis.EWOQKey, + HardhatKey, + } + keysToFund = append(keysToFund, n.PreFundedKeys...) + + genesis, err := NewTestGenesis(networkID, n.Nodes, keysToFund) if err != nil { return err } @@ -317,6 +421,28 @@ func (n *Network) Stop(ctx context.Context) error { return nil } +// Restarts all non-ephemeral nodes in the network. +func (n *Network) Restart(ctx context.Context, w io.Writer) error { + if _, err := fmt.Fprintf(w, " restarting network\n"); err != nil { + return err + } + for _, node := range n.Nodes { + if err := node.Stop(ctx); err != nil { + return fmt.Errorf("failed to stop node %s: %w", node.NodeID, err) + } + if err := n.StartNode(ctx, w, node); err != nil { + return fmt.Errorf("failed to start node %s: %w", node.NodeID, err) + } + if _, err := fmt.Fprintf(w, " waiting for node %s to report healthy\n", node.NodeID); err != nil { + return err + } + if err := WaitForHealthy(ctx, node); err != nil { + return err + } + } + return nil +} + // Ensures the provided node has the configuration it needs to start. If the data dir is not // set, it will be defaulted to [nodeParentDir]/[node ID]. For a not-yet-created network, // no action will be taken. @@ -359,9 +485,141 @@ func (n *Network) EnsureNodeConfig(node *Node) error { } } + // Ensure available subnets are tracked + subnetIDs := make([]string, 0, len(n.Subnets)) + for _, subnet := range n.Subnets { + if subnet.SubnetID == ids.Empty { + continue + } + subnetIDs = append(subnetIDs, subnet.SubnetID.String()) + } + flags[config.TrackSubnetsKey] = strings.Join(subnetIDs, ",") + + return nil +} + +func (n *Network) GetSubnet(name string) *Subnet { + for _, subnet := range n.Subnets { + if subnet.Name == name { + return subnet + } + } + return nil +} + +// Ensure that each subnet on the network is created and that it is validated by all non-ephemeral nodes. +func (n *Network) CreateSubnets(ctx context.Context, w io.Writer) error { + createdSubnets := make([]*Subnet, 0, len(n.Subnets)) + for _, subnet := range n.Subnets { + if _, err := fmt.Fprintf(w, "Creating subnet %q\n", subnet.Name); err != nil { + return err + } + if subnet.SubnetID != ids.Empty { + // The subnet already exists + continue + } + + if subnet.OwningKey == nil { + // Allocate a pre-funded key and remove it from the network so it won't be used for + // other purposes + if len(n.PreFundedKeys) == 0 { + return fmt.Errorf("no pre-funded keys available to create subnet %q", subnet.Name) + } + subnet.OwningKey = n.PreFundedKeys[len(n.PreFundedKeys)-1] + n.PreFundedKeys = n.PreFundedKeys[:len(n.PreFundedKeys)-1] + } + + // Create the subnet on the network + if err := subnet.Create(ctx, n.Nodes[0].URI); err != nil { + return err + } + + if _, err := fmt.Fprintf(w, " created subnet %q as %q\n", subnet.Name, subnet.SubnetID); err != nil { + return err + } + + // Persist the subnet configuration + if err := subnet.Write(n.getSubnetDir(), n.getChainConfigDir()); err != nil { + return err + } + + if _, err := fmt.Fprintf(w, " wrote configuration for subnet %q\n", subnet.Name); err != nil { + return err + } + + createdSubnets = append(createdSubnets, subnet) + } + + if len(createdSubnets) == 0 { + return nil + } + + // Ensure the in-memory subnet state + n.Subnets = append(n.Subnets, createdSubnets...) + + // Ensure the pre-funded key changes are persisted to disk + if err := n.Write(); err != nil { + return err + } + + // Reconfigure nodes for the new subnets and their chains + if _, err := fmt.Fprintf(w, "Configured nodes to track new subnet(s). Restart is required.\n"); err != nil { + return err + } + for _, node := range n.Nodes { + if err := n.EnsureNodeConfig(node); err != nil { + return err + } + } + + // Restart nodes to allow new configuration to take effect + if err := n.Restart(ctx, w); err != nil { + return err + } + + // Add each node as a subnet validator + for _, subnet := range createdSubnets { + if _, err := fmt.Fprintf(w, "Adding validators for subnet %q\n", subnet.Name); err != nil { + return err + } + if err := subnet.AddValidators(ctx, w, n.Nodes); err != nil { + return err + } + } + + // Wait for nodes to become subnet validators + pChainClient := platformvm.NewClient(n.Nodes[0].URI) + for _, subnet := range createdSubnets { + if err := waitForActiveValidators(ctx, w, pChainClient, subnet); err != nil { + return err + } + + // It should now be safe to create chains for the subnet + if err := subnet.CreateChains(ctx, w, n.Nodes[0].URI); err != nil { + return err + } + + // Persist the chain configuration + if err := subnet.Write(n.getSubnetDir(), n.getChainConfigDir()); err != nil { + return err + } + if _, err := fmt.Fprintf(w, " wrote chain configuration for subnet %q\n", subnet.Name); err != nil { + return err + } + } + return nil } +func (n *Network) GetURIForNodeID(nodeID ids.NodeID) (string, error) { + for _, node := range n.Nodes { + if node.NodeID == nodeID { + return node.URI, nil + } + } + return "", fmt.Errorf("%s is not known to the network", nodeID) +} + func (n *Network) GetNodeURIs() []NodeURI { return GetNodeURIs(n.Nodes) } diff --git a/tests/fixture/tmpnet/network_config.go b/tests/fixture/tmpnet/network_config.go index 967a2b1b4ee3..4c68af240e98 100644 --- a/tests/fixture/tmpnet/network_config.go +++ b/tests/fixture/tmpnet/network_config.go @@ -25,7 +25,10 @@ func (n *Network) Read() error { if err := n.readNetwork(); err != nil { return err } - return n.readNodes() + if err := n.readNodes(); err != nil { + return err + } + return n.readSubnets() } // Write network configuration to disk. @@ -218,3 +221,16 @@ func (n *Network) writeEnvFile() error { } return nil } + +func (n *Network) getSubnetDir() string { + return filepath.Join(n.Dir, defaultSubnetDirName) +} + +func (n *Network) readSubnets() error { + subnets, err := readSubnets(n.getSubnetDir()) + if err != nil { + return err + } + n.Subnets = subnets + return nil +} diff --git a/tests/fixture/tmpnet/network_test.go b/tests/fixture/tmpnet/network_test.go index 3cbbb8ffcae8..c04c497c2485 100644 --- a/tests/fixture/tmpnet/network_test.go +++ b/tests/fixture/tmpnet/network_test.go @@ -15,10 +15,9 @@ func TestNetworkSerialization(t *testing.T) { tmpDir := t.TempDir() - network, err := NewDefaultNetwork(&bytes.Buffer{}, "/path/to/avalanche/go", 1) - require.NoError(err) + network := &Network{} + require.NoError(network.EnsureDefaultConfig(&bytes.Buffer{}, "/path/to/avalanche/go", "", 1)) require.NoError(network.Create(tmpDir)) - // Ensure node runtime is initialized require.NoError(network.readNodes()) diff --git a/tests/fixture/tmpnet/node.go b/tests/fixture/tmpnet/node.go index d1f97c7ead9c..59025b649112 100644 --- a/tests/fixture/tmpnet/node.go +++ b/tests/fixture/tmpnet/node.go @@ -83,6 +83,15 @@ func NewNode(dataDir string) *Node { } } +// Initializes the specified number of nodes. +func NewNodes(count int) []*Node { + nodes := make([]*Node, count) + for i := range nodes { + nodes[i] = NewNode("") + } + return nodes +} + // Reads a node's configuration from the specified directory. func ReadNode(dataDir string) (*Node, error) { node := NewNode(dataDir) diff --git a/tests/fixture/tmpnet/subnet.go b/tests/fixture/tmpnet/subnet.go new file mode 100644 index 000000000000..0eb1feab5f38 --- /dev/null +++ b/tests/fixture/tmpnet/subnet.go @@ -0,0 +1,333 @@ +// Copyright (C) 2019-2023, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package tmpnet + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/ava-labs/avalanchego/ids" + "github.com/ava-labs/avalanchego/utils/constants" + "github.com/ava-labs/avalanchego/utils/crypto/secp256k1" + "github.com/ava-labs/avalanchego/utils/perms" + "github.com/ava-labs/avalanchego/utils/set" + "github.com/ava-labs/avalanchego/utils/units" + "github.com/ava-labs/avalanchego/vms/platformvm" + "github.com/ava-labs/avalanchego/vms/platformvm/txs" + "github.com/ava-labs/avalanchego/vms/secp256k1fx" + "github.com/ava-labs/avalanchego/wallet/subnet/primary" + "github.com/ava-labs/avalanchego/wallet/subnet/primary/common" +) + +const defaultSubnetDirName = "subnets" + +type Chain struct { + // Set statically + VMID ids.ID + Config string + Genesis []byte + + // Set at runtime + ChainID ids.ID + PreFundedKey *secp256k1.PrivateKey +} + +// Write the chain configuration to the specified directory. +func (c *Chain) WriteConfig(chainDir string) error { + if len(c.Config) == 0 { + return nil + } + + chainConfigDir := filepath.Join(chainDir, c.ChainID.String()) + if err := os.MkdirAll(chainConfigDir, perms.ReadWriteExecute); err != nil { + return fmt.Errorf("failed to create chain config dir: %w", err) + } + + path := filepath.Join(chainConfigDir, defaultConfigFilename) + if err := os.WriteFile(path, []byte(c.Config), perms.ReadWrite); err != nil { + return fmt.Errorf("failed to write chain config: %w", err) + } + + return nil +} + +type Subnet struct { + // A unique string that can be used to refer to the subnet across different temporary + // networks (since the SubnetID will be different every time the subnet is created) + Name string + + // The ID of the transaction that created the subnet + SubnetID ids.ID + + // The private key that owns the subnet + OwningKey *secp256k1.PrivateKey + + // IDs of the nodes responsible for validating the subnet + ValidatorIDs []ids.NodeID + + Chains []*Chain +} + +// Retrieves a wallet configured for use with the subnet +func (s *Subnet) GetWallet(ctx context.Context, uri string) (primary.Wallet, error) { + keychain := secp256k1fx.NewKeychain(s.OwningKey) + + // Only fetch the subnet transaction if a subnet ID is present. This won't be true when + // the wallet is first used to create the subnet. + txIDs := set.Set[ids.ID]{} + if s.SubnetID != ids.Empty { + txIDs.Add(s.SubnetID) + } + + return primary.MakeWallet(ctx, &primary.WalletConfig{ + URI: uri, + AVAXKeychain: keychain, + EthKeychain: keychain, + PChainTxsToFetch: txIDs, + }) +} + +// Issues the subnet creation transaction and retains the result. The URI of a node is +// required to issue the transaction. +func (s *Subnet) Create(ctx context.Context, uri string) error { + wallet, err := s.GetWallet(ctx, uri) + if err != nil { + return err + } + pWallet := wallet.P() + + subnetTx, err := pWallet.IssueCreateSubnetTx( + &secp256k1fx.OutputOwners{ + Threshold: 1, + Addrs: []ids.ShortID{ + s.OwningKey.Address(), + }, + }, + common.WithContext(ctx), + ) + if err != nil { + return fmt.Errorf("failed to create subnet %s: %w", s.Name, err) + } + s.SubnetID = subnetTx.ID() + + return nil +} + +func (s *Subnet) CreateChains(ctx context.Context, w io.Writer, uri string) error { + wallet, err := s.GetWallet(ctx, uri) + if err != nil { + return err + } + pWallet := wallet.P() + + if _, err := fmt.Fprintf(w, "Creating chains for subnet %q\n", s.Name); err != nil { + return err + } + + for _, chain := range s.Chains { + createChainTx, err := pWallet.IssueCreateChainTx( + s.SubnetID, + chain.Genesis, + chain.VMID, + nil, + "", + common.WithContext(ctx), + ) + if err != nil { + return fmt.Errorf("failed to create chain: %w", err) + } + chain.ChainID = createChainTx.ID() + + if _, err := fmt.Fprintf(w, " created chain %q for VM %q on subnet %q\n", chain.ChainID, chain.VMID, s.Name); err != nil { + return err + } + } + return nil +} + +// Add validators to the subnet +func (s *Subnet) AddValidators(ctx context.Context, w io.Writer, nodes []*Node) error { + apiURI := nodes[0].URI + + wallet, err := s.GetWallet(ctx, apiURI) + if err != nil { + return err + } + pWallet := wallet.P() + + // Collect the end times for current validators to reuse for subnet validators + pvmClient := platformvm.NewClient(apiURI) + validators, err := pvmClient.GetCurrentValidators(ctx, constants.PrimaryNetworkID, nil) + if err != nil { + return err + } + endTimes := make(map[ids.NodeID]uint64) + for _, validator := range validators { + endTimes[validator.NodeID] = validator.EndTime + } + + startTime := time.Now().Add(DefaultValidatorStartTimeDiff) + for _, node := range nodes { + endTime, ok := endTimes[node.NodeID] + if !ok { + return fmt.Errorf("failed to find end time for %s", node.NodeID) + } + + _, err := pWallet.IssueAddSubnetValidatorTx( + &txs.SubnetValidator{ + Validator: txs.Validator{ + NodeID: node.NodeID, + Start: uint64(startTime.Unix()), + End: endTime, + Wght: units.Schmeckle, + }, + Subnet: s.SubnetID, + }, + common.WithContext(ctx), + ) + if err != nil { + return err + } + + if _, err := fmt.Fprintf(w, " added %s as validator for subnet `%s`\n", node.NodeID, s.Name); err != nil { + return err + } + + s.ValidatorIDs = append(s.ValidatorIDs, node.NodeID) + } + + return nil +} + +// Write the subnet configuration to disk +func (s *Subnet) Write(subnetDir string, chainDir string) error { + if err := os.MkdirAll(subnetDir, perms.ReadWriteExecute); err != nil { + return fmt.Errorf("failed to create subnet dir: %w", err) + } + path := filepath.Join(subnetDir, s.Name+".json") + + // Since subnets are expected to be serialized for the first time + // without their chains having been created (i.e. chains will have + // empty IDs), use the absence of chain IDs as a prompt for a + // subnet name uniquness check. + if len(s.Chains) > 0 && s.Chains[0].ChainID == ids.Empty { + _, err := os.Stat(path) + if err != nil && !os.IsNotExist(err) { + return err + } + if err == nil { + return fmt.Errorf("a subnet with name %s already exists", s.Name) + } + } + + bytes, err := DefaultJSONMarshal(s) + if err != nil { + return fmt.Errorf("failed to marshal subnet %s: %w", s.Name, err) + } + if err := os.WriteFile(path, bytes, perms.ReadWrite); err != nil { + return fmt.Errorf("failed to write subnet %s: %w", s.Name, err) + } + + for _, chain := range s.Chains { + if err := chain.WriteConfig(chainDir); err != nil { + return err + } + } + + return nil +} + +func waitForActiveValidators( + ctx context.Context, + w io.Writer, + pChainClient platformvm.Client, + subnet *Subnet, +) error { + ticker := time.NewTicker(DefaultPollingInterval) + defer ticker.Stop() + + if _, err := fmt.Fprintf(w, "Waiting for validators of subnet %q to become active\n", subnet.Name); err != nil { + return err + } + + if _, err := fmt.Fprintf(w, " "); err != nil { + return err + } + + for { + if _, err := fmt.Fprintf(w, "."); err != nil { + return err + } + validators, err := pChainClient.GetCurrentValidators(ctx, subnet.SubnetID, nil) + if err != nil { + return err + } + validatorSet := set.NewSet[ids.NodeID](len(validators)) + for _, validator := range validators { + validatorSet.Add(validator.NodeID) + } + allActive := true + for _, validatorID := range subnet.ValidatorIDs { + if !validatorSet.Contains(validatorID) { + allActive = false + } + } + if allActive { + if _, err := fmt.Fprintf(w, "\n saw the expected active validators of subnet %q\n", subnet.Name); err != nil { + return err + } + return nil + } + + select { + case <-ctx.Done(): + return fmt.Errorf("failed to see the expected active validators of subnet %q before timeout", subnet.Name) + case <-ticker.C: + } + } +} + +// Reads subnets from [network dir]/subnets/[subnet name].json +func readSubnets(subnetDir string) ([]*Subnet, error) { + if _, err := os.Stat(subnetDir); os.IsNotExist(err) { + return nil, nil + } else if err != nil { + return nil, err + } + + entries, err := os.ReadDir(subnetDir) + if err != nil { + return nil, fmt.Errorf("failed to read subnet dir: %w", err) + } + + subnets := []*Subnet{} + for _, entry := range entries { + if entry.IsDir() { + // Looking only for files + continue + } + if filepath.Ext(entry.Name()) != ".json" { + // Subnet files should have a .json extension + continue + } + + subnetPath := filepath.Join(subnetDir, entry.Name()) + bytes, err := os.ReadFile(subnetPath) + if err != nil { + return nil, fmt.Errorf("failed to read subnet file %s: %w", subnetPath, err) + } + subnet := &Subnet{} + if err := json.Unmarshal(bytes, subnet); err != nil { + return nil, fmt.Errorf("failed to unmarshal subnet from %s: %w", subnetPath, err) + } + subnets = append(subnets, subnet) + } + + return subnets, nil +} diff --git a/tests/upgrade/upgrade_test.go b/tests/upgrade/upgrade_test.go index 37c2fd259e66..4dba94c17f23 100644 --- a/tests/upgrade/upgrade_test.go +++ b/tests/upgrade/upgrade_test.go @@ -15,6 +15,7 @@ import ( "github.com/stretchr/testify/require" "github.com/ava-labs/avalanchego/tests/fixture/e2e" + "github.com/ava-labs/avalanchego/tests/fixture/tmpnet" ) func TestUpgrade(t *testing.T) { @@ -46,7 +47,8 @@ var _ = ginkgo.Describe("[Upgrade]", func() { require := require.New(ginkgo.GinkgoT()) ginkgo.It("can upgrade versions", func() { - network := e2e.StartNetwork(avalancheGoExecPath, e2e.DefaultNetworkDir) + network := &tmpnet.Network{} + e2e.StartNetwork(network, e2e.DefaultNetworkDir, avalancheGoExecPath, "" /* pluginDir */) ginkgo.By(fmt.Sprintf("restarting all nodes with %q binary", avalancheGoExecPathToUpgradeTo)) for _, node := range network.Nodes {