tmpnet: Add support for subnets (#2492)

tests/e2e/e2e_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/onsi/gomega"
 
 	"github.com/ava-labs/avalanchego/tests/fixture/e2e"
+	"github.com/ava-labs/avalanchego/tests/fixture/tmpnet"
 
 	// ensure test packages are scanned by ginkgo
 	_ "github.com/ava-labs/avalanchego/tests/e2e/banff"
@@ -34,7 +35,7 @@ func init() {
 
 var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
 	// Run only once in the first ginkgo process
-	return e2e.NewTestEnvironment(flagVars).Marshal()
+	return e2e.NewTestEnvironment(flagVars, &tmpnet.Network{}).Marshal()
 }, func(envBytes []byte) {
 	// Run in every ginkgo process
 

tests/fixture/e2e/env.go

@@ -14,6 +14,8 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/ava-labs/avalanchego/api/info"
+	"github.com/ava-labs/avalanchego/config"
 	"github.com/ava-labs/avalanchego/tests"
 	"github.com/ava-labs/avalanchego/tests/fixture"
 	"github.com/ava-labs/avalanchego/tests/fixture/tmpnet"
@@ -29,10 +31,9 @@ var Env *TestEnvironment
 func InitSharedTestEnvironment(envBytes []byte) {
 	require := require.New(ginkgo.GinkgoT())
 	require.Nil(Env, "env already initialized")
-	Env = &TestEnvironment{
-		require: require,
-	}
+	Env = &TestEnvironment{}
 	require.NoError(json.Unmarshal(envBytes, Env))
+	Env.require = require
 }
 
 type TestEnvironment struct {
@@ -53,7 +54,7 @@ func (te *TestEnvironment) Marshal() []byte {
 }
 
 // Initialize a new test environment with a shared network (either pre-existing or newly created).
-func NewTestEnvironment(flagVars *FlagVars) *TestEnvironment {
+func NewTestEnvironment(flagVars *FlagVars, desiredNetwork *tmpnet.Network) *TestEnvironment {
 	require := require.New(ginkgo.GinkgoT())
 
 	networkDir := flagVars.NetworkDir()
@@ -65,10 +66,44 @@ func NewTestEnvironment(flagVars *FlagVars) *TestEnvironment {
 		network, err = tmpnet.ReadNetwork(networkDir)
 		require.NoError(err)
 		tests.Outf("{{yellow}}Using an existing network configured at %s{{/}}\n", network.Dir)
+
+		// Set the desired subnet configuration to ensure subsequent creation.
+		for _, subnet := range desiredNetwork.Subnets {
+			if existing := network.GetSubnet(subnet.Name); existing != nil {
+				// Already present
+				continue
+			}
+			network.Subnets = append(network.Subnets, subnet)
+		}
 	} else {
-		network = StartNetwork(flagVars.AvalancheGoExecPath(), DefaultNetworkDir)
+		network = desiredNetwork
+		StartNetwork(network, DefaultNetworkDir, flagVars.AvalancheGoExecPath(), flagVars.PluginDir())
 	}
 
+	// A new network will always need subnet creation and an existing
+	// network will also need subnets to be created the first time it
+	// is used.
+	require.NoError(network.CreateSubnets(DefaultContext(), ginkgo.GinkgoWriter))
+
+	// Wait for chains to have bootstrapped on all nodes
+	Eventually(func() bool {
+		for _, subnet := range network.Subnets {
+			for _, validatorID := range subnet.ValidatorIDs {
+				uri, err := network.GetURIForNodeID(validatorID)
+				require.NoError(err)
+				infoClient := info.NewClient(uri)
+				for _, chain := range subnet.Chains {
+					isBootstrapped, err := infoClient.IsBootstrapped(DefaultContext(), chain.ChainID.String())
+					// Ignore errors since a chain id that is not yet known will result in a recoverable error.
+					if err != nil || !isBootstrapped {
+						return false
+					}
+				}
+			}
+		}
+		return true
+	}, DefaultTimeout, DefaultPollingInterval, "failed to see all chains bootstrap before timeout")
+
 	uris := network.GetNodeURIs()
 	require.NotEmpty(uris, "network contains no nodes")
 	tests.Outf("{{green}}network URIs: {{/}} %+v\n", uris)
@@ -83,6 +118,7 @@ func NewTestEnvironment(flagVars *FlagVars) *TestEnvironment {
 		NetworkDir:        network.Dir,
 		URIs:              uris,
 		TestDataServerURI: testDataServerURI,
+		require:           require,
 	}
 }
 
@@ -127,10 +163,22 @@ func (te *TestEnvironment) NewPrivateNetwork() *tmpnet.Network {
 	sharedNetwork, err := tmpnet.ReadNetwork(te.NetworkDir)
 	te.require.NoError(err)
 
+	network := &tmpnet.Network{}
+
 	// The private networks dir is under the shared network dir to ensure it
 	// will be included in the artifact uploaded in CI.
 	privateNetworksDir := filepath.Join(sharedNetwork.Dir, PrivateNetworksDirName)
 	te.require.NoError(os.MkdirAll(privateNetworksDir, perms.ReadWriteExecute))
 
-	return StartNetwork(sharedNetwork.DefaultRuntimeConfig.AvalancheGoPath, privateNetworksDir)
+	pluginDir, err := sharedNetwork.DefaultFlags.GetStringVal(config.PluginDirKey)
+	te.require.NoError(err)
+
+	StartNetwork(
+		network,
+		privateNetworksDir,
+		sharedNetwork.DefaultRuntimeConfig.AvalancheGoPath,
+		pluginDir,
+	)
+
+	return network
 }

tests/fixture/e2e/flags.go

@@ -13,10 +13,19 @@ import (
 
 type FlagVars struct {
 	avalancheGoExecPath string
+	pluginDir           string
 	networkDir          string
 	useExistingNetwork  bool
 }
 
+func (v *FlagVars) AvalancheGoExecPath() string {
+	return v.avalancheGoExecPath
+}
+
+func (v *FlagVars) PluginDir() string {
+	return v.pluginDir
+}
+
 func (v *FlagVars) NetworkDir() string {
 	if !v.useExistingNetwork {
 		return ""
@@ -27,10 +36,6 @@ func (v *FlagVars) NetworkDir() string {
 	return os.Getenv(tmpnet.NetworkDirEnvName)
 }
 
-func (v *FlagVars) AvalancheGoExecPath() string {
-	return v.avalancheGoExecPath
-}
-
 func (v *FlagVars) UseExistingNetwork() bool {
 	return v.useExistingNetwork
 }
@@ -43,6 +48,12 @@ func RegisterFlags() *FlagVars {
 		os.Getenv(tmpnet.AvalancheGoPathEnvName),
 		fmt.Sprintf("avalanchego executable path (required if not using an existing network). Also possible to configure via the %s env variable.", tmpnet.AvalancheGoPathEnvName),
 	)
+	flag.StringVar(
+		&vars.pluginDir,
+		"plugin-dir",
+		os.ExpandEnv("$HOME/.avalanchego/plugins"),
+		"[optional] the dir containing VM plugins.",
+	)
 	flag.StringVar(
 		&vars.networkDir,
 		"network-dir",

tests/fixture/e2e/helpers.go

@@ -34,15 +34,15 @@ const (
 	// contention.
 	DefaultTimeout = 2 * time.Minute
 
-	// Interval appropriate for network operations that should be
-	// retried periodically but not too often.
-	DefaultPollingInterval = 500 * time.Millisecond
+	DefaultPollingInterval = tmpnet.DefaultPollingInterval
 
 	// Setting this env will disable post-test bootstrap
 	// checks. Useful for speeding up iteration during test
 	// development.
 	SkipBootstrapChecksEnvName = "E2E_SKIP_BOOTSTRAP_CHECKS"
 
+	DefaultValidatorStartTimeDiff = tmpnet.DefaultValidatorStartTimeDiff
+
 	DefaultGasLimit = uint64(21000) // Standard gas limit
 
 	// An empty string prompts the use of the default path which ensures a
@@ -217,13 +217,20 @@ func CheckBootstrapIsPossible(network *tmpnet.Network) {
 }
 
 // Start a temporary network with the provided avalanchego binary.
-func StartNetwork(avalancheGoExecPath string, rootNetworkDir string) *tmpnet.Network {
+func StartNetwork(network *tmpnet.Network, rootNetworkDir string, avalancheGoExecPath string, pluginDir string) {
 	require := require.New(ginkgo.GinkgoT())
 
-	network, err := tmpnet.NewDefaultNetwork(ginkgo.GinkgoWriter, avalancheGoExecPath, tmpnet.DefaultNodeCount)
-	require.NoError(err)
-	require.NoError(network.Create(rootNetworkDir))
-	require.NoError(network.Start(DefaultContext(), ginkgo.GinkgoWriter))
+	require.NoError(
+		tmpnet.StartNewNetwork(
+			DefaultContext(),
+			ginkgo.GinkgoWriter,
+			network,
+			rootNetworkDir,
+			avalancheGoExecPath,
+			pluginDir,
+			tmpnet.DefaultNodeCount,
+		),
+	)
 
 	ginkgo.DeferCleanup(func() {
 		tests.Outf("Shutting down network\n")
@@ -233,6 +240,4 @@ func StartNetwork(avalancheGoExecPath string, rootNetworkDir string) *tmpnet.Net
 	})
 
 	tests.Outf("{{green}}Successfully started network{{/}}\n")
-
-	return network
 }

tests/fixture/tmpnet/README.md

@@ -34,6 +34,7 @@ the following non-test files:
 | node.go           | Node        | Orchestrates and configures nodes              |
 | node_config.go    | Node        | Reads and writes node configuration            |
 | node_process.go   | NodeProcess | Orchestrates node processes                    |
+| subnet.go         | Subnet      | Orchestrates subnets                           |
 | utils.go          |             | Defines shared utility functions               |
 
 ## Usage
@@ -74,16 +75,33 @@ network.
 A temporary network can be managed in code:
 
 ```golang
-network, _ := tmpnet.NewDefaultNetwork(
+network := &tmpnet.Network{                   // Configure non-default values for the new network
+    DefaultFlags: tmpnet.FlagsMap{
+        config.LogLevelKey: "INFO",           // Change one of the network's defaults
+    },
+    Subnets: []*tmpnet.Subnet{                // Subnets to create on the new network once it is running
+        {
+            Name: "xsvm-a",                   // User-defined name used to reference subnet in code and on disk
+            Chains: []*tmpnet.Chain{
+                {
+                    VMName: "xsvm",           // Name of the VM the chain will run, will be used to derive the name of the VM binary
+                    Genesis: <genesis bytes>, // Genesis bytes used to initialize the custom chain
+                    PreFundedKey: <key>,      // (Optional) A private key that is funded in the genesis bytes
+                },
+            },
+        },
+    },
+}
+
+_ := tmpnet.StartNewNetwork(              // Start the network
+    ctx,                                  // Context used to limit duration of waiting for network health
     ginkgo.GinkgoWriter,                  // Writer to report progress of initialization
+    network,
+    "",                                   // Empty string uses the default network path (~/tmpnet/networks)
     "/path/to/avalanchego",               // The path to the binary that nodes will execute
+    "/path/to/plugins",                   // The path nodes will use for plugin binaries (suggested value ~/.avalanchego/plugins)
     5,                                    // Number of initial validating nodes
 )
-_ = network.Create("")                    // Finalize network configuration and write to disk
-_ = network.Start(                        // Start the nodes of the network and wait until they report healthy
-    ctx,                                  // Context used to limit duration of waiting for network health
-    ginkgo.GinkgoWriter,                  // Writer to report progress of network start
-)
 
 uris := network.GetNodeURIs()
 
@@ -125,11 +143,16 @@ HOME
             │   │   └── ...
             │   └── process.json                         // Node process details (PID, API URI, staking address)
             ├── chains
-            │   └── C
-            │       └── config.json                      // C-Chain config for all nodes
+            │   ├── C
+            │   │   └── config.json                      // C-Chain config for all nodes
+            │   └── raZ51bwfepaSaZ1MNSRNYNs3ZPfj...U7pa3
+            │       └── config.json                      // Custom chain configuration for all nodes
             ├── config.json                              // Common configuration (including defaults and pre-funded keys)
             ├── genesis.json                             // Genesis for all nodes
-            └── network.env                              // Sets network dir env var to simplify network usage
+            ├── network.env                              // Sets network dir env var to simplify network usage
+            └── subnets                                  // Parent directory for subnet definitions
+                ├─ subnet-a.json                         // Configuration for subnet-a and its chain(s)
+                └─ subnet-b.json                         // Configuration for subnet-b and its chain(s)
 ```
 
 ### Common networking configuration
@@ -148,17 +171,19 @@ content will be generated with reasonable defaults if not
 supplied. Each node in the network can override the default by setting
 an explicit value for `--genesis-file` or `--genesis-file-content`.
 
-### C-Chain config
+### Chain configuration
 
-The C-Chain config for a temporary network is stored at
-`[network-dir]/chains/C/config.json` and referenced by default by all
-nodes in the network. The C-Chain config will be generated with
-reasonable defaults if not supplied. Each node in the network can
-override the default by setting an explicit value for
-`--chain-config-dir` and ensuring the C-Chain config file exists at
-`[chain-config-dir]/C/config.json`.
+The chain configuration for a temporary network is stored at
+`[network-dir]/chains/[chain alias or ID]/config.json` and referenced
+by all nodes in the network. The C-Chain config will be generated with
+reasonable defaults if not supplied. X-Chain and P-Chain will use
+implicit defaults. The configuration for custom chains can be provided
+with subnet configuration and will be writen to the appropriate path.
 
-TODO(marun) Enable configuration of X-Chain and P-Chain.
+Each node in the network can override network-level chain
+configuration by setting `--chain-config-dir` to an explicit value and
+ensuring that configuration files for all chains exist at
+`[custom-chain-config-dir]/[chain alias or ID]/config.json`.
 
 ### Network env