feat(genesis.bin): add testnet/devnet/mainnet gensis files + new cli …

…arg (#203)

docs/sig-cli.md

@@ -0,0 +1,77 @@
+there are a few useful things to know when running a sig validator - while the full list 
+of commands and options is available with the `--help` flag, this doc will better explain the importance of each option and when/when-not to use them (as well as constants
+which you can change and recompile with)
+
+there are a few important commands
+
+## gossip
+
+`sig gossip` is useful for running a gossip client to connect to the network 
+and retrieve data. 
+
+entrypoints (ie, first few nodes to ping / send pull requests) can be defined with the `-e` flag - this will usually be 
+`entrypoint.mainnet-beta.solana.com:8001` or `entrypoint.testnet.solana.com:8001`
+to start talking to the network
+
+you wont need to change any of the defaults unless you are doing something specical 
+requiring a different host or port.
+
+we dont have a way to access the gossip data stored yet.
+
+we do have many helpful metrics stored in a grafana dashboard while gossip runs which 
+you can read me about in `prometheus-grafana/`
+
+## snapshot-download
+
+to load accountsdb, you first need to download a snapshot of the state on the network. 
+to do this you can use the `sig snapshot-download` command. 
+
+downloading a snapshot uses gossip to find peers to download from, you will still need to provide entrypoints
+using the `-e` command. 
+
+*note:* for testing/experiment purposes you should probably only download from testnet (7GB compressed), mainnet snapshots are very expensive to download + load (32GB compressed)
+
+using the `-s` command you can change to where the snapshot is downloaded. 
+
+if you cant find a fast enough peer to download from (default is *at least* 20MB/s) you 
+can reduce the min download speed using the flag `--min-snapshot-download-speed`. 
+
+The download speed can vary over time too, you can change `DOWNLOAD_PROGRESS_UPDATES_NS` 
+(in `src/accountsdb/download.zig`) to wait longer before it checks if the peer is fast enough 
+(default waits 30seconds).
+
+## snapshot-validate / loading accountsdb from a snapshot
+
+loading/validating from can be tricky due to the number of accounts required to load 
+
+the `-s` flag you can point to where the snapshot is
+- the code looks for a file with `.tar.zst` extension to startup from 
+- if there is an `accounts/` directory, it will skip decompressing/unarchiving and 
+load directly from the account files in `accounts/`
+    - you can force a fresh unpack if you use the `-f` flag
+
+if you are running OOM when loading, its likely you dont have enough RAM to generate the 
+account index. you can use disk memory to back the index using `--use-disk-index` and reduce the ram requirements.
+
+#### a note on genesis files
+
+genesis files are typically downloaded as `.tar.bzip` - there isnt much support for bzip in zig (and the C code wasnt easy to port to zig) - because the genesis files shouldnt change we predownloaded and unpacked
+the genesis binaries for mainnet, testnet, and devnet in `genesis-files/`. 
+
+you need to provide a path to the matching genesis file of the snapshot you downloaded
+to complete the verification using the `-g` command. 
+
+for example, if you download a testnet snapshot and want to run verification, you 
+need to run `./zig-out/bin/sig snapshot-validate -s ../testnet-snapshots/ -g genesis-files/testnet-genesis.bin`
+
+## validator
+
+the `sig validator` command is something we are building out overtime to start a full validator.
+
+right now we support 
+- starting gossip 
+- downloading or loading from a snapshot 
+- computing the leaderschedule from a snapshot
+- collecting shreds from the network
+
+most of the important options have been described in the above commands

src/accountsdb/db.zig

@@ -26,12 +26,10 @@ const AllSnapshotFields = sig.accounts_db.snapshots.AllSnapshotFields;
 const SnapshotFieldsAndPaths = sig.accounts_db.snapshots.SnapshotFieldsAndPaths;
 const SnapshotFiles = sig.accounts_db.snapshots.SnapshotFiles;
 const AccountIndex = sig.accounts_db.index.AccountIndex;
-const ValidateAccountFileError = AccountIndex.ValidateAccountFileError;
 const AccountRef = sig.accounts_db.index.AccountRef;
 const DiskMemoryAllocator = sig.accounts_db.index.DiskMemoryAllocator;
 const parallelUnpackZstdTarBall = sig.accounts_db.snapshots.parallelUnpackZstdTarBall;
 const spawnThreadTasks = sig.utils.thread.spawnThreadTasks;
-const readDirectory = sig.utils.directory.readDirectory;
 const RwMux = sig.sync.RwMux;
 const Logger = sig.trace.log.Logger;
 const printTimeEstimate = sig.time.estimate.printTimeEstimate;

src/accountsdb/download.zig

@@ -6,7 +6,6 @@ const sig = @import("../lib.zig");
 
 const SlotAndHash = sig.accounts_db.snapshots.SlotAndHash;
 const Pubkey = sig.core.Pubkey;
-const Hash = sig.core.Hash;
 const GossipTable = sig.gossip.GossipTable;
 const ThreadSafeContactInfo = sig.gossip.data.ThreadSafeContactInfo;
 const GossipService = sig.gossip.GossipService;

src/accountsdb/genesis_config.zig

@@ -194,6 +194,7 @@ pub const GenesisConfig = struct {
     ) !GenesisConfig {
         var file = try std.fs.cwd().openFile(genesis_path, .{});
         defer file.close();
+
         const config = try bincode.read(allocator, GenesisConfig, file.reader(), .{});
 
         return config;
@@ -204,7 +205,7 @@ pub const GenesisConfig = struct {
     }
 };
 
-test "core.genesis_config: deserialize config" {
+test "genesis_config deserialize development config" {
     const allocator = std.testing.allocator;
 
     const genesis_path = "./test_data/genesis.bin";
@@ -213,3 +214,33 @@ test "core.genesis_config: deserialize config" {
 
     try std.testing.expect(config.cluster_type == ClusterType.Development);
 }
+
+test "genesis_config deserialize testnet config" {
+    const allocator = std.testing.allocator;
+
+    const genesis_path = "./genesis-files/testnet-genesis.bin";
+    const config = try GenesisConfig.init(allocator, genesis_path);
+    defer config.deinit(allocator);
+
+    try std.testing.expect(config.cluster_type == ClusterType.Testnet);
+}
+
+test "genesis_config deserialize devnet config" {
+    const allocator = std.testing.allocator;
+
+    const genesis_path = "./genesis-files/devnet-genesis.bin";
+    const config = try GenesisConfig.init(allocator, genesis_path);
+    defer config.deinit(allocator);
+
+    try std.testing.expect(config.cluster_type == ClusterType.Devnet);
+}
+
+test "genesis_config deserialize mainnet config" {
+    const allocator = std.testing.allocator;
+
+    const genesis_path = "./genesis-files/mainnet-genesis.bin";
+    const config = try GenesisConfig.init(allocator, genesis_path);
+    defer config.deinit(allocator);
+
+    try std.testing.expect(config.cluster_type == ClusterType.MainnetBeta);
+}

src/accountsdb/snapshots.zig

@@ -22,7 +22,6 @@ const Inflation = sig.accounts_db.genesis_config.Inflation;
 const SlotHistory = sig.accounts_db.sysvars.SlotHistory;
 const FileId = sig.accounts_db.accounts_file.FileId;
 
-const defaultArrayListOnEOFConfig = bincode.arraylist.defaultArrayListOnEOFConfig;
 const defaultArrayListUnmanagedOnEOFConfig = bincode.arraylist.defaultArrayListUnmanagedOnEOFConfig;
 const readDirectory = sig.utils.directory.readDirectory;
 const parallelUntarToFileSystem = sig.utils.tar.parallelUntarToFileSystem;

src/bloom/bloom.zig

@@ -4,11 +4,9 @@ const sig = @import("../lib.zig");
 const testing = std.testing;
 const bincode = sig.bincode;
 
-const RndGen = std.rand.DefaultPrng;
 const ArrayList = std.ArrayList;
 
 const DynamicArrayBitSet = sig.bloom.bit_set.DynamicArrayBitSet;
-const BitVec = sig.bloom.bit_vec.BitVec;
 const BitVecConfig = sig.bloom.bit_vec.BitVecConfig;
 const FnvHasher = sig.crypto.FnvHasher;
 

src/cmd/cmd.zig

@@ -16,7 +16,6 @@ const ContactInfo = sig.gossip.ContactInfo;
 const GenesisConfig = sig.accounts_db.GenesisConfig;
 const GossipService = sig.gossip.GossipService;
 const IpAddr = sig.net.IpAddr;
-const Level = sig.trace.Level;
 const Logger = sig.trace.Logger;
 const Pubkey = sig.core.Pubkey;
 const ShredCollectorDependencies = sig.shred_collector.ShredCollectorDependencies;
@@ -204,6 +203,15 @@ pub fn run() !void {
         .value_name = "snapshot_dir",
     };
 
+    var genesis_file_path = cli.Option{
+        .long_name = "genesis-file-path",
+        .help = "path to the genesis file",
+        .short_alias = 'g',
+        .value_ref = cli.mkRef(&config.current.genesis_file_path),
+        .required = false,
+        .value_name = "genesis_file_path",
+    };
+
     var min_snapshot_download_speed_mb_option = cli.Option{
         .long_name = "min-snapshot-download-speed",
         .help = "minimum download speed of full snapshots in megabytes per second - default: 20MB/s",
@@ -300,6 +308,7 @@ pub fn run() !void {
                             &force_new_snapshot_download_option,
                             &trusted_validators_option,
                             &number_of_index_bins_option,
+                            &genesis_file_path,
                             // general
                             &leader_schedule_option,
                         },
@@ -351,6 +360,7 @@ pub fn run() !void {
                             &n_threads_snapshot_unpack_option,
                             &force_unpack_snapshot_option,
                             &number_of_index_bins_option,
+                            &genesis_file_path,
                         },
                         .target = .{
                             .action = .{
@@ -389,6 +399,7 @@ pub fn run() !void {
                             &force_new_snapshot_download_option,
                             &trusted_validators_option,
                             &number_of_index_bins_option,
+                            &genesis_file_path,
                             // general
                             &leader_schedule_option,
                         },
@@ -801,6 +812,8 @@ fn loadSnapshot(
     const result = try allocator.create(LoadedSnapshot);
     errdefer allocator.destroy(result);
 
+    result.allocator = allocator;
+
     var snapshots = try getOrDownloadSnapshots(
         allocator,
         logger,
@@ -849,9 +862,10 @@ fn loadSnapshot(
 
     // this should exist before we start to unpack
     logger.infof("reading genesis...", .{});
-    result.genesis_config = readGenesisConfig(allocator, snapshot_dir_str) catch |err| {
+    const genesis_file_path = config.current.genesis_file_path orelse return error.GenesisNotProvided;
+    result.genesis_config = readGenesisConfig(allocator, genesis_file_path) catch |err| {
         if (err == error.GenesisNotFound) {
-            logger.errf("genesis.bin not found - expecting {s}/genesis.bin to exist", .{snapshot_dir_str});
+            logger.errf("genesis config not found - expecting {s} to exist", .{genesis_file_path});
         }
         return err;
     };
@@ -881,6 +895,7 @@ fn loadSnapshot(
 
     var slot_history = try result.accounts_db.getSlotHistory();
     defer slot_history.deinit(result.accounts_db.allocator);
+
     try status_cache.validate(allocator, bank_fields.slot, &slot_history);
 
     logger.infof("accounts-db setup done...", .{});
@@ -889,14 +904,7 @@ fn loadSnapshot(
 }
 
 /// load genesis config with default filenames
-fn readGenesisConfig(allocator: Allocator, snapshot_dir: []const u8) !GenesisConfig {
-    const genesis_path = try std.fmt.allocPrint(
-        allocator,
-        "{s}/genesis.bin",
-        .{snapshot_dir},
-    );
-    defer allocator.free(genesis_path);
-
+fn readGenesisConfig(allocator: Allocator, genesis_path: []const u8) !GenesisConfig {
     std.fs.cwd().access(genesis_path, .{}) catch {
         return error.GenesisNotFound;
     };
@@ -1035,6 +1043,16 @@ fn getOrDownloadSnapshots(
         .{snapshot_dir_str},
     );
     defer allocator.free(accounts_path);
+    errdefer {
+        // if something goes wrong, delete the accounts/ directory
+        // so we unpack the full snapshot the next time.
+        //
+        // NOTE: if we didnt do this, we would try to startup with a incomplete
+        // accounts/ directory the next time we ran the code - see `should_unpack_snapshot`.
+        std.fs.cwd().deleteTree(accounts_path) catch |err| {
+            std.debug.print("failed to delete accounts/ dir: {}\n", .{err});
+        };
+    }
 
     var accounts_path_exists = true;
     std.fs.cwd().access(accounts_path, .{}) catch {

src/cmd/config.zig

@@ -10,6 +10,7 @@ pub const Config = struct {
     shred_collector: ShredCollectorConfig = shred_collector_defaults,
     accounts_db: AccountsDBConfig = .{},
     leader_schedule_path: ?[]const u8 = null,
+    genesis_file_path: ?[]const u8 = null,
     // general config
     log_level: LogLevel = .debug,
     metrics_port: u16 = 12345,

src/gossip/fuzz_service.zig

@@ -11,7 +11,6 @@ const ChunkType = sig.gossip.service.ChunkType;
 const LegacyContactInfo = sig.gossip.data.LegacyContactInfo;
 const SignedGossipData = sig.gossip.data.SignedGossipData;
 const ContactInfo = sig.gossip.data.ContactInfo;
-const Logger = sig.trace.log.Logger;
 const GossipMessage = sig.gossip.message.GossipMessage;
 const GossipPullFilterSet = sig.gossip.pull_request.GossipPullFilterSet;
 const GossipPullFilter = sig.gossip.pull_request.GossipPullFilter;
@@ -28,7 +27,6 @@ const KeyPair = std.crypto.sign.Ed25519.KeyPair;
 const AtomicBool = std.atomic.Value(bool);
 
 const gossipDataToPackets = sig.gossip.service.gossipDataToPackets;
-const getWallclockMs = sig.gossip.data.getWallclockMs;
 
 const Duration = sig.time.Duration;
 

src/gossip/fuzz_table.zig

@@ -1,37 +1,20 @@
 const std = @import("std");
 const sig = @import("../lib.zig");
 const network = @import("zig-network");
-const bincode = sig.bincode.bincode;
 
 const AtomicBool = std.atomic.Value(bool);
 const KeyPair = std.crypto.sign.Ed25519.KeyPair;
 
-const GossipService = sig.gossip.service.GossipService;
-const ChunkType = sig.gossip.service.ChunkType;
-const LegacyContactInfo = sig.gossip.data.LegacyContactInfo;
 const ContactInfo = sig.gossip.data.ContactInfo;
-const gossipDataToPackets = sig.gossip.service.gossipDataToPackets;
-const getWallclockMs = sig.gossip.data.getWallclockMs;
 const Logger = sig.trace.log.Logger;
-const SocketAddr = sig.net.net.SocketAddr;
 const Pubkey = sig.core.pubkey.Pubkey;
-const Bloom = sig.bloom.bloom.Bloom;
-const EndPoint = network.EndPoint;
-const Packet = sig.net.packet.Packet;
-const PACKET_DATA_SIZE = sig.net.packet.PACKET_DATA_SIZE;
-const GossipMessage = sig.gossip.message.GossipMessage;
-const Ping = sig.gossip.ping_pong.Ping;
-const Pong = sig.gossip.ping_pong.Pong;
 const GossipTable = sig.gossip.GossipTable;
 
 const SignedGossipData = sig.gossip.data.SignedGossipData;
 const GossipData = sig.gossip.data.GossipData;
 const GossipKey = sig.gossip.data.GossipKey;
 const Signature = sig.core.Signature;
 
-const GossipPullFilterSet = sig.gossip.pull_request.GossipPullFilterSet;
-const GossipPullFilter = sig.gossip.pull_request.GossipPullFilter;
-const Hash = sig.core.hash.Hash;
 const ThreadPool = sig.sync.thread_pool.ThreadPool;
 const Duration = sig.time.Duration;
 

src/gossip/pull_request.zig

@@ -10,7 +10,6 @@ const exp = std.math.exp;
 const GossipTable = @import("table.zig").GossipTable;
 const _gossip_data = @import("data.zig");
 const SignedGossipData = _gossip_data.SignedGossipData;
-const getWallclockMs = _gossip_data.getWallclockMs;
 const RwMux = @import("../sync/mux.zig").RwMux;
 
 pub const MAX_BLOOM_SIZE: usize = 928;

src/gossip/service.zig

@@ -16,14 +16,12 @@ const UdpSocket = network.Socket;
 
 const Pubkey = sig.core.Pubkey;
 const Hash = sig.core.Hash;
-const Entry = sig.trace.entry.Entry;
 const Logger = sig.trace.log.Logger;
 const Packet = sig.net.Packet;
 const EchoServer = sig.net.echo.Server;
 const SocketAddr = sig.net.SocketAddr;
 const Counter = sig.prometheus.counter.Counter;
 const GetMetricError = sig.prometheus.registry.GetMetricError;
-const Registry = sig.prometheus.Registry;
 const ThreadPoolTask = sig.utils.thread.ThreadPoolTask;
 const ThreadPool = sig.sync.ThreadPool;
 const Task = sig.sync.ThreadPool.Task;
@@ -58,7 +56,6 @@ const getWallclockMs = sig.gossip.data.getWallclockMs;
 const PACKET_DATA_SIZE = sig.net.packet.PACKET_DATA_SIZE;
 const UNIQUE_PUBKEY_CAPACITY = sig.gossip.table.UNIQUE_PUBKEY_CAPACITY;
 const MAX_NUM_PULL_REQUESTS = sig.gossip.pull_request.MAX_NUM_PULL_REQUESTS;
-const MAX_BLOOM_SIZE = sig.gossip.pull_request.MAX_BLOOM_SIZE;
 
 const PacketBatch = ArrayList(Packet);
 const GossipMessageWithEndpoint = struct { from_endpoint: EndPoint, message: GossipMessage };

src/gossip/table.zig

@@ -15,12 +15,10 @@ const GossipKey = sig.gossip.data.GossipKey;
 const LegacyContactInfo = sig.gossip.data.LegacyContactInfo;
 const ContactInfo = sig.gossip.data.ContactInfo;
 const ThreadSafeContactInfo = sig.gossip.data.ThreadSafeContactInfo;
-const Vote = sig.gossip.data.Vote;
 const ThreadPool = sig.sync.ThreadPool;
 const Task = sig.sync.ThreadPool.Task;
 const Batch = sig.sync.ThreadPool.Batch;
 const Hash = sig.core.hash.Hash;
-const Transaction = sig.core.transaction.Transaction;
 const Pubkey = sig.core.Pubkey;
 const SocketAddr = sig.net.SocketAddr;