feat: add state snapshot bridge (#1592)

Cargo.toml

@@ -104,7 +104,9 @@ ethereum_ssz = "0.7.1"
 ethereum_ssz_derive = "0.7.1"
 ethportal-api = { path = "ethportal-api" }
 futures = "0.3.23"
+futures-util = "0.3.23"
 hex = "0.4.3"
+humanize-duration = "0.0.6"
 itertools = "0.13.0"
 jsonrpsee = "0.24.4"
 keccak-hash = "0.10.0"

e2store/Cargo.toml

@@ -25,6 +25,7 @@ tracing = { workspace = true, optional = true }
 trin-utils = { workspace = true, optional = true }
 scraper.workspace = true
 snap.workspace = true
+url.workspace = true
 
 [dev-dependencies]
 rstest.workspace = true

e2store/src/utils.rs

@@ -4,9 +4,11 @@ use anyhow::{ensure, Error};
 use rand::{seq::SliceRandom, thread_rng};
 use reqwest::Client;
 use scraper::{Html, Selector};
+use url::Url;
 
 const ERA_DIR_URL: &str = "https://mainnet.era.nimbus.team/";
 const ERA1_DIR_URL: &str = "https://era1.ethportal.net/";
+const ERA2_DIR_URL: &str = "https://era2.ethportal.net/index.html";
 pub const ERA1_FILE_COUNT: usize = 1897;
 
 pub fn underlying_io_error_kind(error: &Error) -> Option<io::ErrorKind> {
@@ -18,7 +20,7 @@ pub fn underlying_io_error_kind(error: &Error) -> Option<io::ErrorKind> {
     None
 }
 
-pub async fn download_era_links(
+async fn download_era_links(
     http_client: &Client,
     url: &str,
 ) -> anyhow::Result<HashMap<u64, String>> {
@@ -38,7 +40,12 @@ pub async fn download_era_links(
                 .expect("to be able to get epoch")
                 .parse::<u64>()
                 .expect("to be able to parse epoch");
-            (epoch_index, format!("{url}{href}"))
+            let url = Url::parse(url)
+                .and_then(|url| url.join(href))
+                .unwrap_or_else(|_| {
+                    panic!("to construct valid url from base ({url}) and href ({href}).")
+                });
+            (epoch_index, url.to_string())
         })
         .collect();
     Ok(era_files)
@@ -77,6 +84,11 @@ pub async fn get_era1_files(http_client: &Client) -> anyhow::Result<HashMap<u64,
     Ok(era1_files)
 }
 
+pub async fn get_era2_files(http_client: &Client) -> anyhow::Result<HashMap<u64, String>> {
+    let era2_files = download_era_links(http_client, ERA2_DIR_URL).await?;
+    Ok(era2_files)
+}
+
 /// Fetches era1 files hosted on era1.ethportal.net and shuffles them
 pub async fn get_shuffled_era1_files(http_client: &Client) -> anyhow::Result<Vec<String>> {
     let era1_files = get_era1_files(http_client).await?;

portal-bridge/Cargo.toml

@@ -24,6 +24,7 @@ eth_trie.workspace = true
 ethereum_ssz.workspace = true
 ethportal-api.workspace = true
 futures.workspace = true
+humanize-duration.workspace = true
 itertools.workspace = true
 jsonrpsee = { workspace = true, features = [
   "async-client",

portal-bridge/README.md

@@ -44,7 +44,6 @@ cargo run -p portal-bridge -- --executable-path ./target/debug/trin --epoch-accu
     - before gossiping a individual piece of content, the bridge will perform a lookup to see if the content is already in the portal network. If it is, the content will not be gossiped.
 - `"--mode fourfours:single_hunter:10:50`: sample size = 10, threshold = 50
     - same as the above hunter mode, but it will only gossip a single era1 file before exiting
-
 #### Beacon Subnetwork
 
 - `"--mode latest"`: follow the head of the chain and gossip latest blocks
@@ -54,6 +53,8 @@ cargo run -p portal-bridge -- --executable-path ./target/debug/trin --epoch-accu
 
 - `"--mode single:b100"`: backfill, always beginning from block #0 until the specified block (#100)
 - `"--mode single:r50-100"`: backfill, gossips state diffs for blocks in #50-#100 range (inclusive)
+- `"--mode snapshot:1000000"`: gossips a state snapshot at the respective block, in this example the state snapshot at block 1,000,000 will be gossiped. This mode is only used for the State Network.
+
 
 ### Subnetwork configuration