Merge remote-tracking branch 'origin/main' into tb/pos/efficient-l1-p…

…olling

sequencer/src/api.rs

@@ -554,6 +554,7 @@ pub mod test_helpers {
         api_config: Options,
     }
 
+    #[derive(Clone)]
     pub struct TestNetworkConfigBuilder<const NUM_NODES: usize, P, C>
     where
         P: PersistenceOptions,
@@ -592,6 +593,19 @@ pub mod test_helpers {
                 api_config: None,
             }
         }
+
+        pub fn with_max_block_size(&self, max_block_size: u64) -> Self {
+            let cf = ChainConfig {
+                max_block_size: max_block_size.into(),
+                ..Default::default()
+            }
+            .into();
+
+            let mut cfg = self.clone();
+            cfg.state.iter_mut().for_each(|s| s.chain_config = cf);
+
+            cfg
+        }
     }
 
     impl<const NUM_NODES: usize, P, C> TestNetworkConfigBuilder<{ NUM_NODES }, P, C>

sequencer/src/bin/espresso-dev-node.rs

@@ -145,6 +145,15 @@ struct Args {
     #[clap(short, long, env = "ESPRESSO_DEV_NODE_PORT", default_value = "20000")]
     dev_node_port: u16,
 
+    /// Port for connecting to the builder.
+    #[clap(
+        short,
+        long,
+        env = "ESPRESSO_DEV_NODE_MAX_BLOCK_SIZE",
+        default_value = "1000000"
+    )]
+    max_block_size: u64,
+
     #[clap(flatten)]
     sql: persistence::sql::Options,
 
@@ -177,6 +186,7 @@ async fn main() -> anyhow::Result<()> {
         alt_prover_retry_intervals,
         alt_prover_update_intervals,
         l1_interval,
+        max_block_size,
     } = cli_params;
 
     logging.init();
@@ -215,6 +225,7 @@ async fn main() -> anyhow::Result<()> {
     let config = TestNetworkConfigBuilder::<NUM_NODES, _, _>::with_num_nodes()
         .api_config(api_options)
         .network_config(network_config)
+        .with_max_block_size(max_block_size)
         .build();
 
     let network =
@@ -620,6 +631,7 @@ mod tests {
                 tmp_dir.path().as_os_str(),
             )
             .env("ESPRESSO_SEQUENCER_DATABASE_MAX_CONNECTIONS", "25")
+            .env("ESPRESSO_DEV_NODE_MAX_BLOCK_SIZE", "500000")
             .spawn()
             .unwrap();
 
@@ -709,7 +721,7 @@ mod tests {
 
         {
             // transactions with size larger than max_block_size result in an error
-            let extremely_large_tx = Transaction::new(100_u32.into(), vec![0; 50120]);
+            let extremely_large_tx = Transaction::new(100_u32.into(), vec![0; 7 * 1000 * 1000]);
             api_client
                 .post::<Commitment<Transaction>>("submit/submit")
                 .body_json(&extremely_large_tx)

sequencer/src/lib.rs

@@ -481,7 +481,7 @@ pub async fn init_node<P: SequencerPersistence, V: Versions>(
     // Create the HotShot membership
     let membership = EpochCommittees::new_stake(
         network_config.config.known_nodes_with_stake.clone(),
-        network_config.config.known_nodes_with_stake.clone(),
+        network_config.config.known_da_nodes.clone(),
         &instance_state,
         network_config.config.epoch_height,
     );
@@ -972,7 +972,7 @@ pub mod testing {
             // Create the HotShot membership
             let membership = EpochCommittees::new_stake(
                 config.known_nodes_with_stake.clone(),
-                config.known_nodes_with_stake.clone(),
+                config.known_da_nodes.clone(),
                 &node_state,
                 100,
             );

types/src/v0/impls/stake_table.rs

@@ -21,7 +21,7 @@ use hotshot_types::{
 use itertools::Itertools;
 use std::{
     cmp::max,
-    collections::{BTreeMap, BTreeSet, HashMap},
+    collections::{BTreeSet, HashMap},
     num::NonZeroU64,
     str::FromStr,
 };
@@ -126,11 +126,17 @@ struct Committee {
     /// leader but without voting rights.
     eligible_leaders: Vec<StakeTableEntry<PubKey>>,
 
-    /// Stake Table entries indexed by Staker's `PubKey`
-    indexed_stake_table: BTreeMap<PubKey, StakeTableEntry<PubKey>>,
+    /// Keys for nodes participating in the network
+    stake_table: Vec<StakeTableEntry<PubKey>>,
 
-    /// Stake Holder's that are also DA members, indexed by `PubKey`
-    indexed_da_members: BTreeMap<PubKey, StakeTableEntry<PubKey>>,
+    /// Keys for DA members
+    da_members: Vec<StakeTableEntry<PubKey>>,
+
+    /// Stake entries indexed by public key, for efficient lookup.
+    indexed_stake_table: HashMap<PubKey, StakeTableEntry<PubKey>>,
+
+    /// DA entries indexed by public key, for efficient lookup.
+    indexed_da_members: HashMap<PubKey, StakeTableEntry<PubKey>>,
 }
 
 impl EpochCommittees {
@@ -144,14 +150,18 @@ impl EpochCommittees {
         // update events and building the table for us. We will need
         // more subtlety when start fetching only the events since last update.
 
-        let indexed_stake_table: BTreeMap<PubKey, _> = st
+        let stake_table = st.stake_table.0.clone();
+
+        let da_members = st.da_members.0.clone();
+
+        let indexed_stake_table: HashMap<PubKey, _> = st
             .stake_table
             .0
             .iter()
             .map(|entry| (PubKey::public_key(entry), entry.clone()))
             .collect();
 
-        let indexed_da_members: BTreeMap<PubKey, _> = st
+        let indexed_da_members: HashMap<PubKey, _> = st
             .da_members
             .0
             .iter()
@@ -167,6 +177,8 @@ impl EpochCommittees {
 
         let committee = Committee {
             eligible_leaders,
+            stake_table,
+            da_members,
             indexed_stake_table,
             indexed_da_members,
         };
@@ -191,7 +203,7 @@ impl EpochCommittees {
             .collect();
 
         // For each member, get the stake table entry
-        let members: Vec<_> = committee_members
+        let stake_table: Vec<_> = committee_members
             .iter()
             .map(|member| member.stake_table_entry.clone())
             .filter(|entry| entry.stake() > U256::zero())
@@ -205,19 +217,21 @@ impl EpochCommittees {
             .collect();
 
         // Index the stake table by public key
-        let indexed_stake_table: BTreeMap<PubKey, _> = members
+        let indexed_stake_table: HashMap<PubKey, _> = stake_table
             .iter()
             .map(|entry| (PubKey::public_key(entry), entry.clone()))
             .collect();
 
         // Index the stake table by public key
-        let indexed_da_members: BTreeMap<PubKey, _> = da_members
+        let indexed_da_members: HashMap<PubKey, _> = da_members
             .iter()
             .map(|entry| (PubKey::public_key(entry), entry.clone()))
             .collect();
 
         let members = Committee {
             eligible_leaders,
+            stake_table,
+            da_members,
             indexed_stake_table,
             indexed_da_members,
         };
@@ -259,7 +273,7 @@ impl Membership<SeqTypes> for EpochCommittees {
             .collect();
 
         // For each member, get the stake table entry
-        let members: Vec<_> = committee_members
+        let stake_table: Vec<_> = committee_members
             .iter()
             .map(|member| member.stake_table_entry.clone())
             .filter(|entry| entry.stake() > U256::zero())
@@ -273,19 +287,21 @@ impl Membership<SeqTypes> for EpochCommittees {
             .collect();
 
         // Index the stake table by public key
-        let indexed_stake_table: BTreeMap<PubKey, _> = members
+        let indexed_stake_table: HashMap<PubKey, _> = stake_table
             .iter()
             .map(|entry| (PubKey::public_key(entry), entry.clone()))
             .collect();
 
         // Index the stake table by public key
-        let indexed_da_members: BTreeMap<PubKey, _> = da_members
+        let indexed_da_members: HashMap<PubKey, _> = da_members
             .iter()
             .map(|entry| (PubKey::public_key(entry), entry.clone()))
             .collect();
 
         let members = Committee {
             eligible_leaders,
+            stake_table,
+            da_members,
             indexed_stake_table,
             indexed_da_members,
         };
@@ -306,15 +322,15 @@ impl Membership<SeqTypes> for EpochCommittees {
     /// Get the stake table for the current view
     fn stake_table(&self, epoch: Epoch) -> Vec<StakeTableEntry<PubKey>> {
         if let Some(st) = self.state.get(&epoch) {
-            st.indexed_stake_table.clone().into_values().collect()
+            st.stake_table.clone()
         } else {
             vec![]
         }
     }
     /// Get the stake table for the current view
     fn da_stake_table(&self, epoch: Epoch) -> Vec<StakeTableEntry<PubKey>> {
         if let Some(sc) = self.state.get(&epoch) {
-            sc.indexed_da_members.clone().into_values().collect()
+            sc.da_members.clone()
         } else {
             vec![]
         }
@@ -415,44 +431,44 @@ impl Membership<SeqTypes> for EpochCommittees {
     fn total_nodes(&self, epoch: Epoch) -> usize {
         self.state
             .get(&epoch)
-            .map(|sc| sc.indexed_stake_table.len())
+            .map(|sc| sc.stake_table.len())
             .unwrap_or_default()
     }
 
     /// Get the total number of DA nodes in the committee
     fn da_total_nodes(&self, epoch: Epoch) -> usize {
         self.state
             .get(&epoch)
-            .map(|sc: &Committee| sc.indexed_da_members.len())
+            .map(|sc: &Committee| sc.da_members.len())
             .unwrap_or_default()
     }
 
     /// Get the voting success threshold for the committee
     fn success_threshold(&self, epoch: Epoch) -> NonZeroU64 {
-        let quorum = self.state.get(&epoch).unwrap().indexed_stake_table.clone();
-        NonZeroU64::new(((quorum.len() as u64 * 2) / 3) + 1).unwrap()
+        let quorum_len = self.state.get(&epoch).unwrap().stake_table.len();
+        NonZeroU64::new(((quorum_len as u64 * 2) / 3) + 1).unwrap()
     }
 
     /// Get the voting success threshold for the committee
     fn da_success_threshold(&self, epoch: Epoch) -> NonZeroU64 {
-        let da = self.state.get(&epoch).unwrap().indexed_da_members.clone();
-        NonZeroU64::new(((da.len() as u64 * 2) / 3) + 1).unwrap()
+        let da_len = self.state.get(&epoch).unwrap().da_members.len();
+        NonZeroU64::new(((da_len as u64 * 2) / 3) + 1).unwrap()
     }
 
     /// Get the voting failure threshold for the committee
     fn failure_threshold(&self, epoch: Epoch) -> NonZeroU64 {
-        let quorum = self.state.get(&epoch).unwrap().indexed_stake_table.clone();
+        let quorum_len = self.state.get(&epoch).unwrap().stake_table.len();
 
-        NonZeroU64::new(((quorum.len() as u64) / 3) + 1).unwrap()
+        NonZeroU64::new(((quorum_len as u64) / 3) + 1).unwrap()
     }
 
     /// Get the voting upgrade threshold for the committee
     fn upgrade_threshold(&self, epoch: Epoch) -> NonZeroU64 {
-        let quorum = self.state.get(&epoch).unwrap().indexed_stake_table.clone();
+        let quorum_len = self.state.get(&epoch).unwrap().indexed_stake_table.len();
 
         NonZeroU64::new(max(
-            (quorum.len() as u64 * 9) / 10,
-            ((quorum.len() as u64 * 2) / 3) + 1,
+            (quorum_len as u64 * 9) / 10,
+            ((quorum_len as u64 * 2) / 3) + 1,
         ))
         .unwrap()
     }
@@ -466,6 +482,7 @@ impl Membership<SeqTypes> for EpochCommittees {
         self.l1_client
             .get_stake_table(address, block_header.height())
             .await
+            .ok()
             .map(|stake_table| -> Box<dyn FnOnce(&mut Self) + Send> {
                 Box::new(move |committee: &mut Self| {
                     let _ = committee.update_stake_table(epoch, stake_table);