CGNAT Exit and integration test

.github/workflows/rust.yml

@@ -133,4 +133,14 @@ jobs:
         with:
           cache-on-failure: true
       - name: Run integration test
-        run:  bash scripts/integration_tests/all-up-test-ci.sh SNAT_EXIT
+        run:  bash scripts/integration_tests/all-up-test-ci.sh SNAT_EXIT
+  integration-test-cgnat-exit:
+    needs: check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+      - name: Run integration test
+        run:  bash scripts/integration_tests/all-up-test-ci.sh CGNAT_EXIT

althea_kernel_interface/src/exit_server_tunnel.rs

@@ -10,7 +10,7 @@ use crate::run_command;
 use crate::setup_wg_if::get_peers;
 use crate::traffic_control::{create_root_classful_limit, has_limit};
 use althea_types::WgKey;
-use ipnetwork::IpNetwork;
+use ipnetwork::{IpNetwork, Ipv4Network};
 use std::collections::HashSet;
 use std::net::{IpAddr, Ipv4Addr};
 use KernelInterfaceError as Error;
@@ -368,6 +368,52 @@ pub fn teardown_snat(
     Ok(())
 }
 
+/// Sets up the CGNAT rules for the exit server run on startup. The actual internal <-> external ip
+/// allocation is done by the kernel at random, and clients may be assigned any ip in the given range.
+pub fn setup_cgnat(
+    exit_ip: Ipv4Addr,
+    mask: u32,
+    ex_nic: &str,
+    possible_ips: Vec<Ipv4Addr>,
+    internal_subnet: Ipv4Network,
+) -> Result<(), Error> {
+    init_filter_chain()?;
+    let _ = add_ipv4_mask(exit_ip, mask, ex_nic);
+    // get the ip range from first and last in possible ips
+    let ip_range = format!(
+        "{}-{}",
+        possible_ips.first().unwrap(),
+        possible_ips.last().unwrap()
+    );
+    /*
+    nft add rule ip nat POSTROUTING oifname $EXT_IF ip saddr $EXIT_SUBNET counter snat to $EXT_RANGE
+    */
+    run_command(
+        "nft",
+        &[
+            "add",
+            "rule",
+            "ip",
+            "nat",
+            "postrouting",
+            "oifname",
+            ex_nic,
+            "ip",
+            "saddr",
+            &format!("{}", internal_subnet),
+            "counter",
+            "snat",
+            "to",
+            ip_range.as_str(),
+        ],
+    )?;
+    // for each ip in the possible ips range, add it to the external interface
+    for ip in possible_ips {
+        add_ipv4(ip, ex_nic)?;
+    }
+    Ok(())
+}
+
 #[test]
 fn test_iproute_parsing() {
     let str = "fbad::/64,feee::/64";

integration_tests/src/cgnat_exit.rs

@@ -0,0 +1,73 @@
+use ipnetwork::Ipv4Network;
+use settings::exit::ExitIpv4RoutingSettings;
+
+use crate::five_nodes::five_node_config;
+use crate::setup_utils::namespaces::*;
+use crate::setup_utils::rita::{spawn_exit_root_of_trust, thread_spawner};
+use crate::utils::{
+    add_exits_contract_exit_list, deploy_contracts, get_default_settings, populate_routers_eth,
+    register_all_namespaces_to_exit, test_all_internet_connectivity, test_reach_all, test_routes,
+};
+use std::net::Ipv4Addr;
+use std::str::FromStr;
+
+/// Runs a five node fixed network map test scenario, this does basic network setup and tests reachability to
+/// all destinations
+pub async fn run_cgnat_exit_test_scenario() {
+    info!("Starting cgnat exit node test scenario");
+    let node_config = five_node_config();
+    let namespaces = node_config.0;
+    let expected_routes = node_config.1;
+
+    info!("Waiting to deploy contracts");
+    let db_addr = deploy_contracts().await;
+
+    let (client_settings, mut exit_settings, exit_root_addr) =
+        get_default_settings(namespaces.clone(), db_addr);
+
+    // using /29 allows us to test that multiple clients can use the same external IP if randomly assigned
+    exit_settings.exit_network.ipv4_routing = ExitIpv4RoutingSettings::CGNAT {
+        subnet: Ipv4Network::from_str("10.0.0.0/29").unwrap(),
+        static_assignments: Vec::new(),
+        gateway_ipv4: Ipv4Addr::new(10, 0, 0, 1),
+        external_ipv4: Ipv4Addr::new(10, 0, 0, 2),
+        broadcast_ipv4: Ipv4Addr::new(10, 0, 0, 255),
+    };
+
+    namespaces.validate();
+
+    let res = setup_ns(namespaces.clone(), "cgnat");
+    info!("Namespaces setup: {res:?}");
+
+    info!("Starting root server!");
+    spawn_exit_root_of_trust(db_addr).await;
+
+    let rita_identities = thread_spawner(
+        namespaces.clone(),
+        client_settings,
+        exit_settings.clone(),
+        db_addr,
+    )
+    .expect("Could not spawn Rita threads");
+    info!("Thread Spawner: {res:?}");
+
+    // Add exits to the contract exit list so clients get the propers exits they can migrate to
+    add_exits_contract_exit_list(db_addr, exit_settings.exit_network, rita_identities.clone())
+        .await;
+
+    info!("About to populate routers with eth");
+    populate_routers_eth(rita_identities, exit_root_addr).await;
+
+    test_reach_all(namespaces.clone());
+
+    test_routes(namespaces.clone(), expected_routes);
+
+    info!("Registering routers to the exit");
+    register_all_namespaces_to_exit(namespaces.clone()).await;
+
+    info!("Checking for wg_exit tunnel setup");
+    test_all_internet_connectivity(namespaces.clone());
+    info!("All clients successfully registered!");
+
+    info!("cgnat exit node test scenario complete");
+}

integration_tests/src/lib.rs

@@ -6,6 +6,7 @@ extern crate log;
 
 use std::time::Duration;
 
+pub mod cgnat_exit;
 pub mod contract_test;
 pub mod debts;
 pub mod five_nodes;

integration_tests/src/setup_utils/namespaces.rs

@@ -245,6 +245,7 @@ pub fn setup_ns(spaces: NamespaceInfo, exit_mode: &str) -> Result<(), KernelInte
             let veth_exit_to_native = format!("vout-{}-o", name.get_name());
             let exit_ip = match exit_mode {
                 "snat" => "10.0.0.2/24".to_string(),
+                "cgnat" => "10.0.0.2/29".to_string(),
                 _ => format!(
                     "10.0.{}.{}/24",
                     name.id.to_be_bytes()[0],

rita_exit/src/database/ipddr_assignment.rs

@@ -36,9 +36,10 @@ pub struct ClientListAnIpAssignmentMap {
     /// A map of ipv4 addresses assigned to clients, these are used internally for the wg_exit tunnel
     /// and never external, the external ip is determined by the exit's nat settings
     internal_ip_assignments: DualMap<Ipv4Addr, Identity>,
-    /// The external ip for a specific client or set of clients depending on the ipv4 nat mode. Under CGNAT
-    /// each ip will have multiple fixed clients, under SNAT each ip will have one client
-    external_ip_assignemnts: HashMap<Ipv4Addr, HashSet<Identity>>,
+    /// The external ip for a specific client or set of clients depending on the ipv4 nat mode. under SNAT
+    /// each ip will have one client, under CGNAT multiple clients may share an ip but are not explicitly
+    /// assigned to any one.
+    external_ip_assignemnts: HashMap<Ipv4Addr, Identity>,
     /// A set of all clients that have been registered with the exit
     registered_clients: HashSet<Identity>,
     /// A list of clients that have been inactive past WG_INACTIVE_THRESHOLD. in SNAT mode these
@@ -70,7 +71,7 @@ impl ClientListAnIpAssignmentMap {
         }
     }
 
-    pub fn get_external_ip_assignments(&self) -> &HashMap<Ipv4Addr, HashSet<Identity>> {
+    pub fn get_external_ip_assignments(&self) -> &HashMap<Ipv4Addr, Identity> {
         &self.external_ip_assignemnts
     }
 
@@ -147,80 +148,18 @@ impl ClientListAnIpAssignmentMap {
                 Ok(None)
             }
             ExitIpv4RoutingSettings::CGNAT {
-                subnet,
-                static_assignments,
+                static_assignments, ..
             } => {
-                // check static assignmetns first
+                // only static assignments have a fixed external ip
                 for id in static_assignments {
                     if their_record == id.client_id {
-                        // make sure we have assigned this clients external ip. in CGNAT mode static clients just get
-                        // the same ip every time, they don't get that ip exclusively assigned to them, so adding to this
-                        // list is mostly a way to load balance the clients across the available ips including any static assignments
-                        // in that count.
-                        match self.external_ip_assignemnts.get_mut(&id.client_external_ip) {
-                            Some(clients) => {
-                                clients.insert(their_record);
-                            }
-                            None => {
-                                let mut new_clients = HashSet::new();
-                                new_clients.insert(their_record);
-                                self.external_ip_assignemnts
-                                    .insert(id.client_external_ip, new_clients);
-                            }
-                        }
-
+                        // in CGNAT mode static clients are assigned an external ip at random from the available ips
+                        // in the exit's external subnet, so only those with explicit static assignments will have a
+                        // fixed ip returned here
                         return Ok(Some(id.client_external_ip));
                     }
                 }
-
-                // check for already assigned ips
-                for (ip, clients) in self.external_ip_assignemnts.iter() {
-                    if clients.contains(&their_record) {
-                        return Ok(Some(*ip));
-                    }
-                }
-
-                // if we don't have a static assignment, we need to assign an ip, we should pick the ip with the fewest clients
-                // note this code is designed for relatively small subnets, but since public ipv4 are so valuable it's improbable
-                // anyone with a /8 is going to show up and use this.
-                let mut possible_ips: Vec<Ipv4Addr> = subnet.into_iter().collect();
-                // we don't want to assign the first ip in the subnet as it's the gateway
-                possible_ips.remove(0);
-                possible_ips.pop(); // we don't want to assign the last ip in the subnet as it's the broadcast address
-
-                let mut target_ip = None;
-                let mut last_num_assigned = usize::MAX;
-                for ip in possible_ips {
-                    match self.external_ip_assignemnts.get(&ip) {
-                        Some(clients) => {
-                            if clients.len() < last_num_assigned {
-                                target_ip = Some(ip);
-                                last_num_assigned = clients.len();
-                            }
-                        }
-                        None => {
-                            target_ip = Some(ip);
-                            // may as well break here, it's impossible to do better than an ip unused
-                            // by any other clients
-                            break;
-                        }
-                    }
-                }
-
-                // finally we add the newly assigned ip to the list of clients
-                let target_ip = target_ip.unwrap();
-                match self.external_ip_assignemnts.get_mut(&target_ip) {
-                    Some(clients) => {
-                        clients.insert(their_record);
-                    }
-                    None => {
-                        let mut new_clients = HashSet::new();
-                        new_clients.insert(their_record);
-                        self.external_ip_assignemnts.insert(target_ip, new_clients);
-                    }
-                }
-
-                Ok(Some(target_ip))
+                Ok(None)
             }
             ExitIpv4RoutingSettings::SNAT {
                 subnet,
@@ -233,15 +172,13 @@ impl ClientListAnIpAssignmentMap {
                 // so we need to make sure the static ip assignments are handled first by building the full list
                 for id in static_assignments {
                     // duplicate static assignments are a configuration error
-                    let mut new_clients = HashSet::new();
-                    new_clients.insert(id.client_id);
                     self.external_ip_assignemnts
-                        .insert(id.client_external_ip, new_clients);
+                        .insert(id.client_external_ip, id.client_id);
                 }
 
                 // check for already assigned ips
-                for (ip, clients) in self.external_ip_assignemnts.iter() {
-                    if clients.contains(&their_record) {
+                for (ip, client) in self.external_ip_assignemnts.iter() {
+                    if client == &their_record {
                         return Ok(Some(*ip));
                     }
                 }
@@ -263,10 +200,7 @@ impl ClientListAnIpAssignmentMap {
 
                 match target_ip {
                     Some(ip) => {
-                        // since this is SNAT we never have to deal with multiple clients on the same ip
-                        let mut new_clients = HashSet::new();
-                        new_clients.insert(their_record);
-                        self.external_ip_assignemnts.insert(ip, new_clients);
+                        self.external_ip_assignemnts.insert(ip, their_record);
                         self.client_first_connect
                             .insert(their_record.wg_public_key, SystemTime::now());
                         Ok(Some(ip))
@@ -286,7 +220,7 @@ impl ClientListAnIpAssignmentMap {
         match self
             .external_ip_assignemnts
             .iter()
-            .find(|(_, clients)| clients.contains(&id))
+            .find(|(_, client)| *client == &id)
             .map(|(ip, _)| *ip)
         {
             Some(ip) => {
@@ -463,11 +397,7 @@ mod tests {
         ClientIpv4StaticAssignment, ClientIpv6StaticAssignment, ExitInternalIpv4Settings,
         ExitIpv4RoutingSettings, ExitIpv6RoutingSettings,
     };
-    use std::{
-        collections::{HashMap, HashSet},
-        net::Ipv4Addr,
-        vec,
-    };
+    use std::{collections::HashSet, net::Ipv4Addr};
 
     pub fn get_ipv4_internal_test_subnet() -> Ipv4Network {
         "10.0.0.0/8".parse().unwrap()
@@ -545,6 +475,9 @@ mod tests {
         let ipv4_settings = ExitIpv4RoutingSettings::CGNAT {
             subnet: get_ipv4_external_test_subnet(),
             static_assignments,
+            gateway_ipv4: Ipv4Addr::new(172, 168, 1, 1),
+            external_ipv4: Ipv4Addr::new(172, 168, 1, 2),
+            broadcast_ipv4: Ipv4Addr::new(172, 168, 1, 255),
         };
         ipv4_settings.validate().unwrap();
         let internal_ipv4_settings = ExitInternalIpv4Settings {
@@ -579,7 +512,7 @@ mod tests {
     fn test_cgnat_external_ip_assignment() {
         let static_assignments = vec![ClientIpv4StaticAssignment {
             client_id: random_identity(),
-            client_external_ip: "172.168.1.12".parse().unwrap(),
+            client_external_ip: "172.168.1.254".parse().unwrap(),
         }];
         let mut data = get_test_config_cgnat(static_assignments.clone());
 
@@ -593,20 +526,14 @@ mod tests {
             clients.push(random_identity());
         }
 
-        let mut assigned_ip_count = HashMap::new();
-
-        // assign everyone an ip, make sure static assignments are respected
+        // make sure static assignments are respected
         for client in clients {
-            let ip = data.get_or_add_client_external_ip(client).unwrap().unwrap();
+            let ip = data.get_or_add_client_external_ip(client).unwrap();
             for assignment in static_assignments.iter() {
                 if assignment.client_id == client {
-                    assert_eq!(ip, assignment.client_external_ip);
+                    assert_eq!(ip.unwrap(), assignment.client_external_ip);
                 }
             }
-            assigned_ip_count
-                .entry(ip)
-                .and_modify(|e| *e += 1)
-                .or_insert(1);
         }
     }
 
@@ -640,7 +567,6 @@ mod tests {
         }
 
         for assignment in data.get_external_ip_assignments() {
-            assert_eq!(assignment.1.len(), 1);
             // make sure we can't receive the gateway, exit external or broadcast ips
             assert_ne!(assignment.0, &Ipv4Addr::new(172, 168, 1, 1));
             assert_ne!(assignment.0, &Ipv4Addr::new(172, 168, 1, 2));