Fix multi exit test, refactor registration state data flow

This patch fixes the multi-exit test which has been broken since the
patch "Simplify internal and ipv6 assignment". This is becuase that pr
stopped using the fixed internal ip assignment code, where the same
client would be deterministically assigned the same ip across multiple
exit instances.

This sort of id hash based assignment brought in more complexity than it
was worth. But removing it revealed that the exit details struct
shouldn't be left as a single item. After all our exit details including
the default route and our own ip can change between exits.

Including in this patch is an extensive re-write of the dataflow for the
exit registration state, this is to move that state out of the config
and avoid using a lazy-static to share it across threads. This modified
state is more contained, and cleaner, although I would hesitate to call
it clean since we sitll have some duplicated data in the exit manager
state.

Moving the exit state out of settings (and in fact any thing that may
change frequently) is very important becuase the settings lazy static
has a copy -> modify -> write structure without a lock, data
inconsistency is possible, and unavoidable for anything that changes
quickly like exit states now will.

althea_types/src/exits/mod.rs

@@ -52,6 +52,16 @@ pub enum ExitState {
 }
 
 impl ExitState {
+    pub fn get_exit_mesh_ip(&self) -> Option<IpAddr> {
+        match *self {
+            ExitState::Registered {
+                ref general_details,
+                ..
+            } => Some(general_details.server_internal_ip),
+            _ => None,
+        }
+    }
+
     pub fn general_details(&self) -> Option<&ExitDetails> {
         match *self {
             ExitState::Registered {

integration_tests/src/mutli_exit.rs

@@ -74,9 +74,6 @@ pub async fn run_multi_exit_test() {
 
     test_routes(namespaces.clone(), expected_routes);
 
-    // sleep so that the exit manager loop can grab the verified exits from the servers (required before registering)
-    thread::sleep(Duration::from_secs(10));
-
     info!("Registering routers to the exit");
     register_all_namespaces_to_exit(namespaces.clone()).await;
 

integration_tests/src/setup_utils/rita.rs

@@ -191,12 +191,12 @@ pub fn spawn_rita(
         let system = actix::System::new();
 
         start_rita_common_loops();
-        start_rita_client_loops();
+        let em_state = start_rita_client_loops();
         save_to_disk_loop(SettingsOnDisk::RitaClientSettings(Box::new(
             settings::get_rita_client(),
         )));
         start_core_rita_endpoints(1);
-        start_client_dashboard(s.network.rita_dashboard_port);
+        start_client_dashboard(s.network.rita_dashboard_port, em_state);
 
         if let Err(e) = system.run() {
             panic!("Starting client failed with {}", e);

integration_tests/src/utils.rs

@@ -336,17 +336,19 @@ pub fn test_all_internet_connectivity(namespaces: NamespaceInfo) {
                 ],
             )
             .unwrap();
-            if String::from_utf8(out.stdout)
-                .unwrap()
-                .contains("1 received")
-            {
+            let output_string = from_utf8(&out.stdout).unwrap();
+            if output_string.contains("1 received") {
                 info!("Ping test passed for {}!", ns.get_name());
                 break;
             } else {
-                if Instant::now() - start > Duration::from_secs(60) {
+                if Instant::now() - start > Duration::from_secs(600) {
                     panic!("{} does not have internet connectivity", ns.get_name());
                 }
-                error!("Ping failed for {}, trying again", ns.get_name());
+                error!(
+                    "Ping failed for {} with {}, trying again",
+                    ns.get_name(),
+                    output_string
+                );
                 thread::sleep(Duration::from_secs(5));
             }
         }

rita_bin/src/client.rs

@@ -176,12 +176,12 @@ fn main() {
     let system = actix::System::new();
 
     start_rita_common_loops();
-    start_rita_client_loops();
+    let em_ref = start_rita_client_loops();
     save_to_disk_loop(SettingsOnDisk::RitaClientSettings(Box::new(
         settings::get_rita_client(),
     )));
     start_core_rita_endpoints(4);
-    start_client_dashboard(settings.network.rita_dashboard_port);
+    start_client_dashboard(settings.network.rita_dashboard_port, em_ref);
     start_antenna_forwarder(settings);
 
     // utility and rescue fucntions, these perform some upgrade or check

rita_client/src/dashboard/exits.rs

@@ -1,16 +1,18 @@
 //! The Exit info endpoint gathers infromation about exit status and presents it to the dashbaord.
 
-use crate::exit_manager::get_current_exit;
 use crate::exit_manager::requests::exit_setup_request;
+use crate::exit_manager::ExitManager;
 use crate::RitaClientError;
 use actix_web::http::StatusCode;
+use actix_web::web;
 use actix_web::{web::Path, HttpRequest, HttpResponse};
 use althea_kernel_interface::ping_check::ping_check;
 use althea_types::{ExitIdentity, ExitState};
 use babel_monitor::open_babel_stream;
 use babel_monitor::parse_routes;
 use babel_monitor::parsing::do_we_have_route;
 use std::collections::HashMap;
+use std::sync::{Arc, RwLock};
 use std::time::Duration;
 
 #[derive(Serialize)]
@@ -56,7 +58,9 @@ fn is_tunnel_working(
     }
 }
 
-pub fn dashboard_get_exit_info() -> Result<Vec<ExitInfo>, RitaClientError> {
+pub fn dashboard_get_exit_info(
+    em_ref: Arc<Arc<RwLock<ExitManager>>>,
+) -> Result<Vec<ExitInfo>, RitaClientError> {
     let babel_port = settings::get_rita_client().network.babel_port;
     match open_babel_stream(babel_port, Duration::from_secs(5)) {
         Ok(mut stream) => {
@@ -66,8 +70,9 @@ pub fn dashboard_get_exit_info() -> Result<Vec<ExitInfo>, RitaClientError> {
                     let mut output = Vec::new();
                     let rita_client = settings::get_rita_client();
                     let exit_client = rita_client.exit_client;
-                    let reg_state = exit_client.registration_state;
-                    let current_exit = get_current_exit();
+                    let em_ref = em_ref.read().unwrap();
+                    let reg_state = em_ref.get_exit_registration_state();
+                    let current_exit = em_ref.get_current_exit();
 
                     let verified_exit_list = match exit_client.verified_exit_list.clone() {
                         Some(list) => list,
@@ -116,19 +121,22 @@ pub fn dashboard_get_exit_info() -> Result<Vec<ExitInfo>, RitaClientError> {
     }
 }
 
-pub async fn get_exit_info(_req: HttpRequest) -> HttpResponse {
+pub async fn get_exit_info(
+    _req: HttpRequest,
+    em_ref: web::Data<Arc<RwLock<ExitManager>>>,
+) -> HttpResponse {
     debug!("Exit endpoint hit!");
-    match dashboard_get_exit_info() {
+    match dashboard_get_exit_info(em_ref.into_inner()) {
         Ok(a) => HttpResponse::Ok().json(a),
         Err(e) => HttpResponse::build(StatusCode::INTERNAL_SERVER_ERROR).json(format!("{e:?}")),
     }
 }
 
-pub async fn register_to_exit() -> HttpResponse {
+pub async fn register_to_exit(em_ref: web::Data<Arc<RwLock<ExitManager>>>) -> HttpResponse {
     info!("/exit/register hit");
 
     let mut ret = HashMap::new();
-    if let Err(e) = exit_setup_request(None).await {
+    if let Err(e) = exit_setup_request(em_ref.into_inner(), None).await {
         error!("exit_setup_request() failed with: {:?}", e);
         ret.insert("error".to_owned(), "Exit setup request failed".to_owned());
         ret.insert("rust_error".to_owned(), format!("{e:?}"));
@@ -137,12 +145,15 @@ pub async fn register_to_exit() -> HttpResponse {
     HttpResponse::Ok().json(ret)
 }
 
-pub async fn verify_on_exit_with_code(path: Path<String>) -> HttpResponse {
+pub async fn verify_on_exit_with_code(
+    path: Path<String>,
+    em_ref: web::Data<Arc<RwLock<ExitManager>>>,
+) -> HttpResponse {
     let code = path.into_inner();
     debug!("/exit/verify/{} hit", code);
 
     let mut ret = HashMap::new();
-    if let Err(e) = exit_setup_request(Some(code)).await {
+    if let Err(e) = exit_setup_request(em_ref.into_inner(), Some(code)).await {
         error!("exit_setup_request() failed with: {:?}", e);
         ret.insert("error".to_owned(), "Exit setup request failed".to_owned());
         ret.insert("rust_error".to_owned(), format!("{e:?}"));

rita_client/src/dashboard/mod.rs

@@ -26,6 +26,7 @@ use crate::dashboard::operator_fees::*;
 use crate::dashboard::prices::*;
 use crate::dashboard::router::*;
 use crate::dashboard::usage::*;
+use crate::exit_manager::ExitManager;
 use actix::System;
 use actix_web::{web, App, HttpServer};
 use rita_common::dashboard::auth::*;
@@ -51,16 +52,21 @@ use rita_common::dashboard::wg_key::*;
 use rita_common::dashboard::wifi::*;
 use rita_common::middleware;
 use rita_common::network_endpoints::*;
+use std::sync::Arc;
+use std::sync::RwLock;
 use std::thread;
 
 use self::devices_on_lan::get_devices_lan_endpoint;
 
-pub fn start_client_dashboard(rita_dashboard_port: u16) {
+pub fn start_client_dashboard(
+    rita_dashboard_port: u16,
+    exit_manager_ref: Arc<RwLock<ExitManager>>,
+) {
     // dashboard
     thread::spawn(move || {
         let runner = System::new();
         runner.block_on(async move {
-            let _res = HttpServer::new(|| {
+            let _res = HttpServer::new(move || {
                 App::new()
                     .wrap(middleware::AuthMiddlewareFactory)
                     .wrap(middleware::HeadersMiddlewareFactory)
@@ -188,6 +194,7 @@ pub fn start_client_dashboard(rita_dashboard_port: u16) {
                     .route("/phone", web::post().to(set_phone_number))
                     .route("/email", web::get().to(get_email))
                     .route("/email", web::post().to(set_email))
+                    .app_data(web::Data::new(exit_manager_ref.clone()))
             })
             .workers(1)
             .bind(format!("[::0]:{rita_dashboard_port}"))

rita_client/src/dashboard/neighbors.rs

@@ -1,22 +1,21 @@
+use crate::exit_manager::ExitManager;
+use crate::RitaClientError;
 use actix_web::http::StatusCode;
-use actix_web::{HttpRequest, HttpResponse};
+use actix_web::{web, HttpRequest, HttpResponse};
 use althea_types::Identity;
 use arrayvec::ArrayString;
 use babel_monitor::parsing::get_installed_route;
 use babel_monitor::parsing::get_route_via_neigh;
 use babel_monitor::structs::Route;
 use babel_monitor::{open_babel_stream, parse_routes};
-
 use num256::{Int256, Uint256};
 use rita_common::debt_keeper::{dump, NodeDebtData};
 use rita_common::network_monitor::{get_stats, IfaceStats, Stats};
 use rita_common::tunnel_manager::{tm_get_neighbors, Neighbor};
 use std::collections::HashMap;
+use std::sync::{Arc, RwLock};
 use std::time::Duration;
 
-use crate::exit_manager::get_current_exit;
-use crate::RitaClientError;
-
 const BABEL_TIMEOUT: Duration = Duration::from_secs(5);
 
 #[derive(Serialize)]
@@ -53,7 +52,10 @@ pub async fn get_routes(_req: HttpRequest) -> HttpResponse {
 /// since the /debts endpoint was introduced, and should be removed when it can be
 /// coordinated with the frontend.
 /// The routes info might also belong in /exits or a dedicated /routes endpoint
-pub async fn get_neighbor_info(_req: HttpRequest) -> HttpResponse {
+pub async fn get_neighbor_info(
+    _req: HttpRequest,
+    em_ref: web::Data<Arc<RwLock<ExitManager>>>,
+) -> HttpResponse {
     let debts = dump();
     let neighbors = tm_get_neighbors();
     let combined_list = merge_debts_and_neighbors(neighbors, debts);
@@ -65,7 +67,12 @@ pub async fn get_neighbor_info(_req: HttpRequest) -> HttpResponse {
             if let Ok(routes) = routes {
                 let route_table_sample = routes;
                 let stats = get_stats();
-                let output = generate_neighbors_list(stats, route_table_sample, combined_list);
+                let output = generate_neighbors_list(
+                    stats,
+                    route_table_sample,
+                    combined_list,
+                    &em_ref.read().unwrap(),
+                );
                 HttpResponse::Ok().json(output)
             } else {
                 HttpResponse::build(StatusCode::INTERNAL_SERVER_ERROR).json(format!(
@@ -86,6 +93,7 @@ fn generate_neighbors_list(
     stats: Stats,
     route_table_sample: Vec<Route>,
     debts: HashMap<Identity, (NodeDebtData, Neighbor)>,
+    em_ref: &ExitManager,
 ) -> Vec<NodeInfo> {
     let mut output = Vec::new();
 
@@ -107,7 +115,7 @@ fn generate_neighbors_list(
         }
         let neigh_route = maybe_route.unwrap();
 
-        let exit_ip = match get_current_exit() {
+        let exit_ip = match em_ref.get_current_exit() {
             Some(exit) => exit.mesh_ip,
             None => {
                 output.push(nonviable_node_info(