From 97843e181d15a1fb358330623251a811de449f9f Mon Sep 17 00:00:00 2001 From: "benjamin.747" Date: Mon, 8 Jan 2024 11:07:58 +0800 Subject: [PATCH] init e2e testing code --- Cargo.toml | 4 + docs/api.md | 2 +- docs/development.md | 1 - gateway/Cargo.toml | 2 +- git/Cargo.toml | 2 +- p2p/Cargo.toml | 7 +- p2p/src/lib.rs | 12 +-- p2p/src/network/behaviour.rs | 3 +- p2p/src/node/client_http.rs | 63 ++++++++++++++- storage/Cargo.toml | 20 ++--- storage/src/driver/database/mysql_storage.rs | 14 ++-- storage/src/driver/file_storage/mod.rs | 4 +- tests/common_test/mod.rs | 80 ++++++++++++++++++++ tests/compose/http/compose.yaml | 40 ++++++++++ tests/compose/mega_p2p/compose.yaml | 14 ++-- tests/integration_test.rs | 41 ++++++++++ 16 files changed, 262 insertions(+), 47 deletions(-) create mode 100644 tests/common_test/mod.rs create mode 100644 tests/compose/http/compose.yaml create mode 100644 tests/integration_test.rs diff --git a/Cargo.toml b/Cargo.toml index b3cc4175..230db3c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,5 +25,9 @@ tokio = { version = "1.35.1", features = ["macros"] } clap = { version = "4.4.11", features = ["derive"] } serde = { version = "1.0", features = ["derive"] } + +[dev-dependencies] +reqwest = "0.11.23" + [build-dependencies] shadow-rs = "0.26.0" diff --git a/docs/api.md b/docs/api.md index 93e56b93..ee8461ff 100644 --- a/docs/api.md +++ b/docs/api.md @@ -28,7 +28,7 @@ HTTP implement for git transfer data between two repositories The Git LFS client uses an HTTPS server to coordinate fetching and storing large binary objects separately from a Git server. -1. 通过object id下载lfs协议需要的git对象 +1. Downloading the Git objects required by the LFS protocol using an object ID. ```bash GET **/objetcs/:object_id diff --git a/docs/development.md b/docs/development.md index 8f314b7f..7d2d972b 100644 --- a/docs/development.md +++ b/docs/development.md @@ -47,7 +47,6 @@ use std::str::FromStr; use std::sync::{Arc, Mutex}; // 2. Third-Party Crates -use async_trait::async_trait; use bytes::{BufMut, Bytes, BytesMut}; use russh::server::{self, Auth, Msg, Session}; use russh::{Channel, ChannelId}; diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml index 118ed509..23f2c9d6 100644 --- a/gateway/Cargo.toml +++ b/gateway/Cargo.toml @@ -16,7 +16,7 @@ common = {path = "../common"} storage = {path = "../storage"} entity = { path = "../storage/entity" } anyhow = "1.0.77" -axum = "0.7.2" +axum = "0.7.3" tower = "0.4.13" tower-http = { version = "0.5.0", features = ["cors", "trace"] } tokio = {version = "1.35", features = ["net"]} diff --git a/git/Cargo.toml b/git/Cargo.toml index 6854be95..692aabc6 100644 --- a/git/Cargo.toml +++ b/git/Cargo.toml @@ -44,7 +44,7 @@ async-recursion = "1.0" num_cpus = "1.16.0" dotenvy = "0.15.7" diffs = "0.5.1" -sea-orm = { version = "0.12", features = [ +sea-orm = { version = "0.12.10", features = [ "runtime-tokio-rustls", "macros", "mock", diff --git a/p2p/Cargo.toml b/p2p/Cargo.toml index a43710da..dd9777f0 100644 --- a/p2p/Cargo.toml +++ b/p2p/Cargo.toml @@ -25,11 +25,10 @@ async-std = { version = "1.12.0", features = ["attributes"] } libp2p = { version = "0.53", features = ["dcutr", "kad", "yamux", "noise", "identify", "macros", "relay", "tcp", "async-std", "rendezvous", "request-response", "cbor", "secp256k1"] } serde = { version = "1.0", features = ["derive"] } clap = { version = "4.4.11", features = ["derive"] } -serde_json = "1.0" -async-trait = "0.1" +serde_json = "1.0.111" +async-trait = "0.1.77" cbor4ii = { version = "0.3.1", features = ["serde1", "use_std"] } redis = { version = "0.23", features = ["tokio-comp"] } secp256k1 = { version = "0.27.0", features = ["serde", "bitcoin-hashes", "bitcoin-hashes-std", "rand"] } -axum = "0.7.2" - +axum = "0.7.3" diff --git a/p2p/src/lib.rs b/p2p/src/lib.rs index 5540b464..27258d06 100644 --- a/p2p/src/lib.rs +++ b/p2p/src/lib.rs @@ -5,19 +5,19 @@ //! //! -use storage::driver::database::storage::ObjectStorage; -use git::protocol::{PackProtocol, Protocol}; use std::path::PathBuf; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; -pub mod network; -pub mod node; -pub mod peer; +use git::protocol::{PackProtocol, Protocol}; +use storage::driver::database::storage::ObjectStorage; + pub mod cbor; pub mod internal; - +pub mod network; +pub mod node; pub mod nostr; +pub mod peer; fn get_pack_protocol(path: &str, storage: Arc) -> PackProtocol { let path = del_ends_str(path, ".git"); diff --git a/p2p/src/network/behaviour.rs b/p2p/src/network/behaviour.rs index c55f9b3e..82fe8917 100644 --- a/p2p/src/network/behaviour.rs +++ b/p2p/src/network/behaviour.rs @@ -5,9 +5,9 @@ use libp2p::swarm::NetworkBehaviour; use libp2p::{dcutr, identify, relay, rendezvous, request_response}; use serde::{Deserialize, Serialize}; use std::collections::HashSet; + use crate::cbor; use crate::nostr::{NostrReq, NostrRes}; - #[derive(NetworkBehaviour)] #[behaviour(to_swarm = "Event")] pub struct Behaviour { @@ -47,7 +47,6 @@ pub struct GitObjectReq(pub String, pub Vec); #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct GitObjectRes(pub Vec); - #[derive(Debug)] #[allow(clippy::large_enum_variant)] pub enum Event { diff --git a/p2p/src/node/client_http.rs b/p2p/src/node/client_http.rs index 741fd481..5579b4d9 100644 --- a/p2p/src/node/client_http.rs +++ b/p2p/src/node/client_http.rs @@ -30,8 +30,9 @@ pub async fn server(sender: mpsc::Sender) { .nest( "/api/v1", Router::new() - .nest("/mega/", mega_routers()) - .nest("/nostr", nostr_routers()), + .nest("/mega", mega_routers()) + .nest("/nostr", nostr_routers()) + .route("/status", get(life_cycle_check)), ) // .layer(TraceLayer::new_for_http()) .with_state(state); @@ -53,6 +54,10 @@ pub fn mega_routers() -> Router { .route("/pull-object", get(mega_pull_obj)) } +async fn life_cycle_check() -> Result { + Ok(Json("ok")) +} + async fn mega_provide( Query(query): Query>, state: State, @@ -160,3 +165,57 @@ async fn nostr_event_issue( state.0.sender.clone().try_send(line).unwrap(); Ok(Json("ok")) } + +#[cfg(test)] +mod test { + use std::collections::HashMap; + + use async_std::stream::StreamExt; + use axum::{extract::Query, http::Uri}; + use futures::channel::mpsc; + + use crate::node::client_http::{ + mega_clone, mega_clone_obj, mega_pull, mega_pull_obj, P2pNodeState, + }; + use crate::node::client_http::{mega_provide, mega_search}; + + #[tokio::test] + async fn test_mega_routers() { + let query: Query> = Query::try_from_uri( + &"http://localhost:8001/api/v1/mega/provide?repo_name=reponame.git" + .parse::() + .unwrap(), + ) + .unwrap(); + + let addr_query: Query> = Query::try_from_uri( + &"http://localhost:8001/api/v1/mega/clone?mega_address=p2p://peer_id/reponame.git" + .parse::() + .unwrap(), + ) + .unwrap(); + + let (tx, mut rx) = mpsc::channel::(64); + let s = P2pNodeState { sender: tx }; + let state = axum::extract::State(s); + let _ = mega_provide(query.clone(), state.clone()).await; + let _ = mega_search(query.clone(), state.clone()).await; + let _ = mega_clone(addr_query.clone(), state.clone()).await; + let _ = mega_clone_obj(query.clone(), state.clone()).await; + let _ = mega_pull(addr_query.clone(), state.clone()).await; + let _ = mega_pull_obj(query.clone(), state.clone()).await; + + assert_eq!(rx.next().await.unwrap(), "mega provide reponame.git"); + assert_eq!(rx.next().await.unwrap(), "mega search reponame.git"); + assert_eq!( + rx.next().await.unwrap(), + "mega clone p2p://peer_id/reponame.git" + ); + assert_eq!(rx.next().await.unwrap(), "mega clone-object reponame.git"); + assert_eq!( + rx.next().await.unwrap(), + "mega pull p2p://peer_id/reponame.git" + ); + assert_eq!(rx.next().await.unwrap(), "mega pull-object reponame.git"); + } +} diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 4f62b152..0d84683a 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -13,16 +13,16 @@ path = "src/lib.rs" [dependencies] common = {path = "../common"} entity = {path = "./entity"} -anyhow = "1.0" -async-trait = "0.1" +anyhow = "1.0.79" +async-trait = "0.1.77" tracing = "0.1.40" idgenerator = "2.0.0" -chrono = "0.4" -sha256 = "1.4" -serde = "1.0" -serde_json = "1.0" -futures = "0.3" -sea-orm = {version = "0.12", features = [ +chrono = "0.4.31" +sha256 = "1.5" +serde = "1.0.195" +serde_json = "1.0.111" +futures = "0.3.30" +sea-orm = {version = "0.12.10", features = [ "sqlx-postgres", "sqlx-mysql", "runtime-tokio-rustls", @@ -31,8 +31,8 @@ sea-orm = {version = "0.12", features = [ aws-config = {version = "1.1.1", features = ["behavior-version-latest"]} aws-sdk-s3 = "1.11.0" aws-smithy-types = "1.1.1" -thiserror = "1.0.52" +thiserror = "1.0.56" bytes = "1.5.0" [dev-dependencies] -tokio = { version = "1.35.0", features = ["macros"] } \ No newline at end of file +tokio = { version = "1.35.1", features = ["macros"] } \ No newline at end of file diff --git a/storage/src/driver/database/mysql_storage.rs b/storage/src/driver/database/mysql_storage.rs index b8042aa0..b768b213 100644 --- a/storage/src/driver/database/mysql_storage.rs +++ b/storage/src/driver/database/mysql_storage.rs @@ -4,24 +4,20 @@ //! use async_trait::async_trait; - -use entity::commit; - -use entity::objects; -use entity::refs; - use sea_orm::DatabaseBackend; use sea_orm::DatabaseConnection; use sea_orm::DatabaseTransaction; use sea_orm::EntityTrait; - use sea_orm::Statement; use sea_orm::TryIntoModel; -use crate::driver::database::storage::ObjectStorage; +use common::errors::MegaError; +use entity::commit; +use entity::objects; +use entity::refs; use crate::driver::database::storage::batch_save_model; -use common::errors::MegaError; +use crate::driver::database::storage::ObjectStorage; #[derive(Debug, Default)] pub struct MysqlStorage { diff --git a/storage/src/driver/file_storage/mod.rs b/storage/src/driver/file_storage/mod.rs index f2ea54f6..af77be67 100644 --- a/storage/src/driver/file_storage/mod.rs +++ b/storage/src/driver/file_storage/mod.rs @@ -6,11 +6,11 @@ use std::{ use async_trait::async_trait; use bytes::Bytes; + use common::errors::MegaError; use crate::driver::file_storage::local_storage::LocalStorage; - -use self::remote_storage::RemoteStorage; +use crate::driver::file_storage::remote_storage::RemoteStorage; pub mod local_storage; pub mod remote_storage; diff --git a/tests/common_test/mod.rs b/tests/common_test/mod.rs new file mode 100644 index 00000000..ffbecdcf --- /dev/null +++ b/tests/common_test/mod.rs @@ -0,0 +1,80 @@ +use std::{ + process::Command, + thread::{self, sleep}, + time::Duration, +}; + +#[derive(Clone)] +pub struct P2pTestConfig { + pub compose_path: String, + pub lifecycle_url: String, + pub lifecycle_retrying: u64, +} +pub async fn init_p2p_server(config: P2pTestConfig) { + // docker compose -f tests/compose/mega_p2p/compose.yaml up --build + let P2pTestConfig { + compose_path, + lifecycle_url, + lifecycle_retrying, + } = config; + thread::spawn(move || { + let mut child = Command::new("docker") + .arg("compose") // Provide arguments if needed + .arg("-f") + .arg(compose_path) + .arg("up") + .stdin(std::process::Stdio::piped()) // Pass the standard input stream as an argument + .stdout(std::process::Stdio::piped()) + .spawn() + .expect("Failed to execute command"); + // Wait for the child process to finish and get the result + let _ = child.wait().expect("Failed to wait for child process"); + }); + + loop { + let resp = reqwest::get(&lifecycle_url).await.unwrap(); + if resp.status() == 200 { + break; + } else { + println!( + "lifecycle check failed, retrying in {} secs ...", + lifecycle_retrying + ); + } + sleep(Duration::from_secs(lifecycle_retrying)); + } +} + +pub fn provide_data_before_test() { + let res = Command::new("git") + .arg("remote") + .arg("set-url") + .arg("local") + .arg("http://localhost:8000/projects/mega.git") + .output() + .expect("Failed to execute command"); + assert!(res.status.success()); + let res2 = Command::new("git") + .arg("push") + .arg("local") + .arg("main") + .output() + .expect("Failed to execute command"); + assert!(res2.status.success()); +} + +pub fn stop_p2p_server(config: P2pTestConfig) { + let P2pTestConfig { + compose_path, + lifecycle_url: _, + lifecycle_retrying: _, + } = config; + println!("stoping p2p server and cleaning resources..."); + Command::new("docker") + .arg("compose") // Provide arguments if needed + .arg("-f") + .arg(compose_path) + .arg("down") + .output() + .expect("Failed to execute command"); +} diff --git a/tests/compose/http/compose.yaml b/tests/compose/http/compose.yaml new file mode 100644 index 00000000..119fc211 --- /dev/null +++ b/tests/compose/http/compose.yaml @@ -0,0 +1,40 @@ +x-environment: &commonEnvironment + MEGA_DB_MAX_CONNECTIONS: 16 + MEGA_DB_MIN_CONNECTIONS: 2 + MEGA_DB_SQLX_LOGGING: false + MEGA_OBJ_STORAGR_TYPE: "LOCAL" + MEGA_OBJ_LOCAL_PATH: "/tmp/.mega" + MEGA_BIG_OBJ_THRESHOLD_SIZE: 1024 + GIT_INTERNAL_DECODE_CACHE_SIZE: 1000 + GIT_INTERNAL_DECODE_STORAGE_BATCH_SIZE: 10000 + GIT_INTERNAL_DECODE_STORAGE_TQUEUE_SIZE: 10 + GIT_INTERNAL_DECODE_CACHE_TYEP: "lru" + REDIS_CONFIG: "redis://172.17.0.1:6379" + +services: + http_server: + build: + context: ../../../ + ports: + - "8000:8000" + environment: + <<: *commonEnvironment + MEGA_DB_POSTGRESQL_URL: "postgres://postgres:postgres@172.17.0.1:5433/mega" + command: service https --host 0.0.0.0 + depends_on: + - redis + - postgres + redis: + image: "redis:alpine" + ports: + - "6379:6379" + postgres: + image: postgres:latest + ports: + - "5433:5432" + volumes: + - ../../../sql/postgres/pg_20231106__init.sql:/docker-entrypoint-initdb.d/init.sql + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: mega diff --git a/tests/compose/mega_p2p/compose.yaml b/tests/compose/mega_p2p/compose.yaml index cf8d1fd3..32de97d1 100644 --- a/tests/compose/mega_p2p/compose.yaml +++ b/tests/compose/mega_p2p/compose.yaml @@ -11,7 +11,7 @@ x-environment: &commonEnvironment GIT_INTERNAL_DECODE_CACHE_TYEP: "lru" REDIS_CONFIG: "redis://172.17.0.1:6379" -services: +services: p2prelay: build: context: ../../../ @@ -22,8 +22,7 @@ services: depends_on: - redis node_a: - build: - context: ../../../ + image: mega_p2p-p2prelay ports: - "8300:8300" #p2p node port - "8301:8001" #p2p http api port @@ -31,20 +30,19 @@ services: environment: <<: *commonEnvironment MEGA_DB_POSTGRESQL_URL: "postgres://postgres:postgres@172.17.0.1:5433/mega" - command: service start http p2p --host 0.0.0.0 --p2p-port 8300 --bootstrap-node /ip4/172.17.0.1/tcp/8200 + command: service start http p2p --host 0.0.0.0 --p2p-port 8300 --bootstrap-node /ip4/172.17.0.1/tcp/8200 --secret-key 6b911fd37cdf5c81d4c0adb1ab7fa822ed253ab0ad9aa18d77257c88b29b7181 depends_on: - postgres_a - p2prelay node_b: - build: - context: ../../../ + image: mega_p2p-p2prelay ports: - "8400:8400" - "8401:8001" - environment: + environment: <<: *commonEnvironment MEGA_DB_POSTGRESQL_URL: "postgres://postgres:postgres@172.17.0.1:5434/mega" - command: service p2p --host 0.0.0.0 --p2p-port 8400 --bootstrap-node /ip4/172.17.0.1/tcp/8200 + command: service p2p --host 0.0.0.0 --p2p-port 8400 --bootstrap-node /ip4/172.17.0.1/tcp/8200 --secret-key 6b911fd37cdf5c81d4c0adb1ab7fa822ed253ab0ad9aa18d77257c88b29b718e depends_on: - postgres_b - p2prelay diff --git a/tests/integration_test.rs b/tests/integration_test.rs new file mode 100644 index 00000000..316d1eb6 --- /dev/null +++ b/tests/integration_test.rs @@ -0,0 +1,41 @@ +use crate::common_test::P2pTestConfig; +mod common_test; + +#[tokio::test] +#[ignore] +async fn test_peovide_and_clone_e2e() { + let init_config = P2pTestConfig { + compose_path: "tests/compose/mega_p2p/compose.yaml".to_string(), + lifecycle_url: "http://localhost:8301/api/v1/status".to_string(), + lifecycle_retrying: 5, + }; + common_test::init_p2p_server(init_config.clone()).await; + common_test::provide_data_before_test(); + test_mega_provide().await; + test_mega_clone().await; + test_mega_clone_obj().await; + common_test::stop_p2p_server(init_config); +} + +async fn test_mega_provide() { + let client = reqwest::Client::new(); + let resp = client + .put("http://localhost:8301/api/v1/mega/provide?repo_name=mega.git") + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); +} + +async fn test_mega_clone() { + //note that if secret of nodeA in compose file has been change, should also update the perrid in the below link + let resp = reqwest::get("http://localhost:8401/api/v1/mega/clone?mega_address=p2p://16Uiu2HAmCpKDLiX1NK6ULnYycq88jqaptNMRo1f4mRSu3VqHMry1/mega.git") + .await.unwrap(); + assert_eq!(resp.status(), 200); +} + +async fn test_mega_clone_obj() { + let resp = reqwest::get("http://localhost:8501/api/v1/mega/clone-obj?mega_address=p2p://16Uiu2HAmCpKDLiX1NK6ULnYycq88jqaptNMRo1f4mRSu3VqHMry1/mega.git") + .await.unwrap(); + assert_eq!(resp.status(), 200); +}