init e2e testing code

Cargo.toml

@@ -25,5 +25,9 @@ tokio = { version = "1.35.1", features = ["macros"] }
 clap = { version = "4.4.11", features = ["derive"] }
 serde = { version = "1.0", features = ["derive"] }
 
+
+[dev-dependencies]
+reqwest = "0.11.23"
+
 [build-dependencies]
 shadow-rs = "0.26.0"

docs/api.md

@@ -28,7 +28,7 @@ HTTP implement for git transfer data between two repositories
 
 The Git LFS client uses an HTTPS server to coordinate fetching and storing large binary objects separately from a Git server.
 
-1. 通过object id下载lfs协议需要的git对象
+1. Downloading the Git objects required by the LFS protocol using an object ID.
 
     ```bash
     GET **/objetcs/:object_id

docs/development.md

@@ -47,7 +47,6 @@ use std::str::FromStr;
 use std::sync::{Arc, Mutex};
 
 // 2. Third-Party Crates
-use async_trait::async_trait;
 use bytes::{BufMut, Bytes, BytesMut};
 use russh::server::{self, Auth, Msg, Session};
 use russh::{Channel, ChannelId};

gateway/Cargo.toml

@@ -16,7 +16,7 @@ common = {path = "../common"}
 storage = {path = "../storage"}
 entity = { path = "../storage/entity" }
 anyhow = "1.0.77"
-axum = "0.7.2"
+axum = "0.7.3"
 tower = "0.4.13"
 tower-http = { version = "0.5.0", features = ["cors", "trace"] }
 tokio = {version = "1.35", features = ["net"]}

git/Cargo.toml

@@ -44,7 +44,7 @@ async-recursion = "1.0"
 num_cpus = "1.16.0"
 dotenvy = "0.15.7"
 diffs = "0.5.1"
-sea-orm = { version = "0.12", features = [
+sea-orm = { version = "0.12.10", features = [
     "runtime-tokio-rustls",
     "macros",
     "mock",

p2p/Cargo.toml

@@ -25,11 +25,10 @@ async-std = { version = "1.12.0", features = ["attributes"] }
 libp2p = { version = "0.53", features = ["dcutr", "kad", "yamux", "noise", "identify", "macros", "relay", "tcp", "async-std", "rendezvous", "request-response", "cbor", "secp256k1"] }
 serde = { version = "1.0", features = ["derive"] }
 clap = { version = "4.4.11", features = ["derive"] }
-serde_json = "1.0"
-async-trait = "0.1"
+serde_json = "1.0.111"
+async-trait = "0.1.77"
 cbor4ii = { version = "0.3.1", features = ["serde1", "use_std"] }
 redis = { version = "0.23", features = ["tokio-comp"] }
 secp256k1 = { version = "0.27.0", features = ["serde", "bitcoin-hashes", "bitcoin-hashes-std", "rand"] }
-axum = "0.7.2"
-
+axum = "0.7.3"
 

p2p/src/lib.rs

@@ -5,19 +5,19 @@
 //!
 //!
 
-use storage::driver::database::storage::ObjectStorage;
-use git::protocol::{PackProtocol, Protocol};
 use std::path::PathBuf;
 use std::sync::Arc;
 use std::time::{SystemTime, UNIX_EPOCH};
 
-pub mod network;
-pub mod node;
-pub mod peer;
+use git::protocol::{PackProtocol, Protocol};
+use storage::driver::database::storage::ObjectStorage;
+
 pub mod cbor;
 pub mod internal;
-
+pub mod network;
+pub mod node;
 pub mod nostr;
+pub mod peer;
 
 fn get_pack_protocol(path: &str, storage: Arc<dyn ObjectStorage>) -> PackProtocol {
     let path = del_ends_str(path, ".git");

p2p/src/network/behaviour.rs

@@ -5,9 +5,9 @@ use libp2p::swarm::NetworkBehaviour;
 use libp2p::{dcutr, identify, relay, rendezvous, request_response};
 use serde::{Deserialize, Serialize};
 use std::collections::HashSet;
+
 use crate::cbor;
 use crate::nostr::{NostrReq, NostrRes};
-
 #[derive(NetworkBehaviour)]
 #[behaviour(to_swarm = "Event")]
 pub struct Behaviour {
@@ -47,7 +47,6 @@ pub struct GitObjectReq(pub String, pub Vec<String>);
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct GitObjectRes(pub Vec<objects::Model>);
 
-
 #[derive(Debug)]
 #[allow(clippy::large_enum_variant)]
 pub enum Event {

p2p/src/node/client_http.rs

@@ -30,8 +30,9 @@ pub async fn server(sender: mpsc::Sender<String>) {
         .nest(
             "/api/v1",
             Router::new()
-                .nest("/mega/", mega_routers())
-                .nest("/nostr", nostr_routers()),
+                .nest("/mega", mega_routers())
+                .nest("/nostr", nostr_routers())
+                .route("/status", get(life_cycle_check)),
         )
         // .layer(TraceLayer::new_for_http())
         .with_state(state);
@@ -53,6 +54,10 @@ pub fn mega_routers() -> Router<P2pNodeState> {
         .route("/pull-object", get(mega_pull_obj))
 }
 
+async fn life_cycle_check() -> Result<impl IntoResponse, (StatusCode, String)> {
+    Ok(Json("ok"))
+}
+
 async fn mega_provide(
     Query(query): Query<HashMap<String, String>>,
     state: State<P2pNodeState>,
@@ -160,3 +165,57 @@ async fn nostr_event_issue(
     state.0.sender.clone().try_send(line).unwrap();
     Ok(Json("ok"))
 }
+
+#[cfg(test)]
+mod test {
+    use std::collections::HashMap;
+
+    use async_std::stream::StreamExt;
+    use axum::{extract::Query, http::Uri};
+    use futures::channel::mpsc;
+
+    use crate::node::client_http::{
+        mega_clone, mega_clone_obj, mega_pull, mega_pull_obj, P2pNodeState,
+    };
+    use crate::node::client_http::{mega_provide, mega_search};
+
+    #[tokio::test]
+    async fn test_mega_routers() {
+        let query: Query<HashMap<String, String>> = Query::try_from_uri(
+            &"http://localhost:8001/api/v1/mega/provide?repo_name=reponame.git"
+                .parse::<Uri>()
+                .unwrap(),
+        )
+        .unwrap();
+
+        let addr_query: Query<HashMap<String, String>> = Query::try_from_uri(
+            &"http://localhost:8001/api/v1/mega/clone?mega_address=p2p://peer_id/reponame.git"
+                .parse::<Uri>()
+                .unwrap(),
+        )
+        .unwrap();
+
+        let (tx, mut rx) = mpsc::channel::<String>(64);
+        let s = P2pNodeState { sender: tx };
+        let state = axum::extract::State(s);
+        let _ = mega_provide(query.clone(), state.clone()).await;
+        let _ = mega_search(query.clone(), state.clone()).await;
+        let _ = mega_clone(addr_query.clone(), state.clone()).await;
+        let _ = mega_clone_obj(query.clone(), state.clone()).await;
+        let _ = mega_pull(addr_query.clone(), state.clone()).await;
+        let _ = mega_pull_obj(query.clone(), state.clone()).await;
+
+        assert_eq!(rx.next().await.unwrap(), "mega provide reponame.git");
+        assert_eq!(rx.next().await.unwrap(), "mega search reponame.git");
+        assert_eq!(
+            rx.next().await.unwrap(),
+            "mega clone p2p://peer_id/reponame.git"
+        );
+        assert_eq!(rx.next().await.unwrap(), "mega clone-object reponame.git");
+        assert_eq!(
+            rx.next().await.unwrap(),
+            "mega pull p2p://peer_id/reponame.git"
+        );
+        assert_eq!(rx.next().await.unwrap(), "mega pull-object reponame.git");
+    }
+}

storage/Cargo.toml

@@ -13,16 +13,16 @@ path = "src/lib.rs"
 [dependencies]
 common = {path = "../common"}
 entity = {path = "./entity"}
-anyhow = "1.0"
-async-trait = "0.1"
+anyhow = "1.0.79"
+async-trait = "0.1.77"
 tracing = "0.1.40"
 idgenerator = "2.0.0"
-chrono = "0.4"
-sha256 = "1.4"
-serde = "1.0"
-serde_json = "1.0"
-futures = "0.3"
-sea-orm = {version = "0.12", features = [
+chrono = "0.4.31"
+sha256 = "1.5"
+serde = "1.0.195"
+serde_json = "1.0.111"
+futures = "0.3.30"
+sea-orm = {version = "0.12.10", features = [
     "sqlx-postgres",
     "sqlx-mysql",
     "runtime-tokio-rustls",
@@ -31,8 +31,8 @@ sea-orm = {version = "0.12", features = [
 aws-config = {version = "1.1.1", features = ["behavior-version-latest"]}
 aws-sdk-s3 = "1.11.0"
 aws-smithy-types = "1.1.1"
-thiserror = "1.0.52"
+thiserror = "1.0.56"
 bytes = "1.5.0"
 
 [dev-dependencies]
-tokio = { version = "1.35.0", features = ["macros"] }
+tokio = { version = "1.35.1", features = ["macros"] }

storage/src/driver/database/mysql_storage.rs

@@ -4,24 +4,20 @@
 //!
 
 use async_trait::async_trait;
-
-use entity::commit;
-
-use entity::objects;
-use entity::refs;
-
 use sea_orm::DatabaseBackend;
 use sea_orm::DatabaseConnection;
 use sea_orm::DatabaseTransaction;
 use sea_orm::EntityTrait;
-
 use sea_orm::Statement;
 use sea_orm::TryIntoModel;
 
-use crate::driver::database::storage::ObjectStorage;
+use common::errors::MegaError;
+use entity::commit;
+use entity::objects;
+use entity::refs;
 
 use crate::driver::database::storage::batch_save_model;
-use common::errors::MegaError;
+use crate::driver::database::storage::ObjectStorage;
 
 #[derive(Debug, Default)]
 pub struct MysqlStorage {

storage/src/driver/file_storage/mod.rs

@@ -6,11 +6,11 @@ use std::{
 
 use async_trait::async_trait;
 use bytes::Bytes;
+
 use common::errors::MegaError;
 
 use crate::driver::file_storage::local_storage::LocalStorage;
-
-use self::remote_storage::RemoteStorage;
+use crate::driver::file_storage::remote_storage::RemoteStorage;
 
 pub mod local_storage;
 pub mod remote_storage;

tests/common_test/mod.rs

@@ -0,0 +1,80 @@
+use std::{
+    process::Command,
+    thread::{self, sleep},
+    time::Duration,
+};
+
+#[derive(Clone)]
+pub struct P2pTestConfig {
+    pub compose_path: String,
+    pub lifecycle_url: String,
+    pub lifecycle_retrying: u64,
+}
+pub async fn init_p2p_server(config: P2pTestConfig) {
+    // docker compose -f tests/compose/mega_p2p/compose.yaml up --build
+    let P2pTestConfig {
+        compose_path,
+        lifecycle_url,
+        lifecycle_retrying,
+    } = config;
+    thread::spawn(move || {
+        let mut child = Command::new("docker")
+            .arg("compose") // Provide arguments if needed
+            .arg("-f")
+            .arg(compose_path)
+            .arg("up")
+            .stdin(std::process::Stdio::piped()) // Pass the standard input stream as an argument
+            .stdout(std::process::Stdio::piped())
+            .spawn()
+            .expect("Failed to execute command");
+        // Wait for the child process to finish and get the result
+        let _ = child.wait().expect("Failed to wait for child process");
+    });
+
+    loop {
+        let resp = reqwest::get(&lifecycle_url).await.unwrap();
+        if resp.status() == 200 {
+            break;
+        } else {
+            println!(
+                "lifecycle check failed, retrying in {} secs ...",
+                lifecycle_retrying
+            );
+        }
+        sleep(Duration::from_secs(lifecycle_retrying));
+    }
+}
+
+pub fn provide_data_before_test() {
+    let res = Command::new("git")
+        .arg("remote")
+        .arg("set-url")
+        .arg("local")
+        .arg("http://localhost:8000/projects/mega.git")
+        .output()
+        .expect("Failed to execute command");
+    assert!(res.status.success());
+    let res2 = Command::new("git")
+        .arg("push")
+        .arg("local")
+        .arg("main")
+        .output()
+        .expect("Failed to execute command");
+    assert!(res2.status.success());
+}
+
+pub fn stop_p2p_server(config: P2pTestConfig) {
+    let P2pTestConfig {
+        compose_path,
+        lifecycle_url: _,
+        lifecycle_retrying: _,
+    } = config;
+    println!("stoping p2p server and cleaning resources...");
+    Command::new("docker")
+        .arg("compose") // Provide arguments if needed
+        .arg("-f")
+        .arg(compose_path)
+        .arg("down")
+        .output()
+        .expect("Failed to execute command");
+}

tests/compose/http/compose.yaml

@@ -0,0 +1,40 @@
+x-environment: &commonEnvironment
+  MEGA_DB_MAX_CONNECTIONS: 16
+  MEGA_DB_MIN_CONNECTIONS: 2
+  MEGA_DB_SQLX_LOGGING: false
+  MEGA_OBJ_STORAGR_TYPE: "LOCAL"
+  MEGA_OBJ_LOCAL_PATH: "/tmp/.mega"
+  MEGA_BIG_OBJ_THRESHOLD_SIZE: 1024
+  GIT_INTERNAL_DECODE_CACHE_SIZE: 1000
+  GIT_INTERNAL_DECODE_STORAGE_BATCH_SIZE: 10000
+  GIT_INTERNAL_DECODE_STORAGE_TQUEUE_SIZE: 10
+  GIT_INTERNAL_DECODE_CACHE_TYEP: "lru"
+  REDIS_CONFIG: "redis://172.17.0.1:6379"
+
+services: 
+  http_server:
+    build:
+      context: ../../../
+    ports:
+      - "8000:8000"
+    environment:
+      <<: *commonEnvironment
+      MEGA_DB_POSTGRESQL_URL: "postgres://postgres:[email protected]:5433/mega"
+    command: service https --host 0.0.0.0
+    depends_on:
+      - redis
+      - postgres
+  redis:
+    image: "redis:alpine"
+    ports:
+      - "6379:6379"
+  postgres:
+    image: postgres:latest
+    ports:
+      - "5433:5432"
+    volumes:
+      - ../../../sql/postgres/pg_20231106__init.sql:/docker-entrypoint-initdb.d/init.sql
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: mega