Merge pull request #304 from benjamin-747/main

add http integration test code

Cargo.toml

@@ -27,7 +27,13 @@ serde = { version = "1.0", features = ["derive"] }
 
 
 [dev-dependencies]
-reqwest = "0.11.23"
+reqwest = {version = "0.11.23", features = ["stream", "multipart"]}
+env_logger = "0.10.1"
+futures-util = "0.3.30"
+tokio-util = "0.7.10"
+go-defer = "0.1.0"
+bytes = "1.5.0"
+
 
 [build-dependencies]
 shadow-rs = "0.26.0"

gateway/src/api_service/obj_service.rs

@@ -7,11 +7,13 @@ use axum::{http::StatusCode, response::Response};
 use git::internal::object::commit::Commit;
 use git::internal::object::tree::Tree;
 use git::internal::object::ObjectT;
+use git::internal::pack::counter::GitTypeCounter;
 use storage::driver::database::storage::ObjectStorage;
 
-use crate::model::object_detail::{BlobObjects, Directories, Item};
+use crate::model::objects::{BlobObjects, Directories, Item};
 use crate::model::query::DirectoryQuery;
 
+#[derive(Clone)]
 pub struct ObjectService {
     pub storage: Arc<dyn ObjectStorage>,
 }
@@ -140,7 +142,7 @@ impl ObjectService {
         for item in &mut items {
             let related_c_id = item.commit_id.clone().unwrap();
             let commit = related_c_map.get(&related_c_id).unwrap();
-            item.commit_msg = Some(remove_useless_str(
+            item.commit_msg = Some(utils::remove_useless_str(
                 commit.message.clone(),
                 SIGNATURE_END.to_owned(),
             ));
@@ -171,14 +173,50 @@ impl ObjectService {
             .unwrap();
         Ok(res)
     }
+
+    pub async fn count_object_num(
+        &self,
+        repo_path: &str,
+    ) -> Result<Json<GitTypeCounter>, (StatusCode, String)> {
+        let query_res = self
+            .storage
+            .count_obj_from_commit_and_node(repo_path)
+            .await
+            .unwrap();
+        let tree = query_res
+            .iter()
+            .find(|x| x.node_type == "tree")
+            .map(|x| x.count)
+            .unwrap()
+            .try_into()
+            .unwrap();
+        let blob = query_res
+            .iter()
+            .find(|x| x.node_type == "blob")
+            .map(|x| x.count)
+            .unwrap()
+            .try_into()
+            .unwrap();
+        let counter = GitTypeCounter {
+            commit: 0,
+            tree,
+            blob,
+            tag: 0,
+            ofs_delta: 0,
+            ref_delta: 0,
+        };
+        Ok(Json(counter))
+    }
 }
 
-fn remove_useless_str(content: String, remove_str: String) -> String {
-    if let Some(index) = content.find(&remove_str) {
-        let filtered_text = &content[index + remove_str.len()..].replace('\n', "");
-        let truncated_text = filtered_text.chars().take(50).collect::<String>();
-        truncated_text.to_owned()
-    } else {
-        "".to_owned()
+pub mod utils {
+    pub(crate) fn remove_useless_str(content: String, remove_str: String) -> String {
+        if let Some(index) = content.find(&remove_str) {
+            let filtered_text = &content[index + remove_str.len()..].replace('\n', "");
+            let truncated_text = filtered_text.chars().take(50).collect::<String>();
+            truncated_text.to_owned()
+        } else {
+            "".to_owned()
+        }
     }
 }

gateway/src/api_service/router.rs

@@ -7,53 +7,62 @@ use axum::{
     routing::get,
     Json, Router,
 };
+use git::internal::pack::counter::GitTypeCounter;
 
 use crate::{
     api_service::obj_service::ObjectService,
     model::{
-        object_detail::{BlobObjects, Directories},
+        objects::{BlobObjects, Directories},
         query::DirectoryQuery,
     },
 };
 
-use crate::AppState;
+#[derive(Clone)]
+pub struct ApiServiceState {
+    pub object_service: ObjectService,
+}
 
-pub fn routers<S>(state: AppState) -> Router<S> {
+pub fn routers<S>(state: ApiServiceState) -> Router<S> {
     Router::new()
         .route("/blob", get(get_blob_object))
         .route("/tree", get(get_directories))
         .route("/object", get(get_origin_object))
+        .route("/status", get(life_cycle_check))
+        .route("/count-nums", get(get_count_nums))
         .with_state(state)
 }
 
+async fn life_cycle_check() -> Result<impl IntoResponse, (StatusCode, String)> {
+    Ok(Json("http ready"))
+}
+
+async fn get_count_nums(
+    Query(query): Query<HashMap<String, String>>,
+    state: State<ApiServiceState>,
+) -> Result<Json<GitTypeCounter>, (StatusCode, String)> {
+    let repo_path = query.get("repo_path").unwrap();
+    state.object_service.count_object_num(repo_path).await
+}
+
 async fn get_blob_object(
     Query(query): Query<HashMap<String, String>>,
-    state: State<AppState>,
+    state: State<ApiServiceState>,
 ) -> Result<Json<BlobObjects>, (StatusCode, String)> {
     let object_id = query.get("object_id").unwrap();
-    let object_service = ObjectService {
-        storage: state.storage.clone(),
-    };
-    object_service.get_blob_objects(object_id).await
+    state.object_service.get_blob_objects(object_id).await
 }
 
 async fn get_directories(
     Query(query): Query<DirectoryQuery>,
-    state: State<AppState>,
+    state: State<ApiServiceState>,
 ) -> Result<Json<Directories>, (StatusCode, String)> {
-    let object_service = ObjectService {
-        storage: state.storage.clone(),
-    };
-    object_service.get_directories(query).await
+    state.object_service.get_directories(query).await
 }
 
 async fn get_origin_object(
     Query(query): Query<HashMap<String, String>>,
-    state: State<AppState>,
+    state: State<ApiServiceState>,
 ) -> Result<impl IntoResponse, (StatusCode, String)> {
     let object_id = query.get("object_id").unwrap();
-    let object_service = ObjectService {
-        storage: state.storage.clone(),
-    };
-    object_service.get_objects_data(object_id).await
+    state.object_service.get_objects_data(object_id).await
 }

gateway/src/https_server.rs

@@ -29,6 +29,8 @@ use storage::driver::database;
 use storage::driver::database::storage::ObjectStorage;
 use tower_http::trace::TraceLayer;
 
+use crate::api_service::obj_service::ObjectService;
+use crate::api_service::router::ApiServiceState;
 use crate::{api_service, git_protocol, lfs};
 
 #[derive(Args, Clone, Debug)]
@@ -92,8 +94,15 @@ pub async fn start_server(options: &HttpOptions) {
         storage: database::init(data_source).await,
         options: options.to_owned(),
     };
+
+    let api_state = ApiServiceState {
+        object_service: ObjectService {
+            storage: state.storage.clone(),
+        }
+    };
+
     let app = Router::new()
-        .nest("/api/v1", api_service::router::routers(state.clone()))
+        .nest("/api/v1", api_service::router::routers(api_state))
         .route(
             "/*path",
             get(get_method_router)

gateway/src/model/mod.rs

@@ -1,2 +1,2 @@
-pub mod object_detail;
+pub mod objects;
 pub mod query;

git/src/internal/pack/counter.rs

@@ -1,13 +1,15 @@
 use std::fmt::Display;
+
+use serde::{Serialize, Deserialize};
 /// A Counter for counting git object types
-#[derive(Default,Clone, Copy)]
+#[derive(Default,Clone, Copy, Serialize, Deserialize)]
 pub struct GitTypeCounter{
-    commit:usize,
-    tree:usize,
-    blob:usize,
-    tag:usize,
-    ofs_delta:usize,
-    ref_delta:usize,
+    pub commit:usize,
+    pub tree:usize,
+    pub blob:usize,
+    pub tag:usize,
+    pub ofs_delta:usize,
+    pub ref_delta:usize,
 
 }
 impl GitTypeCounter {

git/src/internal/pack/mod.rs

@@ -14,7 +14,7 @@ use crate::hash::Hash;
 use crate::internal::object::ObjectT;
 
 mod cache;
-mod counter;
+pub mod counter;
 mod cqueue;
 pub mod decode;
 pub mod delta;

git/src/protocol/pack.rs

@@ -391,14 +391,14 @@ pub fn read_pkt_line(bytes: &mut Bytes) -> (usize, Bytes) {
         return (0, Bytes::new());
     }
     let pkt_length = bytes.copy_to_bytes(4);
-    let pkt_length =
-        usize::from_str_radix(&String::from_utf8(pkt_length.to_vec()).unwrap(), 16).unwrap();
-
+    let pkt_length = usize::from_str_radix(&String::from_utf8(pkt_length.to_vec()).unwrap(), 16)
+        .unwrap_or_else(|_| panic!("{:?} is not a valid digit?", pkt_length));
     if pkt_length == 0 {
         return (0, Bytes::new());
     }
     // this operation will change the original bytes
     let pkt_line = bytes.copy_to_bytes(pkt_length - 4);
+    tracing::debug!("pkt line: {:?}", pkt_line);
 
     (pkt_length, pkt_line)
 }

p2p/src/node/client_http.rs

@@ -55,7 +55,7 @@ pub fn mega_routers() -> Router<P2pNodeState> {
 }
 
 async fn life_cycle_check() -> Result<impl IntoResponse, (StatusCode, String)> {
-    Ok(Json("ok"))
+    Ok(Json("p2p node http ready"))
 }
 
 async fn mega_provide(