From ff1f13f6f74c0e5631260e82f68a2663d3034c4f Mon Sep 17 00:00:00 2001 From: Lorenzo Leonardo Date: Fri, 27 Oct 2023 08:21:54 +0800 Subject: [PATCH] added resume download tests and example --- Cargo.lock | 68 ++++++++++++++++++++++++++++++++++++- Cargo.toml | 5 +-- README.md | 38 +++++++++++++++++++++ examples/resume_download.rs | 35 +++++++++++++++++++ src/lib.rs | 39 +++++++++++++++++++++ src/test/download.rs | 42 ++++++++++++++++++++++- src/test/test_setup.rs | 59 +++++++++++++++++++++++++++++--- 7 files changed, 278 insertions(+), 8 deletions(-) create mode 100644 examples/resume_download.rs diff --git a/Cargo.lock b/Cargo.lock index eeccbdf..7d8b7d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -158,7 +158,7 @@ dependencies = [ [[package]] name = "curl-http-client" -version = "0.1.1" +version = "0.1.2" dependencies = [ "async-curl", "curl", @@ -166,6 +166,7 @@ dependencies = [ "futures", "http", "tempdir", + "test-case", "thiserror", "tokio", "url", @@ -660,6 +661,30 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro2" version = "1.0.69" @@ -936,6 +961,41 @@ dependencies = [ "remove_dir_all", ] +[[package]] +name = "test-case" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8f1e820b7f1d95a0cdbf97a5df9de10e1be731983ab943e56703ac1b8e9d425" +dependencies = [ + "test-case-macros", +] + +[[package]] +name = "test-case-core" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54c25e2cb8f5fcd7318157634e8838aa6f7e4715c96637f969fabaccd1ef5462" +dependencies = [ + "cfg-if", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.38", +] + +[[package]] +name = "test-case-macros" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37cfd7bbc88a0104e304229fba519bdc45501a30b760fb72240342f1289ad257" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.38", + "test-case-core", +] + [[package]] name = "thiserror" version = "1.0.50" @@ -1082,6 +1142,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + [[package]] name = "waker-fn" version = "1.1.1" diff --git a/Cargo.toml b/Cargo.toml index 2cc53fc..aa72f29 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ tokio = { version = "1.33", features = ["rt"] } url = "2.4" [dev-dependencies] -wiremock = "=0.5.15" -tempdir = "0.3" futures = "0.3" +tempdir = "0.3" +test-case = "3.2" +wiremock = "=0.5.15" diff --git a/README.md b/README.md index 60c9a33..9f27882 100644 --- a/README.md +++ b/README.md @@ -183,3 +183,41 @@ async fn main() { } } ``` + +## Resume Downloading a File +```rust +use std::fs; +use std::path::PathBuf; + +use async_curl::async_curl::AsyncCurl; +use curl_http_client::{ + collector::{Collector, FileInfo}, + http_client::{BytesOffset, HttpClient}, + request::HttpRequest, +}; +use http::{HeaderMap, Method}; +use url::Url; + +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<(), Box> { + let curl = AsyncCurl::new(); + let save_to = PathBuf::from(""); + let collector = Collector::File(FileInfo::path(save_to.clone())); + + let partial_download_file_size = fs::metadata(save_to.as_path())?.len() as usize; + let request = HttpRequest { + url: Url::parse("")?, + method: Method::GET, + headers: HeaderMap::new(), + body: None, + }; + + let response = HttpClient::new(curl, collector) + .resume_from(BytesOffset::from(partial_download_file_size))? + .request(request)? + .perform() + .await?; + println!("Response: {:?}", response); + Ok(()) +} +``` \ No newline at end of file diff --git a/examples/resume_download.rs b/examples/resume_download.rs new file mode 100644 index 0000000..a5d5054 --- /dev/null +++ b/examples/resume_download.rs @@ -0,0 +1,35 @@ +use std::fs; +use std::path::PathBuf; + +use async_curl::async_curl::AsyncCurl; +use curl_http_client::{ + collector::{Collector, FileInfo}, + http_client::{BytesOffset, HttpClient}, + request::HttpRequest, +}; +use http::{HeaderMap, Method}; +use url::Url; + +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<(), Box> { + let curl = AsyncCurl::new(); + let save_to = PathBuf::from(""); + let collector = Collector::File(FileInfo::path(save_to.clone())); + + let partial_download_file_size = fs::metadata(save_to.as_path())?.len() as usize; + let request = HttpRequest { + url: Url::parse("")?, + method: Method::GET, + headers: HeaderMap::new(), + body: None, + }; + + let response = HttpClient::new(curl, collector) + .resume_from(BytesOffset::from(partial_download_file_size))? + .request(request)? + .perform() + .await?; + + println!("Response: {:?}", response); + Ok(()) +} diff --git a/src/lib.rs b/src/lib.rs index 39fdd74..9b4b82a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -183,6 +183,45 @@ //! } //! ``` //! +//! # Downloading a File +//! ```rust,no_run +//! use std::fs; +//! use std::path::PathBuf; +//! +//! use async_curl::async_curl::AsyncCurl; +//! use curl_http_client::{ +//! collector::{Collector, FileInfo}, +//! http_client::{BytesOffset, HttpClient}, +//! request::HttpRequest, +//! }; +//! use http::{HeaderMap, Method}; +//! use url::Url; +//! +//! #[tokio::main(flavor = "current_thread")] +//! async fn main() -> Result<(), Box> { +//! let curl = AsyncCurl::new(); +//! let save_to = PathBuf::from(""); +//! let collector = Collector::File(FileInfo::path(save_to.clone())); +//! +//! let partial_download_file_size = fs::metadata(save_to.as_path())?.len() as usize; +//! let request = HttpRequest { +//! url: Url::parse("")?, +//! method: Method::GET, +//! headers: HeaderMap::new(), +//! body: None, +//! }; +//! +//! let response = HttpClient::new(curl, collector) +//! .resume_from(BytesOffset::from(partial_download_file_size))? +//! .request(request)? +//! .perform() +//! .await?; +//! +//! println!("Response: {:?}", response); +//! Ok(()) +//! } +//! ``` +//! pub mod collector; pub mod error; pub mod http_client; diff --git a/src/test/download.rs b/src/test/download.rs index 8744bc8..f4da91f 100644 --- a/src/test/download.rs +++ b/src/test/download.rs @@ -2,10 +2,11 @@ use std::fs; use async_curl::async_curl::AsyncCurl; use http::{HeaderMap, Method, StatusCode}; +use test_case::test_case; use url::Url; use crate::collector::{Collector, FileInfo}; -use crate::http_client::{BytesPerSec, HttpClient}; +use crate::http_client::{BytesOffset, BytesPerSec, HttpClient}; use crate::request::HttpRequest; use crate::test::test_setup::{setup_test_environment, MockResponder, ResponderType}; @@ -66,3 +67,42 @@ async fn test_download_with_speed_control() { assert_eq!(response.body, None); assert_eq!(fs::read(save_to).unwrap(), include_bytes!("sample.jpg")); } + +#[test_case(4500, StatusCode::PARTIAL_CONTENT; "Offset 4500 bytes")] +#[test_case(0, StatusCode::OK ; "Offset 0 bytes")] +#[test_case(include_bytes!("sample.jpg").len(), StatusCode::PARTIAL_CONTENT ; "Offset max bytes")] +#[tokio::test] +async fn test_resume_download(offset: usize, expected_status_code: StatusCode) { + let responder = MockResponder::new(ResponderType::File); + let (server, tempdir) = setup_test_environment(responder).await; + let target_url = Url::parse(format!("{}/test", server.uri()).as_str()).unwrap(); + + let save_to = tempdir.path().join("downloaded_file.jpg"); + + let partial_saved_file = include_bytes!("sample.jpg"); + fs::write(save_to.as_path(), &partial_saved_file[0..offset]).unwrap(); + + let partial_file_size = fs::metadata(save_to.as_path()).unwrap().len() as usize; + + let curl = AsyncCurl::new(); + let collector = Collector::File(FileInfo::path(save_to.clone())); + let request = HttpRequest { + url: target_url, + method: Method::GET, + headers: HeaderMap::new(), + body: None, + }; + let response = HttpClient::new(curl, collector) + .resume_from(BytesOffset::from(partial_file_size)) + .unwrap() + .request(request) + .unwrap() + .perform() + .await + .unwrap(); + + println!("Response: {:?}", response); + assert_eq!(response.status_code, expected_status_code); + assert_eq!(response.body, None); + assert_eq!(fs::read(save_to).unwrap(), include_bytes!("sample.jpg")); +} diff --git a/src/test/test_setup.rs b/src/test/test_setup.rs index 832f6be..fd5f4bc 100644 --- a/src/test/test_setup.rs +++ b/src/test/test_setup.rs @@ -1,7 +1,11 @@ +use std::str::FromStr; + use http::StatusCode; use tempdir::TempDir; use wiremock::{ - http::Method, matchers::path, Mock, MockServer, Request, Respond, ResponseTemplate, + http::{HeaderName, HeaderValue, HeaderValues, Method}, + matchers::path, + Mock, MockServer, Request, Respond, ResponseTemplate, }; pub enum ResponderType { @@ -20,12 +24,45 @@ impl MockResponder { impl Respond for MockResponder { fn respond(&self, request: &Request) -> ResponseTemplate { - //println!("Request: {:?}", request); match request.method { Method::Get => match &self.responder { ResponderType::File => { - let contents = include_bytes!("sample.jpg"); - ResponseTemplate::new(StatusCode::OK).set_body_bytes(contents.as_slice()) + let mock_file = include_bytes!("sample.jpg"); + let header_name = HeaderName::from_str("range").unwrap(); + let total_file_size = mock_file.len(); + println!("Request: {:?}", request); + if let Some(value) = request.headers.get(&header_name) { + let offset = parse_range(value).unwrap() as usize; + println!("Offset: {}", offset); + + let content_length = format!("{}", total_file_size - offset); + println!("Content-Length: {}", content_length); + let content_range = format!( + "bytes {}-{}/{}", + offset, + total_file_size - 1, + total_file_size + ); + println!("Content-Range: {}", content_range); + + ResponseTemplate::new(StatusCode::PARTIAL_CONTENT) + .append_header( + HeaderName::from_str("Content-Range").unwrap(), + HeaderValue::from_str(content_range.as_str()).unwrap(), + ) + .append_header( + HeaderName::from_str("Content-Length").unwrap(), + HeaderValue::from_str(content_length.as_str()).unwrap(), + ) + .append_header( + HeaderName::from_str("Accept-Ranges").unwrap(), + HeaderValue::from_str("bytes").unwrap(), + ) + .set_body_bytes(&mock_file[offset..]) + } else { + let contents = include_bytes!("sample.jpg"); + ResponseTemplate::new(StatusCode::OK).set_body_bytes(contents.as_slice()) + } } ResponderType::Body(body) => { ResponseTemplate::new(StatusCode::OK).set_body_bytes(body.as_slice()) @@ -55,6 +92,20 @@ impl Respond for MockResponder { } } +fn parse_range(input: &HeaderValues) -> Option { + let input = input.to_string(); + if let Some(start_pos) = input.find('=') { + if let Some(end_pos) = input.rfind('-') { + let numeric_value = &input[start_pos + 1..end_pos]; + numeric_value.parse::().ok() + } else { + None + } + } else { + None + } +} + pub async fn setup_test_environment(responder: MockResponder) -> (MockServer, TempDir) { let mock_server = MockServer::start().await; let tempdir = TempDir::new_in("./", "test").unwrap();