diff --git a/Cargo.lock b/Cargo.lock index 27b3364..4ae9c2a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -88,12 +88,6 @@ dependencies = [ "bytemuck", ] -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - [[package]] name = "autocfg" version = "1.3.0" @@ -292,16 +286,6 @@ dependencies = [ "url", ] -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -441,15 +425,6 @@ dependencies = [ "serde", ] -[[package]] -name = "encoding_rs" -version = "0.8.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" -dependencies = [ - "cfg-if", -] - [[package]] name = "equivalent" version = "1.0.1" @@ -545,21 +520,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.1" @@ -668,25 +628,6 @@ version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" -[[package]] -name = "h2" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "hashbrown" version = "0.14.5" @@ -806,7 +747,6 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2", "http", "http-body", "httparse", @@ -832,22 +772,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", -] - -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes", - "http-body-util", - "hyper", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", + "webpki-roots", ] [[package]] @@ -1019,12 +944,12 @@ checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "load_config" -version = "1.0.0" -source = "git+https://github.com/9-FS/load_config?tag=1.0.0#aeafea6bbe56d755ff68130ea6a403d6eebb11db" +version = "1.1.0" +source = "git+https://github.com/9-FS/load_config?tag=1.1.0#5ba280fa9333b5c18e6ece44612ea3b2386ee855" dependencies = [ "figment", - "log", "serde", + "thiserror", "toml", ] @@ -1109,31 +1034,13 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "native-tls" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "nhentai_archivist" -version = "3.0.3" +version = "3.1.0" dependencies = [ "chrono", "load_config", "log", - "openssl", "reqwest", "scaler", "serde", @@ -1225,60 +1132,6 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" -[[package]] -name = "openssl" -version = "0.10.66" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1" -dependencies = [ - "bitflags", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "openssl-probe" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" - -[[package]] -name = "openssl-src" -version = "300.3.2+3.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a211a18d945ef7e648cc6e0058f4c548ee46aab922ea203e0d30e966ea23647b" -dependencies = [ - "cc", -] - -[[package]] -name = "openssl-sys" -version = "0.9.103" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6" -dependencies = [ - "cc", - "libc", - "openssl-src", - "pkg-config", - "vcpkg", -] - [[package]] name = "parking" version = "2.2.1" @@ -1474,6 +1327,54 @@ dependencies = [ "psl-types", ] +[[package]] +name = "quinn" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" +dependencies = [ + "bytes", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "quinn-proto" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" +dependencies = [ + "bytes", + "rand", + "ring", + "rustc-hash", + "rustls", + "slab", + "thiserror", + "tinyvec", + "tracing", +] + +[[package]] +name = "quinn-udp" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b" +dependencies = [ + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.59.0", +] + [[package]] name = "quote" version = "1.0.37" @@ -1515,9 +1416,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" +checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853" dependencies = [ "bitflags", ] @@ -1532,38 +1433,37 @@ dependencies = [ "bytes", "cookie", "cookie_store", - "encoding_rs", "futures-core", "futures-util", - "h2", "http", "http-body", "http-body-util", "hyper", "hyper-rustls", - "hyper-tls", "hyper-util", "ipnet", "js-sys", "log", "mime", - "native-tls", "once_cell", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", "rustls-pemfile", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", - "system-configuration", "tokio", - "tokio-native-tls", + "tokio-rustls", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", + "webpki-roots", "windows-registry", ] @@ -1608,11 +1508,17 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" + [[package]] name = "rustix" -version = "0.38.36" +version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f55e80d50763938498dd5ebb18647174e0c76dc38c5505294bb224624f30f36" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ "bitflags", "errno", @@ -1623,9 +1529,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.12" +version = "0.23.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" dependencies = [ "once_cell", "ring", @@ -1677,44 +1583,12 @@ dependencies = [ "log", ] -[[package]] -name = "schannel" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" -dependencies = [ - "windows-sys 0.59.0", -] - [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "serde" version = "1.0.210" @@ -2130,27 +2004,6 @@ dependencies = [ "futures-core", ] -[[package]] -name = "system-configuration" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" -dependencies = [ - "bitflags", - "core-foundation", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "tempfile" version = "3.12.0" @@ -2245,16 +2098,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.0" @@ -2277,19 +2120,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-util" -version = "0.7.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", -] - [[package]] name = "toml" version = "0.8.19" @@ -2412,9 +2242,9 @@ checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-normalization" diff --git a/Cargo.toml b/Cargo.toml index dbc6db9..9d31100 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,18 +9,18 @@ license = "MIT" name = "nhentai_archivist" readme = "readme.md" repository = "https://github.com/9-FS/nhentai_archivist" -version = "3.0.3" +version = "3.1.0" [dependencies] chrono = { version = "^0.4.0", features = ["serde"] } -load_config = { git = "https://github.com/9-FS/load_config", tag = "1.0.0", features = [ +load_config = { git = "https://github.com/9-FS/load_config", tag = "1.1.0", features = [ "toml_file", ] } log = "^0.4.0" -openssl = { version = "^0.10.0", features = [ - "vendored", -] } # because otherwise some wild fuckywuckys during cross compilation -reqwest = { version = "^0.12.0", features = ["cookies"] } +reqwest = { version = "^0.12.0", default-features = false, features = [ + "cookies", + "rustls-tls", +] } scaler = "^1.0.0" serde = { version = "^1.0.0", features = ["derive"] } serde-xml-rs = "^0.6.0" diff --git a/docker-compose.yaml b/docker-compose.yaml index c6ce0e8..32cb3fd 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -3,7 +3,7 @@ version: "3" services: nhentai_archivist: container_name: "nhentai_archivist" - image: "ghcr.io/9-fs/nhentai_archivist:3.0.3" + image: "ghcr.io/9-fs/nhentai_archivist:3.1.0" environment: HOST_OS: "Unraid" PGID: 100 diff --git a/src/config.rs b/src/config.rs index f292a3c..298ad86 100644 --- a/src/config.rs +++ b/src/config.rs @@ -10,6 +10,7 @@ pub struct Config pub CF_CLEARANCE: String, // bypass bot protection pub CSRFTOKEN: String, // bypass bot protection pub DATABASE_URL: String, // url to database file + pub DEBUG: Option, // debug mode? pub DOWNLOADME_FILEPATH: String, // path to file containing hentai ID to download pub LIBRARY_PATH: String, // path to download hentai to pub LIBRARY_SPLIT: u32, // split library into subdirectories of maximum this many hentai, 0 to disable @@ -27,6 +28,7 @@ impl Default for Config CF_CLEARANCE: "".to_string(), CSRFTOKEN: "".to_string(), DATABASE_URL: "sqlite://./db/db.sqlite".to_owned(), + DEBUG: None, // no entry in default config, defaults to false DOWNLOADME_FILEPATH: "./config/downloadme.txt".to_owned(), LIBRARY_PATH: "./hentai/".to_string(), LIBRARY_SPLIT: 0, diff --git a/src/connect_to_db.rs b/src/connect_to_db.rs index 63e00f9..e6e13bf 100644 --- a/src/connect_to_db.rs +++ b/src/connect_to_db.rs @@ -1,4 +1,5 @@ // Copyright (c) 2024 구FS, all rights reserved. Subject to the MIT licence in `licence.md`. +use sqlx::ConnectOptions; use sqlx::migrate::MigrateDatabase; @@ -59,6 +60,7 @@ pub async fn connect_to_db(database_url: &str) -> Result Result", |o| o.as_str()))] + Reqwest(#[from] reqwest::Error), + + #[error("Creating HTTP client failed with: {source}")] + ReqwestClientBuilder {source: reqwest::Error}, + + #[error("Test connecting to \"{url}\" failed with status code {status}.")] + ReqwestStatus {url: String, status: reqwest::StatusCode}, + + #[error("Connecting to database failed with: {0}")] + Sqlx(#[from] sqlx::Error), +} - #[error("")] - Download {}, - #[error("")] +#[derive(Debug, thiserror::Error)] +pub enum HentaiNewError +{ + #[error + ( + "Hentai has {} page types specified, but {} pages were expected.", + scaler::Formatter::new().set_scaling(scaler::Scaling::None).set_rounding(scaler::Rounding::Magnitude(0)).format(*page_types), + scaler::Formatter::new().set_scaling(scaler::Scaling::None).set_rounding(scaler::Rounding::Magnitude(0)).format(*num_pages) + )] HentaiLengthInconsistency {page_types: u16, num_pages: u16}, #[error(transparent)] + SearchByIdError(#[from] SearchByIdError), + + #[error("Loading hentai tags from database failed with: {0}")] + Sqlx(#[from] sqlx::Error), +} + + +#[derive(Debug, thiserror::Error)] +pub enum HentaiDownloadError +{ + #[error("Saving hentai failed, because \"{directory_path}\" already is a directory.")] + BlockedByDirectory {directory_path: String}, // directory blocked + + #[error("Downloading hentai failed multiple times. Giving up...")] + Download(), // download failed multiple times, more specific error messages already in download logged + + #[error("Serialising hentai metadata failed with: {0}")] + SerdeXml(#[from] serde_xml_rs::Error), // serde xml error + + #[error("Saving hentai failed with: {0}")] + StdIo(#[from] std::io::Error), // std io error + + #[error("Saving hentai failed with: {0}")] + Zip(#[from] zip::result::ZipError), // zip error +} + + +#[derive(Debug, thiserror::Error)] +pub enum HentaiDownloadImageError +{ + #[error("Saving hentai image failed, because \"{directory_path}\" already is a directory.")] + BlockedByDirectory {directory_path: String}, // directory blocked + + #[error("Downloading hentai image from \"{}\" failed with: {0}", .0.url().map_or_else(|| "", |o| o.as_str()))] Reqwest(#[from] reqwest::Error), - #[error("{status}")] + #[error("Downloading hentai image from \"{url}\" failed with status code {status}.")] ReqwestStatus {url: String, status: reqwest::StatusCode}, - #[error(transparent)] + #[error("Saving hentai image at \"{filepath}\" failed with: {source}")] + StdIo {filepath: String, source: std::io::Error}, +} + + +#[derive(Debug, thiserror::Error)] +pub enum SearchByIdError +{ + #[error("Hentai metadata could not be loaded from database and downloading from \"{}\" failed with: {0}", .0.url().map_or_else(|| "", |o| o.as_str()))] + Reqwest(#[from] reqwest::Error), + + #[error("Hentai metadata could not be loaded from database and downloading from \"{url}\" failed with status code {status}.")] + ReqwestStatus {url: String, status: reqwest::StatusCode}, + + #[error("Hentai metadata could not be loaded from database and after downloading, deserialising API response failed with: {0}")] SerdeJson(#[from] serde_json::Error), +} - #[error(transparent)] - SerdeXml(#[from] serde_xml_rs::Error), - #[error(transparent)] - Sqlx(#[from] sqlx::Error), +#[derive(Debug, thiserror::Error)] +pub enum SearchByTagError +{ + #[error("Downloading hentai metadata page 1 from \"{}\" failed with: {0}", .0.url().map_or_else(|| "", |o| o.as_str()))] + Reqwest(#[from] reqwest::Error), - #[error(transparent)] - StdIo(#[from] std::io::Error), + #[error("Downloading hentai metadata page 1 from \"{url}\" failed with status code {status}.")] + ReqwestStatus {url: String, status: reqwest::StatusCode}, - #[error(transparent)] - Zip(#[from] zip::result::ZipError), + #[error("Saving hentai metadata page 1 in database failed with: {0}")] + SerdeJson(#[from] serde_json::Error), } -pub type Result = std::result::Result; // strict error handling, only takes pre defined Error type \ No newline at end of file +#[derive(Debug, thiserror::Error)] +pub enum SearchByTagOnPageError +{ + #[error + ( + "Downloading hentai metadata page {} / {} from \"{}\" failed with: {source}", + scaler::Formatter::new().set_scaling(scaler::Scaling::None).set_rounding(scaler::Rounding::Magnitude(0)).format(*page_no), + scaler::Formatter::new().set_scaling(scaler::Scaling::None).set_rounding(scaler::Rounding::Magnitude(0)).format(*num_pages), + source.url().map_or_else(|| "", |o| o.as_str()) + )] + Reqwest {page_no: u32, num_pages: u32, source: reqwest::Error}, + + #[error + ( + "Downloading hentai metadata page {} / {} from \"{url}\" failed with status code {status}.", + scaler::Formatter::new().set_scaling(scaler::Scaling::None).set_rounding(scaler::Rounding::Magnitude(0)).format(*page_no), + scaler::Formatter::new().set_scaling(scaler::Scaling::None).set_rounding(scaler::Rounding::Magnitude(0)).format(*num_pages), + )] + ReqwestStatus {page_no: u32, num_pages: u32, url: String, status: reqwest::StatusCode}, + + #[error + ( + "Saving hentai metadata page {} / {} in database failed with: {source}", + scaler::Formatter::new().set_scaling(scaler::Scaling::None).set_rounding(scaler::Rounding::Magnitude(0)).format(*page_no), + scaler::Formatter::new().set_scaling(scaler::Scaling::None).set_rounding(scaler::Rounding::Magnitude(0)).format(*num_pages), + )] + SerdeJson {page_no: u32, num_pages: u32, source: serde_json::Error}, +} \ No newline at end of file diff --git a/src/get_hentai_id_list.rs b/src/get_hentai_id_list.rs index 90c4a36..9884908 100644 --- a/src/get_hentai_id_list.rs +++ b/src/get_hentai_id_list.rs @@ -1,5 +1,4 @@ // Copyright (c) 2024 구FS, all rights reserved. Subject to the MIT licence in `licence.md`. -use crate::error::*; use crate::search_api::*; use tokio::io::AsyncWriteExt; @@ -19,7 +18,7 @@ use tokio::io::AsyncWriteExt; /// /// # Returns /// - list of hentai ID to download -pub async fn get_hentai_id_list(downloadme_filepath: &std::path::Path, http_client: &reqwest::Client, nhentai_tag_search_url: &str, nhentai_tag: &Option, db: &sqlx::sqlite::SqlitePool) -> Vec +pub async fn get_hentai_id_list(downloadme_filepath: &str, http_client: &reqwest::Client, nhentai_tag_search_url: &str, nhentai_tag: &Option, db: &sqlx::sqlite::SqlitePool) -> Vec { let mut hentai_id_list: Vec = Vec::new(); // list of hentai id to download @@ -31,14 +30,14 @@ pub async fn get_hentai_id_list(downloadme_filepath: &std::path::Path, http_clie Ok(content) => { hentai_id_list = content.lines().filter_map(|line| line.parse::().ok()).collect(); // String -> Vec, discard unparseable lines - log::info!("Loaded hentai ID list from {downloadme_filepath:?}."); + log::info!("Loaded hentai ID list from \"{downloadme_filepath}\"."); }, - Err(e) => log::error!("Loading hentai ID list from {downloadme_filepath:?} failed with: {e}"), + Err(e) => log::warn!("Loading hentai ID list from \"{downloadme_filepath}\" failed with: {e}"), }; } else { - log::info!("No hentai ID list found at {downloadme_filepath:?}."); + log::info!("No hentai ID list found at \"{downloadme_filepath}\"."); } if !hentai_id_list.is_empty() // if hentai_id_list is not empty: work is done { @@ -59,29 +58,7 @@ pub async fn get_hentai_id_list(downloadme_filepath: &std::path::Path, http_clie ).await { Ok(o) => hentai_id_list = o, - Err(e) => - { - match e - { - Error::Reqwest(e) => log::error! - ( - "Downloading hentai \"{}\" metadata page 1 from \"{}\" failed with: {e}", - nhentai_tag_unwrapped, - e.url().map_or_else(|| "", |o| o.as_str()) - ), - Error::ReqwestStatus {url, status} => log::error! - ( - "Downloading hentai \"{}\" metadata page 1 from \"{url}\" failed with status code {status}.", - nhentai_tag_unwrapped, - ), - Error::SerdeJson(e) => log::error! - ( - "Saving hentai \"{}\" metadata page 1 in database failed with: {e}", - nhentai_tag_unwrapped, - ), - _ => panic!("Unhandled error: {e}"), - }; - } + Err(e) => log::error!("{e}"), } } else // if nhentai_tag is not set: request manual user input @@ -97,11 +74,11 @@ pub async fn get_hentai_id_list(downloadme_filepath: &std::path::Path, http_clie { match file.write_all(hentai_id_list.iter().map(|id| id.to_string()).collect::>().join("\n").as_bytes()).await { - Ok(_) => log::info!("Saved hentai ID list from tag search in {downloadme_filepath:?}."), - Err(e) => log::error!("Writing hentai ID list to {downloadme_filepath:?} failed with: {e}"), + Ok(_) => log::info!("Saved hentai ID list from tag search in {downloadme_filepath}."), + Err(e) => log::warn!("Writing hentai ID list to {downloadme_filepath} failed with: {e}"), } }, - Err(e) => log::error!("Saving hentai ID list at {downloadme_filepath:?} failed with: {e}"), + Err(e) => log::warn!("Saving hentai ID list at {downloadme_filepath} failed with: {e}"), } #[cfg(not(target_family = "unix"))] match tokio::fs::OpenOptions::new().create_new(true).write(true).open(downloadme_filepath).await @@ -110,11 +87,11 @@ pub async fn get_hentai_id_list(downloadme_filepath: &std::path::Path, http_clie { match file.write_all(hentai_id_list.iter().map(|id| id.to_string()).collect::>().join("\n").as_bytes()).await { - Ok(_) => log::info!("Saved hentai ID list from tag search in {downloadme_filepath:?}."), - Err(e) => log::error!("Writing hentai ID list to {downloadme_filepath:?} failed with: {e}"), + Ok(_) => log::info!("Saved hentai ID list from tag search in {downloadme_filepath}."), + Err(e) => log::warn!("Writing hentai ID list to {downloadme_filepath} failed with: {e}"), } }, - Err(e) => log::error!("Saving hentai ID list at {downloadme_filepath:?} failed with: {e}"), + Err(e) => log::warn!("Saving hentai ID list at {downloadme_filepath} failed with: {e}"), } log::debug!("{hentai_id_list:?}"); return hentai_id_list; diff --git a/src/hentai.rs b/src/hentai.rs index 03dde51..b0192d2 100644 --- a/src/hentai.rs +++ b/src/hentai.rs @@ -49,7 +49,7 @@ impl Hentai /// /// # Returns /// - created hentai or error - pub async fn new(id: u32, db: &sqlx::sqlite::SqlitePool, http_client: &reqwest::Client, nhentai_hentai_search_url: &str, library_path: &str, library_split: u32) -> Result + pub async fn new(id: u32, db: &sqlx::sqlite::SqlitePool, http_client: &reqwest::Client, nhentai_hentai_search_url: &str, library_path: &str, library_split: u32) -> Result { const FILENAME_SIZE_MAX: u16 = 255; // maximum filename size [B] const TITLE_CHARACTERS_FORBIDDEN: &str = "\\/:*?\"<>|\t\n"; // forbidden characters in Windows file names @@ -65,19 +65,19 @@ impl Hentai .fetch_optional(db).await // load hentai metadata from database { hentai_table_row = s; - log::info!("Loaded hentai {id} metadata from database."); + log::info!("Loaded hentai metadata from database."); } else // if any step to load from database failed { - log::info!("Hentai {id} metadata could not be loaded from database. Downloading from nhentai.net API..."); + log::info!("Hentai metadata could not be loaded from database. Downloading from nhentai.net API..."); hentai_table_row = search_by_id(http_client, nhentai_hentai_search_url, id, db).await?; // load hentai metadata from api - log::info!("Downloaded hentai {id} metadata."); + log::info!("Downloaded hentai metadata."); } tags = sqlx::query_as("SELECT Tag.* FROM Tag JOIN (SELECT tag_id FROM Hentai_Tag WHERE hentai_id = ?) AS tags_attached_to_hentai_desired ON Tag.id = tags_attached_to_hentai_desired.tag_id") .bind(id) .fetch_all(db).await?; // load tags from database - log::info!("Loaded hentai {id} tags from database."); + log::info!("Loaded hentai tags from database."); for (i, page_type) in hentai_table_row.page_types.char_indices() @@ -87,7 +87,7 @@ impl Hentai } if hentai_table_row.page_types.len() != hentai_table_row.num_pages as usize // if number of pages does not match number of page types: inconsistency { - return Err(Error::HentaiLengthInconsistency {page_types: hentai_table_row.page_types.len() as u16, num_pages: hentai_table_row.num_pages}); + return Err(HentaiNewError::HentaiLengthInconsistency {page_types: hentai_table_row.page_types.len() as u16, num_pages: hentai_table_row.num_pages}); } cbz_filename = hentai_table_row.title_english.clone().unwrap_or_default(); @@ -133,7 +133,7 @@ impl Hentai /// /// # Returns /// - nothing or error - pub async fn download(&self, http_client: &reqwest::Client) -> Result<()> + pub async fn download(&self, http_client: &reqwest::Client) -> Result<(), HentaiDownloadError> { const WORKERS: usize = 5; // number of parallel workers let cbz_final_filepath: String; //filepath to final cbz in library @@ -167,13 +167,12 @@ impl Hentai { if o.is_file() // if final cbz already exists { - log::info!("Hentai {} already exists. Skipped download.", self.id); + log::info!("Hentai already exists. Skipped download."); return Ok(()); // skip download } if o.is_dir() // if cbz filepath blocked by directory { - log::error!("\"{}\" already exists as directory. Skipped download.", cbz_final_filepath); - return Err(Error::BlockedByDirectory {directory_path: cbz_final_filepath.clone()}); // give up + return Err(HentaiDownloadError::BlockedByDirectory {directory_path: cbz_final_filepath.clone()}); // give up } } @@ -187,7 +186,6 @@ impl Hentai { let f_clone: scaler::Formatter = f.clone(); let http_client_clone: reqwest::Client = http_client.clone(); - let id_clone: u32 = self.id; let image_filepath: String = format!("{}{}/{}", self.library_path, self.id, self.images_filename.get(i).expect("Index out of bounds even though should have same size as images_url.")); let image_url_clone: String = self.images_url.get(i).expect("Index out of bounds even though checked before that it fits.").clone(); let num_pages_clone: u16 = self.num_pages; @@ -200,40 +198,12 @@ impl Hentai { Ok(_) => { - log::debug!("Downloaded hentai {id_clone} image {} / {}.", f_clone.format((i+1) as f64), f_clone.format(num_pages_clone as f64)); + log::debug!("Downloaded hentai image {} / {}.", f_clone.format((i+1) as f64), f_clone.format(num_pages_clone as f64)); result = Some(()); // success } Err(e) => { - match e - { - Error::BlockedByDirectory {directory_path} => log::error! - ( - "Saving hentai {id_clone} image {} / {} failed, because \"{directory_path}\" already is a directory.", - f_clone.format((i+1) as f64), - f_clone.format(num_pages_clone as f64), - ), - Error::Reqwest(e) => log::error! - ( - "Downloading hentai {id_clone} image {} / {} from \"{}\" failed with: {e}", - f_clone.format((i+1) as f64), - f_clone.format(num_pages_clone as f64), - e.url().map_or_else(|| "", |o| o.as_str()), - ), - Error::ReqwestStatus {url, status} => log::error! - ( - "Downloading hentai {id_clone} image {} / {} from \"{url}\" failed with status code {status}.", - f_clone.format((i+1) as f64), - f_clone.format(num_pages_clone as f64), - ), - Error::StdIo(e) => log::error! - ( - "Saving hentai {id_clone} image {} / {} failed with: {e}", - f_clone.format((i+1) as f64), - f_clone.format(num_pages_clone as f64), - ), - _ => panic!("Unhandled error: {e}"), - } + log::warn!("{e}"); result = None; // failure } } @@ -247,8 +217,8 @@ impl Hentai } if image_download_success {break;} // if all images were downloaded successfully: continue with cbz creation } - if !image_download_success {return Err(Error::Download {})}; // if after 5 attempts still not all images downloaded successfully: give up - log::info!("Downloaded hentai {} images.", self.id); + if !image_download_success {return Err(HentaiDownloadError::Download {})}; // if after 5 attempts still not all images downloaded successfully: give up + log::info!("Downloaded hentai images."); let zip_file: std::fs::File; @@ -274,7 +244,7 @@ impl Hentai std::fs::File::open(format!("{}{}/{image_filename}", self.library_path, self.id))?.read_to_end(&mut image)?; // open image file, read image into memory zip_writer.start_file(image_filename, zip::write::SimpleFileOptions::default().unix_permissions(0o666))?; // create image file in cbz with permissions "rw-rw-rw-" zip_writer.write_all(&image)?; // write image into cbz - log::debug!("Saved hentai {} image {} / {} in cbz.", self.id, f.format((i+1) as f64), f.format(self.num_pages)); + log::debug!("Saved hentai image {} / {} in cbz.", f.format((i+1) as f64), f.format(self.num_pages)); } #[cfg(target_family = "unix")] zip_writer.start_file("ComicInfo.xml", zip::write::SimpleFileOptions::default().unix_permissions(0o666))?; // create metadata file in cbz with permissions "rw-rw-rw-" @@ -292,7 +262,7 @@ impl Hentai if let Some(parent) = std::path::Path::new(cbz_final_filepath.as_str()).parent() {tokio::fs::DirBuilder::new().recursive(true).create(parent).await?;} // create all parent directories } tokio::fs::rename(cbz_temp_filepath, cbz_final_filepath).await?; // move finished cbz to final location in library - log::info!("Saved hentai {} cbz.", self.id); + log::info!("Saved hentai cbz."); if let Err(e) = tokio::fs::remove_dir_all(format!("{}{}", self.library_path, self.id)).await // cleanup, delete image directory @@ -314,31 +284,55 @@ impl Hentai /// /// # Returns /// - nothing or error - async fn download_image(http_client: &reqwest::Client, image_url: &str, image_filepath: &str) -> Result<()> + async fn download_image(http_client: &reqwest::Client, image_url: &str, image_filepath: &str) -> Result<(), HentaiDownloadImageError> { if let Ok(o) = tokio::fs::metadata(image_filepath).await { if o.is_file() {return Ok(());} // if image already exists: skip download - if o.is_dir() {return Err(Error::BlockedByDirectory {directory_path: image_filepath.to_owned()});} // if image filepath blocked by directory: give up + if o.is_dir() {return Err(HentaiDownloadImageError::BlockedByDirectory {directory_path: image_filepath.to_owned()});} // if image filepath blocked by directory: give up } let r: reqwest::Response = http_client.get(image_url).send().await?; // tag search, page - if r.status() != reqwest::StatusCode::OK {return Err(Error::ReqwestStatus {url: r.url().to_string(), status: r.status()});} // if status is not ok: something went wrong + if r.status() != reqwest::StatusCode::OK {return Err(HentaiDownloadImageError::ReqwestStatus {url: r.url().to_string(), status: r.status()});} // if status is not ok: something went wrong let mut file: tokio::fs::File; #[cfg(target_family = "unix")] { - if let Some(parent) = std::path::Path::new(image_filepath).parent() {tokio::fs::DirBuilder::new().recursive(true).mode(0o777).create(parent).await?;} // create all parent directories with permissions "drwxrwxrwx" - file = tokio::fs::OpenOptions::new().create_new(true).mode(0o666).write(true).open(image_filepath).await?; + if let Some(parent) = std::path::Path::new(image_filepath).parent() // create all parent directories with permissions "drwxrwxrwx" + { + if let Err(e) = tokio::fs::DirBuilder::new().recursive(true).mode(0o777).create(parent).await + { + return Err(HentaiDownloadImageError::StdIo {filepath: image_filepath.to_owned(), source: e}); + } + } + match tokio::fs::OpenOptions::new().create_new(true).mode(0o666).write(true).open(image_filepath).await + { + Ok(o) => file = o, + Err(e) => {return Err(HentaiDownloadImageError::StdIo {filepath: image_filepath.to_owned(), source: e});} + } } #[cfg(not(target_family = "unix"))] { - if let Some(parent) = std::path::Path::new(image_filepath).parent() {tokio::fs::DirBuilder::new().recursive(true).create(parent).await?;} // create all parent directories - file = tokio::fs::OpenOptions::new().create_new(true).write(true).open(image_filepath).await?; + if let Some(parent) = std::path::Path::new(image_filepath).parent() // create all parent directories + { + if let Err(e) = tokio::fs::DirBuilder::new().recursive(true).create(parent).await + { + return Err(HentaiDownloadImageError::StdIo {filepath: image_filepath.to_owned(), source: e}); + } + } + match tokio::fs::OpenOptions::new().create_new(true).write(true).open(image_filepath).await + { + Ok(o) => file = o, + Err(e) => {return Err(HentaiDownloadImageError::StdIo {filepath: image_filepath.to_owned(), source: e});} + } + } + + if let Err(e) = file.write_all_buf(&mut r.bytes().await?).await // save image with permissions "rw-rw-rw-" + { + return Err(HentaiDownloadImageError::StdIo {filepath: image_filepath.to_owned(), source: e}); } - file.write_all_buf(&mut r.bytes().await?).await?; // save image with permissions "rw-rw-rw-" return Ok(()); } diff --git a/src/main.rs b/src/main.rs index da4c5d0..c8d80a2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,44 +15,50 @@ mod search_api; fn main() -> std::process::ExitCode { - const DEBUG: bool = false; // debug mode? let mut crate_logging_level: std::collections::HashMap = std::collections::HashMap::new(); // logging level for individual crates let config: Config; // config, settings let tokio_rt: tokio::runtime::Runtime = tokio::runtime::Runtime::new().expect("Creating tokio runtime failed."); // async runtime - crate_logging_level.insert("hyper_util".to_owned(), log::Level::Info); // shut up - crate_logging_level.insert("serde_xml_rs".to_owned(), log::Level::Error); // shut up - crate_logging_level.insert("sqlx::query".to_owned(), log::Level::Error); // shut up - if DEBUG == true // setup logging - { - setup_logging::setup_logging(log::Level::Debug, Some(crate_logging_level), "./log/%Y-%m-%dT%H_%M.log"); - } - else - { - setup_logging::setup_logging(log::Level::Info, Some(crate_logging_level), "./log/%Y-%m-%d.log"); - } - std::panic::set_hook(Box::new(|panic_info: &std::panic::PanicInfo| // override panic behaviour { log::error!("{}", panic_info); // log panic source and reason log::error!("{}", std::backtrace::Backtrace::capture()); // log backtrace })); - match load_config::load_config + match load_config::load_config // load config ( vec! [ load_config::Source::Env, load_config::Source::File(load_config::SourceFile::Toml("./config/.env".to_string())), ], - Some(load_config::SourceFile::Toml("./config/.env".to_string())) + Some(load_config::SourceFile::Toml("./config/.env".to_string())), ) { - Ok(o) => {config = o;} // loaded config successfully - Err(_) => {return std::process::ExitCode::FAILURE;} // loading config failed + Ok(o) => config = o, // loaded config successfully + Err(e) => // loading config failed + { + setup_logging::setup_logging(log::Level::Info, None, "./log/%Y-%m-%d.log"); // setup logging with default settings to log error + log::error!("{e}"); + return std::process::ExitCode::FAILURE; + } } + crate_logging_level.insert("hyper_util".to_owned(), log::Level::Info); // shut up + crate_logging_level.insert("serde_xml_rs".to_owned(), log::Level::Error); // shut up + crate_logging_level.insert("sqlx::query".to_owned(), log::Level::Error); // shut up + if config.DEBUG.unwrap_or(false) // setup logging, if DEBUG unset default to false + { + setup_logging::setup_logging(log::Level::Debug, None, "./log/%Y-%m-%dT%H_%M.log"); + } + else + { + setup_logging::setup_logging(log::Level::Info, None, "./log/%Y-%m-%d.log"); + } + + log::debug!("Loaded {config:?}."); // log loaded config + match std::panic::catch_unwind(|| tokio_rt.block_on(main_inner(config.clone()))) // execute main_inner, catch panic { @@ -63,22 +69,23 @@ fn main() -> std::process::ExitCode Ok(()) => {return std::process::ExitCode::SUCCESS;} // program executed successfully Err(e) => // program failed in a controlled manner { - match e // log error + log::error!("{e}"); // log error + match e { - Error::Reqwest(e) => log::error!("Test connecting to \"{}\" failed with: {e}", e.url().map_or_else(|| "", |o| o.as_str())), - Error::ReqwestStatus { url, status } => + Error::Reqwest(e) => log::error!("{e}"), + Error::ReqwestClientBuilder { source } => log::error!("{source}"), + Error::ReqwestStatus {url, status} => { if status == reqwest::StatusCode::FORBIDDEN { - log::error!("Test connecting to \"{url}\" failed with status code {status}. Check if cookies \"cf_clearance\" and \"csrftoken\" and user agent are set and current."); + log::error!("Test connecting to \"{url}\" failed with status code {status}. Check if cookies \"cf_clearance\", \"csrftoken\", and user agent are set and current."); } else { log::error!("Test connecting to \"{url}\" failed with status code {status}."); } } - Error::Sqlx(e) => log::error!("Connecting to database at \"{}\" failed with: {e}\nIf you're creating a new database, ensure all parent directories already exist.", config.DATABASE_URL), - _ => panic!("Unhandled error: {e}"), + Error::Sqlx(e) => log::error!("{e}"), } return std::process::ExitCode::FAILURE; } diff --git a/src/main_inner.rs b/src/main_inner.rs index 4327295..e89dde8 100644 --- a/src/main_inner.rs +++ b/src/main_inner.rs @@ -6,7 +6,7 @@ use crate::get_hentai_id_list::*; use crate::hentai::*; -pub async fn main_inner(config: Config) -> Result<()> +pub async fn main_inner(config: Config) -> Result<(), Error> { const NHENTAI_HENTAI_SEARCH_URL: &str="https://nhentai.net/api/gallery/"; // nhentai search by id api url const NHENTAI_TAG_SEARCH_URL: &str="https://nhentai.net/api/galleries/search"; // nhentai search by tag api url @@ -25,28 +25,26 @@ pub async fn main_inner(config: Config) -> Result<()> { let mut headers: reqwest::header::HeaderMap = reqwest::header::HeaderMap::new(); // headers - headers.insert(reqwest::header::USER_AGENT, reqwest::header::HeaderValue::from_str(&config.USER_AGENT).unwrap_or_else - ( - |e| - { - log::warn!("Adding user agent to HTTP client headers failed with: {e}\nUsing empty user agent instead."); - reqwest::header::HeaderValue::from_str("").expect("Creating empty user agent failed.") - } - )); - headers.insert(reqwest::header::COOKIE, reqwest::header::HeaderValue::from_str(format!("cf_clearance={}; csrftoken={}", config.CF_CLEARANCE, config.CSRFTOKEN).as_str()).unwrap_or_else - ( - |e| - { - log::warn!("Adding cookies \"cf_clearance\" and \"csrftoken\" to HTTP client headers failed with: {e}\nUsing no cookies instead."); - reqwest::header::HeaderValue::from_str("").expect("Creating empty cookies failed.") - } - )); - http_client = reqwest::Client::builder() // create http client + match reqwest::header::HeaderValue::from_str(&config.USER_AGENT) + { + Ok(o) => _ = headers.insert(reqwest::header::USER_AGENT, o), + Err(e) => log::warn!("Adding user agent to HTTP client headers failed with: {e}\nUsing empty user agent instead."), + } + match reqwest::header::HeaderValue::from_str(format!("cf_clearance={}; csrftoken={}", config.CF_CLEARANCE, config.CSRFTOKEN).as_str()) + { + Ok(o) => _ = headers.insert(reqwest::header::COOKIE, o), + Err(e) => log::warn!("Adding cookies \"cf_clearance\" and \"csrftoken\" to HTTP client headers failed with: {e}\nUsing no cookies instead."), + } + match reqwest::Client::builder() // create http client .connect_timeout(timeout) .cookie_store(true) // enable cookies .default_headers(headers) .read_timeout(timeout) - .build().expect("Creating HTTP client failed."); + .build() + { + Ok(o) => http_client = o, + Err(e) => return Err(Error::ReqwestClientBuilder {source: e}), + } let r: reqwest::Response = http_client.get(NHENTAI_TAG_SEARCH_URL).query(&[("query", "language:english"), ("page", "1")]).send().await?; // send test request if r.status() != reqwest::StatusCode::OK // if status is not ok: something went wrong { @@ -60,7 +58,7 @@ pub async fn main_inner(config: Config) -> Result<()> db = connect_to_db(&config.DATABASE_URL).await?; // connect to database hentai_id_list = get_hentai_id_list ( - std::path::Path::new(config.DOWNLOADME_FILEPATH.as_str()), + config.DOWNLOADME_FILEPATH.as_str(), &http_client, NHENTAI_TAG_SEARCH_URL, &config.NHENTAI_TAG, @@ -71,7 +69,7 @@ pub async fn main_inner(config: Config) -> Result<()> for (i, hentai_id) in hentai_id_list.iter().enumerate() { log::info!("--------------------------------------------------"); - log::info!("{} / {} ({})", f0.format((i+1) as f64), f0.format(hentai_id_list.len() as f64), fm2.format((i+1) as f64 / hentai_id_list.len() as f64)); + log::info!("{} / {} ({}) | hentai {hentai_id}", f0.format((i+1) as f64), f0.format(hentai_id_list.len() as f64), fm2.format((i+1) as f64 / hentai_id_list.len() as f64)); let hentai: Hentai; // hentai to download @@ -87,30 +85,14 @@ pub async fn main_inner(config: Config) -> Result<()> Ok(o) => hentai = o, // hentai created successfully Err(e) => // hentai creation failed { - match e - { - Error::HentaiLengthInconsistency { page_types, num_pages } => log::error!("Hentai {hentai_id} has {} page types specified, but {} pages were expected.", f0.format(page_types), f0.format(num_pages)), - Error::Reqwest(e) => log::error!("Hentai {hentai_id} metadata could not be loaded from database and downloading from \"{}\" failed with: {e}", e.url().map_or_else(|| "", |o| o.as_str())), - Error::ReqwestStatus {url, status} => log::error!("Hentai {hentai_id} metadata could not be loaded from database and downloading from \"{url}\" failed with status code {status}."), - Error::SerdeJson(e) => log::error!("Hentai {hentai_id} metadata could not be loaded from database and after downloading, deserialising API response failed with: {e}"), - Error::Sqlx(e) => log::error!("Loading hentai {hentai_id} tags from database failed with: {e}"), - _ => panic!("Unhandled error: {e}"), - } + log::error!("{e}"); continue; // skip download } } if let Err(e) = hentai.download(&http_client).await { - match e - { - Error::BlockedByDirectory {directory_path} => {log::error!("Downloading hentai {hentai_id} failed, because \"{directory_path}\" already is a directory.");} // directory blocked - Error::Download {} => log::error!("Downloading hentai {hentai_id} failed multiple times. Giving up..."), // download failed multiple times, more specific error messages already in download logged - Error::SerdeXml(e) => log::error!("Serialising hentai {hentai_id} metadata failed with: {e}"), // serde xml error - Error::StdIo(e) => log::error!("Saving hentai {hentai_id} failed with: {e}"), // std io error - Error::Zip(e) => log::error!("Saving hentai {hentai_id} failed with: {e}"), // zip error - _ => panic!("Unhandled error: {e}"), - } + log::error!{"{e}"}; } } log::info!("--------------------------------------------------"); @@ -120,7 +102,7 @@ pub async fn main_inner(config: Config) -> Result<()> if let Err(e) = tokio::fs::remove_file(&config.DOWNLOADME_FILEPATH).await // server mode cleanup, delete downloadme { - log::error!("Deleting \"{}\" failed with: {e}", config.DOWNLOADME_FILEPATH); + log::warn!("Deleting \"{}\" failed with: {e}", config.DOWNLOADME_FILEPATH); } db.close().await; // close database connection log::info!("Disconnected from database at \"{}\".", config.DATABASE_URL); diff --git a/src/search_api.rs b/src/search_api.rs index bda9ac4..d2db1bf 100644 --- a/src/search_api.rs +++ b/src/search_api.rs @@ -15,13 +15,13 @@ use crate::hentai::*; /// /// # Returns /// - HentaiTableRow entry or error -pub async fn search_by_id(http_client: &reqwest::Client, nhentai_hentai_search_url: &str, id: u32, db: &sqlx::sqlite::SqlitePool) -> Result +pub async fn search_by_id(http_client: &reqwest::Client, nhentai_hentai_search_url: &str, id: u32, db: &sqlx::sqlite::SqlitePool) -> Result { let r_serialised: HentaiSearchResponse; // response in json format let r: reqwest::Response = http_client.get(format!("{nhentai_hentai_search_url}{id}").as_str()).send().await?; // search hentai - if r.status() != reqwest::StatusCode::OK {return Err(Error::ReqwestStatus {url: r.url().to_string(), status: r.status()});} // if status is not ok: something went wrong + if r.status() != reqwest::StatusCode::OK {return Err(SearchByIdError::ReqwestStatus {url: r.url().to_string(), status: r.status()});} // if status is not ok: something went wrong r_serialised = serde_json::from_str(r.text().await?.as_str())?; // deserialise json, get this response here to get number of pages before starting parallel workers if let Err(e) = r_serialised.write_to_db(&db).await // save data to database, if unsuccessful: warning { @@ -53,7 +53,7 @@ pub async fn search_by_id(http_client: &reqwest::Client, nhentai_hentai_search_u /// /// # Returns /// - list of hentai ID to download or error -pub async fn search_by_tag(http_client: &reqwest::Client, nhentai_tag_search_url: &str, nhentai_tag: &str, db: &sqlx::sqlite::SqlitePool) -> Result> +pub async fn search_by_tag(http_client: &reqwest::Client, nhentai_tag_search_url: &str, nhentai_tag: &str, db: &sqlx::sqlite::SqlitePool) -> Result, SearchByTagError> { const WORKERS: usize = 2; // number of concurrent workers let f = scaler::Formatter::new() @@ -67,13 +67,13 @@ pub async fn search_by_tag(http_client: &reqwest::Client, nhentai_tag_search_url { let r: reqwest::Response = http_client.get(nhentai_tag_search_url).query(&[("query", nhentai_tag), ("page", "1")]).send().await?; // tag search, page - if r.status() != reqwest::StatusCode::OK {return Err(Error::ReqwestStatus {url: r.url().to_string(), status: r.status()});} // if status is not ok: something went wrong + if r.status() != reqwest::StatusCode::OK {return Err(SearchByTagError::ReqwestStatus {url: r.url().to_string(), status: r.status()});} // if status is not ok: something went wrong r_serialised = serde_json::from_str(r.text().await?.as_str())?; // deserialise json, get this response here to get number of pages before starting parallel workers if let Err(e) = r_serialised.write_to_db(&db).await // save data to database, if unsuccessful: warning { - log::warn!("Saving hentai \"{nhentai_tag}\" metadata page 1 / {} in database failed with: {e}", f.format(r_serialised.num_pages)); + log::warn!("Saving hentai metadata page 1 / {} in database failed with: {e}", f.format(r_serialised.num_pages)); } - log::info!("Downloaded hentai \"{nhentai_tag}\" metadata page 1 / {}.", f.format(r_serialised.num_pages)); + log::info!("Downloaded hentai metadata page 1 / {}.", f.format(r_serialised.num_pages)); } for hentai in r_serialised.result // collect hentai id @@ -98,34 +98,12 @@ pub async fn search_by_tag(http_client: &reqwest::Client, nhentai_tag_search_url { Ok(o) => { - log::info!("Downloaded hentai \"{nhentai_tag}\" metadata page {} / {}.", f_clone.format(page_no), f_clone.format(r_serialised.num_pages)); + log::info!("Downloaded hentai metadata page {} / {}.", f_clone.format(page_no), f_clone.format(r_serialised.num_pages)); result = Some(o); } Err(e) => { - match e - { - Error::Reqwest(e) => log::error! - ( - "Downloading hentai \"{nhentai_tag}\" metadata page {} / {} from \"{}\" failed with: {e}", - f_clone.format(page_no), - f_clone.format(r_serialised.num_pages), - e.url().map_or_else(|| "", |o| o.as_str()), - ), - Error::ReqwestStatus {url, status} => log::error! - ( - "Downloading hentai \"{nhentai_tag}\" metadata page {} / {} from \"{url}\" failed with status code {status}.", - f_clone.format(page_no), - f_clone.format(r_serialised.num_pages), - ), - Error::SerdeJson(e) => log::error! - ( - "Deserialising hentai \"{nhentai_tag}\" metadata page {} / {} failed with: {e}", - f_clone.format(page_no), - f_clone.format(r_serialised.num_pages), - ), - _ => panic!("Unhandled error: {e}"), - }; + log::warn!("{e}"); result = None; } } @@ -155,32 +133,46 @@ pub async fn search_by_tag(http_client: &reqwest::Client, nhentai_tag_search_url /// /// # Returns /// - list of hentai ID to download or error -async fn search_by_tag_on_page(http_client: reqwest::Client, nhentai_tag_search_url: String, nhentai_tag: String, page_no: u32, num_pages: u32, db: sqlx::sqlite::SqlitePool) -> Result> +async fn search_by_tag_on_page(http_client: reqwest::Client, nhentai_tag_search_url: String, nhentai_tag: String, page_no: u32, num_pages: u32, db: sqlx::sqlite::SqlitePool) -> Result, SearchByTagOnPageError> { let f = scaler::Formatter::new() .set_scaling(scaler::Scaling::None) .set_rounding(scaler::Rounding::Magnitude(0)); // formatter let mut hentai_id_list: Vec = Vec::new(); // list of hentai id to download let mut r: reqwest::Response; // nhentai.net api response - let r_serialised: TagSearchResponse; // response in json format + let r_serialised: TagSearchResponse; // response in json format+ + let r_text: String; // response text loop { - r = http_client.get(nhentai_tag_search_url.clone()).query(&[("query", nhentai_tag.clone()), ("page", page_no.to_string())]).send().await?; // tag search, page + match http_client.get(nhentai_tag_search_url.clone()).query(&[("query", nhentai_tag.clone()), ("page", page_no.to_string())]).send().await // tag search, page + { + Ok(o) => r = o, + Err(e) => return Err(SearchByTagOnPageError::Reqwest {page_no, num_pages, source: e}), + } if r.status() == reqwest::StatusCode::TOO_MANY_REQUESTS // if status is too many requests: wait and retry { - log::debug!("Downloading hentai \"{nhentai_tag}\" metadata page {} from \"{}\" failed with status code {}. Waiting 2 s and retrying...", f.format(page_no), r.url().to_string(), r.status()); + log::debug!("Downloading hentai metadata page {} from \"{}\" failed with status code {}. Waiting 2 s and retrying...", f.format(page_no), r.url().to_string(), r.status()); tokio::time::sleep(std::time::Duration::from_secs(2)).await; continue; } - if r.status() != reqwest::StatusCode::OK {return Err(Error::ReqwestStatus {url: r.url().to_string(), status: r.status()});} // if status is not ok: something went wrong + if r.status() != reqwest::StatusCode::OK {return Err(SearchByTagOnPageError::ReqwestStatus {page_no, num_pages, url: r.url().to_string(), status: r.status()});} // if status is not ok: something went wrong break; // everything went well, continue with processing } - r_serialised = serde_json::from_str(r.text().await?.as_str())?; // deserialise json + match r.text().await // get response text + { + Ok(o) => r_text = o, + Err(e) => return Err(SearchByTagOnPageError::Reqwest {page_no, num_pages, source: e}), + } + match serde_json::from_str(r_text.as_str()) // deserialise json + { + Ok(o) => r_serialised = o, + Err(e) => return Err(SearchByTagOnPageError::SerdeJson {page_no, num_pages, source: e}), + } if let Err(e) = r_serialised.write_to_db(&db).await // save data to database { - log::warn!("Saving hentai \"{nhentai_tag}\" metadata page {} / {} in database failed with: {e}", f.format(page_no), f.format(num_pages)); + log::warn!("Saving hentai metadata page {} / {} in database failed with: {e}", f.format(page_no), f.format(num_pages)); } for hentai in r_serialised.result // collect hentai id