added proper database setup via migrations; removed DATABASE_URL sett…

…ing; WARNING: recreation of the database is necessary, just delete the "./db/" directory

Cargo.toml

@@ -9,7 +9,7 @@ license     = "MIT"
 name        = "nhentai_archivist"
 readme      = "readme.md"
 repository  = "https://github.com/9-FS/nhentai_archivist"
-version     = "3.7.0"
+version     = "3.8.0"
 
 [dependencies]
 chrono = { version = "^0.4.0", features = ["serde"] }

db_migrations/001_3.0.0.sql

@@ -0,0 +1,31 @@
+CREATE TABLE Hentai
+(
+    id INTEGER NOT NULL,
+    cover_type TEXT NOT NULL,
+    media_id INTEGER NOT NULL,
+    num_favorites INTEGER NOT NULL,
+    num_pages INTEGER NOT NULL,
+    page_types TEXT NOT NULL,
+    scanlator TEXT,
+    title_english TEXT,
+    title_japanese TEXT,
+    title_pretty TEXT,
+    upload_date TEXT NOT NULL,
+    PRIMARY KEY(id)
+);
+CREATE TABLE Tag
+(
+    id INTEGER NOT NULL,
+    name TEXT NOT NULL,
+    type TEXT NOT NULL,
+    url TEXT NOT NULL,
+    PRIMARY KEY(id)
+);
+CREATE TABLE Hentai_Tag
+(
+    hentai_id INTEGER NOT NULL,
+    tag_id INTEGER NOT NULL,
+    PRIMARY KEY(hentai_id, tag_id),
+    FOREIGN KEY(hentai_id) REFERENCES Hentai(id),
+    FOREIGN KEY(tag_id) REFERENCES Tag(id)
+);

docker-compose.yaml

@@ -1,7 +1,7 @@
 services:
     nhentai_archivist:
         container_name: "nhentai_archivist"
-        image: "ghcr.io/9-fs/nhentai_archivist:3.7.0"
+        image: "ghcr.io/9-fs/nhentai_archivist:3.8.0"
         environment:
             HOST_OS: "Unraid"
             TZ: "UTC"

readme.md

@@ -45,10 +45,6 @@ I'm happy about anyone who finds my software useful and feedback is also always
 
     Setting this to `false` prevents the temporary directory containing the original images from being deleted after the CBZ file has been created. In addition to that it also saves a `ComicBook.xml` in the directory. This can be useful to improve compatibility with third party readers or deduplication software.
 
-- `DATABASE_URL`
-
-    This is the URL to the SQLite database file. If you changed `DATABASE_URL`, confirm the database directory already exists. It is possible that it is not created automatically because the URL could point to a remote directory. The database file will and should be created automatically.
-
 - `DONTDOWNLOADME_FILEPATH`, optional, defaults to `None`
 
     This is the path to the file containing the nHentai ID you explicitly do not want to download, separated by line breaks. It has priority over all input methods. If you want to systematically exclude hentai by tag, use the `-` operator in the tag search instead.
@@ -120,7 +116,6 @@ I'm happy about anyone who finds my software useful and feedback is also always
 Example `./config/.env`:
 
 ```TOML
-DATABASE_URL = "./db/db.sqlite"
 LIBRARY_PATH = "./hentai/"
 ```
 
@@ -134,7 +129,6 @@ Example `./config/.env`:
 
 ```TOML
 CSRFTOKEN = "your token here"
-DATABASE_URL = "./db/db.sqlite"
 DONTDOWNLOADME_FILEPATH = "./config/dontdownloadme.txt"
 DOWNLOADME_FILEPATH = "./config/downloadme.txt"
 LIBRARY_PATH = "./hentai/"
@@ -154,7 +148,6 @@ Example `./config/.env`:
 ```TOML
 CF_CLEARANCE = ""
 CSRFTOKEN = "your token here"
-DATABASE_URL = "./db/db.sqlite"
 DONTDOWNLOADME_FILEPATH = "./config/dontdownloadme.txt"
 DOWNLOADME_FILEPATH = "./config/downloadme.txt"
 LIBRARY_PATH = "./hentai/"

src/config.rs

@@ -10,7 +10,6 @@ pub struct Config
     pub CF_CLEARANCE: Option<String>, // bypass bot protection
     pub CLEANUP_TEMPORARY_FILES: Option<bool>, // clean up temporary files after downloading? some prefer off for deduplication or compatibility with other tools
     pub CSRFTOKEN: Option<String>, // bypass bot protection
-    pub DATABASE_URL: String, // url to database file
     pub DEBUG: Option<bool>, // debug mode?
     pub DONTDOWNLOADME_FILEPATH: Option<String>, // path to file containing hentai ID to not download, blacklist
     pub DOWNLOADME_FILEPATH: Option<String>, // path to file containing hentai ID to download
@@ -31,7 +30,6 @@ impl Default for Config
             CF_CLEARANCE: None,
             CLEANUP_TEMPORARY_FILES: None,
             CSRFTOKEN: Some("".to_owned()),
-            DATABASE_URL: "./db/db.sqlite".to_owned(),
             DEBUG: None, // no entry in default config, defaults to false
             DONTDOWNLOADME_FILEPATH: Some("./config/dontdownloadme.txt".to_owned()),
             DOWNLOADME_FILEPATH: Some("./config/downloadme.txt".to_owned()),

src/connect_to_db.rs

@@ -4,98 +4,57 @@ use sqlx::migrate::MigrateDatabase;
 
 
 /// # Summary
-/// Connects to database at `database_url` and returns a connection pool. If database does not exist, creates a new database and initialises it with the instructions in `./db/create_db.sql`.
+/// Creates a new database or connects to an existing one at `db_url`, runs the instructions in `migrations_path`, and returns a connection pool.
 ///
 /// # Arguments
-/// - `database_url`: path to database file
+/// - `db_url`: url to database file, might not be local but is recommended to be so
+/// - `migrations_path`: path to directory containing migration files
 ///
 /// # Returns
 /// - connection pool to database or error
-pub async fn connect_to_db(database_url: &str) -> Result<sqlx::sqlite::SqlitePool, sqlx::Error>
+pub async fn connect_to_db(db_url: &str, db_migrations_path: &str) -> Result<sqlx::SqlitePool, sqlx::Error>
 {
-    const CREATE_DB_QUERY_STRING: &str = // query string to create all tables except the dynamically created Hentai_{id}_Pages
-        "CREATE TABLE Hentai
-        (
-            id INTEGER NOT NULL,
-            cover_type TEXT NOT NULL,
-            media_id INTEGER NOT NULL,
-            num_favorites INTEGER NOT NULL,
-            num_pages INTEGER NOT NULL,
-            page_types TEXT NOT NULL,
-            scanlator TEXT,
-            title_english TEXT,
-            title_japanese TEXT,
-            title_pretty TEXT,
-            upload_date TEXT NOT NULL,
-            PRIMARY KEY(id)
-        );
-        CREATE TABLE Tag
-        (
-            id INTEGER NOT NULL,
-            name TEXT NOT NULL,
-            type TEXT NOT NULL,
-            url TEXT NOT NULL,
-            PRIMARY KEY(id)
-        );
-        CREATE TABLE Hentai_Tag
-        (
-            hentai_id INTEGER NOT NULL,
-            tag_id INTEGER NOT NULL,
-            PRIMARY KEY(hentai_id, tag_id),
-            FOREIGN KEY(hentai_id) REFERENCES Hentai(id),
-            FOREIGN KEY(tag_id) REFERENCES Tag(id)
-        );";
-    let db: sqlx::sqlite::SqlitePool; // database containing all metadata from nhentai.net api
+    let db: sqlx::SqlitePool; // database connection pool
 
 
-    if !sqlx::sqlite::Sqlite::database_exists(database_url).await? // if database does not exist
+    if !sqlx::Sqlite::database_exists(db_url).await? // if database does not exist
     {
-        match std::path::Path::new(database_url).parent()
+        match std::path::Path::new(db_url).parent()
         {
             Some(parent) =>
             {
                 #[cfg(target_family = "unix")]
                 if let Err(e) = tokio::fs::DirBuilder::new().recursive(true).mode(0o777).create(parent).await // create all parent directories with permissions "drwxrwxrwx"
                 {
-                    log::warn!("Creating parent directories for new database at \"{database_url}\" failed with {e}.\nThis could be expected behaviour, usually if this is a remote pointing URL and not a local filepath. In that case create the parent directories manually.");
+                    log::warn!("Creating parent directories for new database at \"{db_url}\" failed with {e}.\nThis could be expected behaviour, usually if this is a remote pointing URL and not a local filepath. In that case create the parent directories manually.");
                 }
                 #[cfg(not(target_family = "unix"))]
                 if let Err(e) = tokio::fs::DirBuilder::new().recursive(true).create(parent).await // create all parent directories
                 {
-                    log::warn!("Creating parent directories for new database at \"{database_url}\" failed with {e}.\nThis could be expected behaviour, usually if this is a remote pointing URL and not a local filepath. In that case create the parent directories manually.");
+                    log::warn!("Creating parent directories for new database at \"{db_url}\" failed with {e}.\nThis could be expected behaviour, usually if this is a remote pointing URL and not a local filepath. In that case create the parent directories manually.");
                 }
             }
-            None => log::warn!("Creating parent directories for new database at \"{database_url}\", because the directory part could not be parsed.\nThis could be expected behaviour, usually if this is a remote pointing URL and not a local filepath. In that case create the parent directories manually."),
+            None => log::warn!("Creating parent directories for new database at \"{db_url}\", because the directory part could not be parsed.\nThis could be expected behaviour, usually if this is a remote pointing URL and not a local filepath. In that case create the parent directories manually."),
         }
-        sqlx::sqlite::Sqlite::create_database(database_url).await?; // create new database
-        log::info!("Created new database at \"{}\".", database_url);
+        sqlx::Sqlite::create_database(db_url).await?; // create new database
+        log::info!("Created new database at \"{db_url}\".");
+    }
 
-        db = sqlx::sqlite::SqlitePoolOptions::new()
-            .max_connections(1) // only 1 connection to database at the same time, otherwise concurrent writers fail
-            .max_lifetime(None) // keep connection open indefinitely otherwise database locks up after lifetime, closing and reconnecting manually
-            .connect(database_url).await?; // connect to database
-        db.set_connect_options(sqlx::sqlite::SqliteConnectOptions::new()
-            .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal) // use write-ahead journal for better performance
-            .locking_mode(sqlx::sqlite::SqliteLockingMode::Exclusive) // do not release file lock until all transactions are complete
-            .log_slow_statements(log::LevelFilter::Warn, std::time::Duration::from_secs(5)) // log slow statements only after 5 s
-            .synchronous(sqlx::sqlite::SqliteSynchronous::Normal)); // ensure data is written to disk after each transaction for consistent state
-        log::info!("Connected to database at \"{}\".", database_url);
+    db = sqlx::sqlite::SqlitePoolOptions::new()
+        .max_connections(1) // only 1 connection to database at the same time, otherwise concurrent writers fail
+        .max_lifetime(None) // keep connection open indefinitely otherwise database locks up after lifetime, closing and reconnecting manually
+        .connect(db_url).await?; // connect to database
+    db.set_connect_options(sqlx::sqlite::SqliteConnectOptions::new()
+        .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal) // use write-ahead journal for better performance
+        .locking_mode(sqlx::sqlite::SqliteLockingMode::Exclusive) // do not release file lock until all transactions are complete
+        .log_slow_statements(log::LevelFilter::Warn, std::time::Duration::from_secs(5)) // log slow statements only after 5 s
+        .synchronous(sqlx::sqlite::SqliteSynchronous::Normal)); // ensure data is written to disk after each transaction for consistent state
+    log::info!("Connected to database at \"{db_url}\".");
 
-        sqlx::query(CREATE_DB_QUERY_STRING).execute(&db).await?; // initialise database by creating tables
-        log::info!("Created database tables.");
-    }
-    else // if database already exists
+    if std::path::Path::new(db_migrations_path).exists() // if migrations path exists
     {
-        db = sqlx::sqlite::SqlitePoolOptions::new()
-            .max_connections(1) // only 1 connection to database at the same time, otherwise concurrent writers fail
-            .max_lifetime(None) // keep connection open indefinitely otherwise database locks up after lifetime, closing and reconnecting manually
-            .connect(database_url).await?; // connect to database
-        db.set_connect_options(sqlx::sqlite::SqliteConnectOptions::new()
-            .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal) // use write-ahead journal for better performance
-            .locking_mode(sqlx::sqlite::SqliteLockingMode::Exclusive) // do not release file lock until all transactions are complete
-            .log_slow_statements(log::LevelFilter::Warn, std::time::Duration::from_secs(5)) // log slow statements only after 5 s
-            .synchronous(sqlx::sqlite::SqliteSynchronous::Normal)); // ensure data is written to disk after each transaction for consistent state
-        log::info!("Connected to database at \"{}\".", database_url);
+        sqlx::migrate::Migrator::new(std::path::Path::new(db_migrations_path)).await?.run(&db).await?; // run migrations to create and update tables
+        log::debug!("Executed migrations at \"{db_migrations_path}\".");
     }
 
     return Ok(db);

src/main.rs

@@ -52,14 +52,14 @@ fn main() -> std::process::ExitCode
 
     crate_logging_level.insert("hyper_util".to_owned(), log::Level::Info); // shut up
     crate_logging_level.insert("serde_xml_rs".to_owned(), log::Level::Error); // shut up
-    crate_logging_level.insert("sqlx::query".to_owned(), log::Level::Error); // shut up
+    crate_logging_level.insert("sqlx::query".to_owned(), log::Level::Warn); // shut up
     if config.DEBUG.unwrap_or(false) // setup logging, if DEBUG unset default to false
     {
-        setup_logging::setup_logging(log::Level::Debug, None, "./log/%Y-%m-%dT%H_%M.log");
+        setup_logging::setup_logging(log::Level::Debug, Some(crate_logging_level), "./log/%Y-%m-%dT%H_%M.log");
     }
     else
     {
-        setup_logging::setup_logging(log::Level::Info, None, "./log/%Y-%m-%d.log");
+        setup_logging::setup_logging(log::Level::Info, Some(crate_logging_level), "./log/%Y-%m-%d.log");
     }
 
     log::debug!("Loaded {config:?}."); // log loaded config

src/main_inner.rs

@@ -8,6 +8,8 @@ use crate::hentai::*;
 
 pub async fn main_inner(config: Config) -> Result<(), Error>
 {
+    const DB_FILEPATH: &str = "./db/db.sqlite"; // database filepath
+    const DB_MIGRATIONS_PATH: &str = "./db_migrations/"; // migrations path
     const HTTP_TIMEOUT: u64 = 30; // connection timeout
     const NHENTAI_HENTAI_SEARCH_URL: &str = "https://nhentai.net/api/gallery/"; // nhentai search by id api url
     const NHENTAI_TAG_SEARCH_URL: &str = "https://nhentai.net/api/galleries/search"; // nhentai search by tag api url
@@ -76,7 +78,7 @@ pub async fn main_inner(config: Config) -> Result<(), Error>
                 break 'iteration; // if server mode: only abort iteration, go straight to sleeping
             }
 
-            match connect_to_db(&config.DATABASE_URL).await // connect to database
+            match connect_to_db(DB_FILEPATH, DB_MIGRATIONS_PATH).await // connect to database
             {
                 Ok(o) => db = o,
                 Err(e) =>
@@ -132,7 +134,7 @@ pub async fn main_inner(config: Config) -> Result<(), Error>
 
 
             db.close().await; // close database connection
-            log::info!("Disconnected from database at \"{}\".", config.DATABASE_URL);
+            log::info!("Disconnected from database at \"{}\".", DB_FILEPATH);
 
             if config.NHENTAI_TAGS.is_none() {break 'program;} // if tag not set: client mode, exit