user-catalogue scheduled refreshes (#399)

* core, adds 'scheduled-user-catalogue-refresh' if a catalogue is more than 28 days old, it will trigger an automatic refresh. * cli, shifted a chunk of refresh-user-catalogue logic into core. * config, adds preference 'keep-user-catalogue-updated'. * jfx, adds 'keep user catalogue updated' toggle to preference menu * updated CHANGELOG
ogri-la · May 17, 2023 · c25c831 · c25c831
1 parent 27e4306
commit c25c831
Show file tree

Hide file tree

Showing 16 changed files with 396 additions and 168 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+* an (opt-in) automatic refesh of the user-catalogue every 28 days.
+    - see `Preferences` -> `Keep user catalogue updated`.
+    - the user-catalogue is a catalogue of the addons added to strongbox using `File` -> `Import addon` 
+        - or through 'starring' a regular catalogue addon.
 * a "clear" button to the search addons tab that removes all search filters, including search terms.
 * new column for installed addons "starred" that will add an installed addon to the 'user-catalogue'.
     - star button disabled when addon is being ignored or isn't matched against the catalogue.

diff --git a/TODO.md b/TODO.md
@@ -71,21 +71,17 @@ see CHANGELOG.md for a more formal list of changes by release
     - input star and dropdown are a few pixels shorter than the buttons on either side
     - done
 
-## todo
-
-* user-catalogue
-    - switch to log window to see progress when refresh-catalogue triggered
-
 * user catalogue, schedule refreshes
     - ensure the user catalogue doesn't get too stale and perform an update in the background if it looks like it's very old
         - update README
     - perhaps a preference?
+        - opt-in, opt-out?
+    - some task on startup that examines the catalogue's age and decides to do a refresh?
+        - would play havoc with testing.
+            - disable during testing?
+    - done
 
-* display github requests remaining
-    - ...
-
-* user catalogue, refreshing may guarantee exceeding github limit.
-    - if we know this, add a warning? refuse?
+## todo
 
 * 'core/state :db-stats', seems like a nice idea to put more information here
     - known-hosts
@@ -96,8 +92,19 @@ see CHANGELOG.md for a more formal list of changes by release
     - ...?
     - a button on the opposite of the log popup button but similar that will show some overall stats
 
+* user-catalogue
+    - switch to log window to see progress when refresh-catalogue triggered
+
+* display github requests remaining
+    - ...
+
+* user catalogue, refreshing may guarantee exceeding github limit.
+    - if we know this, add a warning? refuse?
+
 * gui, raw data, 'key' column too small for 'supported-game-tracks'
 
+* gui, installed, 'updated' column too small for 'NN months ago'
+
 * update screenshots
 
 ## todo bucket (no particular order)

diff --git a/cloverage.clj b/cloverage.clj
@@ -2,6 +2,7 @@
   (:require
    [strongbox
     [main :as main]
+    [constants :as constants]
     [utils :as utils]
     [http :as http]
     [joblib :as joblib]
@@ -24,7 +25,8 @@
                   http/*default-attempts* 1
                   ;;joblib/tick-delay joblib/*tick*
                   catalogue/host-disabled? (constantly false)
-                  utils/folder-size-bytes (constantly 0)]
+                  utils/folder-size-bytes (constantly 0)
+                  constants/max-user-catalogue-age 9999]
       (core/reset-logging!)
       (apply require (map symbol ns-list))
       {:errors (reduce + ((juxt :error :fail)

diff --git a/repl/strongbox/user.clj b/repl/strongbox/user.clj
@@ -5,6 +5,7 @@
    [clojure.test]
    [clojure.tools.namespace.repl :as tn :refer [refresh]]
    [strongbox
+    [constants :as constants]
     [main :as main :refer [restart stop]]
     [catalogue :as catalogue]
     [http :as http]
@@ -35,7 +36,9 @@
                   ;;core/check-for-updates core/check-for-updates-serially
                   ;; for testing purposes, no addon host is disabled
                   catalogue/host-disabled? (constantly false)
-                  utils/folder-size-bytes (constantly 0)]
+                  utils/folder-size-bytes (constantly 0)
+                  constants/max-user-catalogue-age 9999
+                  ]
       (core/reset-logging!)
 
       (if ns-kw

diff --git a/src/strongbox/cli.clj b/src/strongbox/cli.clj
@@ -7,7 +7,6 @@
    [me.raynes.fs :as fs]
    [strongbox
     [zip :as zip]
-    [constants :as constants]
     [joblib :as joblib]
     [github-api :as github-api]
     [tukui-api :as tukui-api]
@@ -17,7 +16,6 @@
     [logging :as logging]
     [addon :as addon]
     [specs :as sp]
-    [catalogue :as catalogue]
     [http :as http]
     [utils :as utils :refer [if-let* message-list]]
     [core :as core :refer [get-state paths]]]))
@@ -628,72 +626,12 @@
 
 ;; importing addons
 
-;; todo: logic might be better off in core.clj
-(defn-spec find-addon (s/or :ok :addon/summary, :error nil?)
-  "given a URL of a supported addon host, parses it, looks for it in the catalogue, expands addon and attempts to install a dry run installation.
-  if successful, returns the addon-summary.
-  dry-run installation attempt can be skipped by setting `attempt-dry-run` to false."
-  [addon-url string?, attempt-dry-run? boolean?]
-  (binding [http/*cache* (core/cache)]
-    (if-let* [addon-summary-stub (catalogue/parse-user-string addon-url)
-              source (:source addon-summary-stub)
-              match-on-list [[[:source :url] [:source :url]]
-                             [[:source :source-id] [:source :source-id]]]
-              addon-summary (cond
-                              (= source "github")
-                              (or (github-api/find-addon (:source-id addon-summary-stub))
-                                  (error (message-list
-                                          "Failed. URL must be:"
-                                          ["valid"
-                                           "originate from github.com"
-                                           "addon uses 'releases'"
-                                           "latest release has a packaged 'asset'"
-                                           "asset must be a .zip file"
-                                           "zip file must be structured like an addon"])))
-
-                              (= source "gitlab")
-                              (or (gitlab-api/find-addon (:source-id addon-summary-stub))
-                                  (error (message-list
-                                          "Failed. URL must be:"
-                                          ["valid"
-                                           "originate from gitlab.com"
-                                           "addon uses releases"
-                                           "latest release has a custom asset with a 'link'"
-                                           "link type must be either a 'package' or 'other'"])))
-
-                              (= source "curseforge")
-                              (error (str "addon host 'curseforge' was disabled " constants/curseforge-cutoff-label "."))
-
-                              :else
-                              ;; look in the current catalogue. emit an error if we fail
-                              (or (:catalogue-match (core/-find-first-in-db (or (core/get-state :db) []) addon-summary-stub match-on-list))
-                                  (error (format "couldn't find addon in catalogue '%s'"
-                                                 (name (core/get-state :cfg :selected-catalogue))))))
-
-              ;; game track doesn't matter when adding it to the user catalogue.
-              ;; prefer retail though (it's the most common) and `strict` here is `false`
-              addon (or (catalogue/expand-summary addon-summary :retail false)
-                        (error "failed to fetch details of addon"))
-
-              ;; a dry-run is good when importing an addon for the first time but
-              ;; not necessary when updating the user-catalogue.
-              _ (if-not attempt-dry-run?
-                  true
-                  (or (core/install-addon-guard addon (core/selected-addon-dir) true)
-                      (error "failed dry-run installation")))]
-
-             ;; if-let* was successful!
-             addon-summary
-
-             ;; failed if-let* :(
-             nil)))
-
 (defn-spec import-addon nil?
   "goes looking for given `addon-url` and, if found, adds it to the user catalogue and then installs it."
   [addon-url string?]
   (binding [http/*cache* (core/cache)]
     (if-let* [dry-run? true
-              addon-summary (find-addon addon-url dry-run?)
+              addon-summary (core/find-addon addon-url dry-run?)
               addon (core/expand-summary-wrapper addon-summary)]
              ;; success! add to user-catalogue and proceed to install
              (do (core/add-user-addon! addon-summary)
@@ -706,41 +644,9 @@
              ;; failed to find or expand summary, probably because of selected game track.
              nil)))
 
-(defn-spec refresh-user-catalogue-item nil?
-  "refresh the details of an individual `addon` in the user catalogue, optionally writing the updated catalogue to file."
-  [addon :addon/summary, db :addon/summary-list]
-  (logging/with-addon addon
-    (info "refreshing user-catalogue entry")
-    (try
-      (let [{:keys [source source-id url]} addon
-            refreshed-addon (core/db-addon-by-source-and-source-id db source source-id)
-            attempt-dry-run? false
-            refreshed-addon (or refreshed-addon
-                                (find-addon url attempt-dry-run?))]
-        (if-not refreshed-addon
-          (warn "failed to refresh user-catalogue entry as the addon was not found in the catalogue or online")
-          (core/add-user-addon! refreshed-addon)))
-
-      (catch Exception e
-        (error (format "an unexpected error happened while refreshing the user-catalogue entry: %s" (.getMessage e)))))))
-
-(defn-spec refresh-user-catalogue nil?
-  "refresh the details of all addons in the user catalogue, writing the updated catalogue to file once."
+(defn refresh-user-catalogue
   []
-  (binding [http/*cache* (core/cache)]
-    (let [path (fs/base-name (core/paths :user-catalogue-file)) ;; "user-catalogue.json"
-          ;; we can't assume the full-catalogue is available.
-          _ (core/download-catalogue (core/get-catalogue-location :full))
-          db (catalogue/read-catalogue (core/catalogue-local-path :full))
-          full-catalogue (or (:addon-summary-list db) [])]
-      (info (format "refreshing \"%s\", this may take a minute ..." path))
-      (doseq [user-addon (core/get-state :user-catalogue :addon-summary-list)]
-        (refresh-user-catalogue-item user-addon full-catalogue))
-      (core/write-user-catalogue!)
-      (info (format "\"%s\" has been refreshed" path))))
-  nil)
-
-;;
+  (core/refresh-user-catalogue))
 
 ;; todo: shift to core.clj or addon.clj
 (defn-spec addon-source-map-to-url (s/or :ok ::sp/url, :error nil?)

diff --git a/src/strongbox/config.clj b/src/strongbox/config.clj
@@ -44,7 +44,10 @@
 
                  ;; see `specs/column-preset-list` for selectable presets
                  ;; see `cli/column-map` for all known columns
-                 :ui-selected-columns sp/default-column-list}})
+                 :ui-selected-columns sp/default-column-list
+
+                 ;; refresh the user-catalogue every N days
+                 :keep-user-catalogue-updated false}})
 
 (defn handle-install-dir
   "`:install-dir` was once supported in the user configuration but is now only supported in the command line options.

diff --git a/src/strongbox/constants.clj b/src/strongbox/constants.clj
@@ -122,3 +122,5 @@
    "1.3" "World of Warcraft: Ruins of the Dire Maul"
    "1.2" "World of Warcraft: Mysteries of Maraudon"
    "1" "World of Warcraft"})
+
+(def max-user-catalogue-age 28)
diff --git a/src/strongbox/core.clj b/src/strongbox/core.clj
@@ -15,9 +15,11 @@
     [zip :as zip]
     [http :as http]
     [logging :as logging]
-    [utils :as utils :refer [join nav-map delete-many-files! expand-path if-let*]]
+    [utils :as utils :refer [join nav-map delete-many-files! expand-path if-let* message-list]]
     [catalogue :as catalogue]
     [specs :as sp]
+    [github-api :as github-api]
+    [gitlab-api :as gitlab-api]
     [joblib :as joblib]])
   (:import
    [org.apache.commons.compress.compressors CompressorStreamFactory CompressorException]
@@ -1188,6 +1190,116 @@
 
 ;;
 
+(defn-spec find-addon (s/or :ok :addon/summary, :error nil?)
+  "given a URL of a supported addon host, parses it, looks for it in the catalogue, expands addon and attempts to install a dry run installation.
+  if successful, returns the addon-summary.
+  dry-run installation attempt can be skipped by setting `attempt-dry-run` to false."
+  [addon-url string?, attempt-dry-run? boolean?]
+  (binding [http/*cache* (cache)]
+    (if-let* [addon-summary-stub (catalogue/parse-user-string addon-url)
+              source (:source addon-summary-stub)
+              match-on-list [[[:source :url] [:source :url]]
+                             [[:source :source-id] [:source :source-id]]]
+              addon-summary (cond
+                              (= source "github")
+                              (or (github-api/find-addon (:source-id addon-summary-stub))
+                                  (error (message-list
+                                          "Failed. URL must be:"
+                                          ["valid"
+                                           "originate from github.com"
+                                           "addon uses 'releases'"
+                                           "latest release has a packaged 'asset'"
+                                           "asset must be a .zip file"
+                                           "zip file must be structured like an addon"])))
+
+                              (= source "gitlab")
+                              (or (gitlab-api/find-addon (:source-id addon-summary-stub))
+                                  (error (message-list
+                                          "Failed. URL must be:"
+                                          ["valid"
+                                           "originate from gitlab.com"
+                                           "addon uses releases"
+                                           "latest release has a custom asset with a 'link'"
+                                           "link type must be either a 'package' or 'other'"])))
+
+                              (= source "curseforge")
+                              (error (str "addon host 'curseforge' was disabled " constants/curseforge-cutoff-label "."))
+
+                              :else
+                              ;; look in the current catalogue. emit an error if we fail
+                              (or (:catalogue-match (-find-first-in-db (or (get-state :db) []) addon-summary-stub match-on-list))
+                                  (error (format "couldn't find addon in catalogue '%s'"
+                                                 (name (get-state :cfg :selected-catalogue))))))
+
+              ;; game track doesn't matter when adding it to the user catalogue.
+              ;; prefer retail though (it's the most common) and `strict` here is `false`
+              addon (or (catalogue/expand-summary addon-summary :retail false)
+                        (error "failed to fetch details of addon"))
+
+              ;; a dry-run is good when importing an addon for the first time but
+              ;; not necessary when updating the user-catalogue.
+              _ (if-not attempt-dry-run?
+                  true
+                  (or (install-addon-guard addon (selected-addon-dir) true)
+                      (error "failed dry-run installation")))]
+
+             ;; if-let* was successful!
+             addon-summary
+
+             ;; failed if-let* :(
+             nil)))
+
+(defn-spec refresh-user-catalogue-item nil?
+  "refresh the details of an individual `addon` in the user catalogue, optionally writing the updated catalogue to file."
+  [addon :addon/summary, db :addon/summary-list]
+  (logging/with-addon addon
+    (info "refreshing user-catalogue entry")
+    (try
+      (let [{:keys [source source-id url]} addon
+            refreshed-addon (db-addon-by-source-and-source-id db source source-id)
+            attempt-dry-run? false
+            refreshed-addon (or refreshed-addon
+                                (find-addon url attempt-dry-run?))]
+        (if-not refreshed-addon
+          (warn "failed to refresh user-catalogue entry as the addon was not found in the catalogue or online")
+          (add-user-addon! refreshed-addon)))
+
+      (catch Exception e
+        (error (format "an unexpected error happened while refreshing the user-catalogue entry: %s" (.getMessage e)))))))
+
+(defn-spec refresh-user-catalogue nil?
+  "refresh the details of all addons in the user catalogue, writing the updated catalogue to file once."
+  []
+  (binding [http/*cache* (cache)]
+    (let [path (fs/base-name (paths :user-catalogue-file)) ;; "user-catalogue.json"
+          ;; we can't assume the full-catalogue is available.
+          _ (download-catalogue (get-catalogue-location :full))
+          db (catalogue/read-catalogue (catalogue-local-path :full))
+          full-catalogue (or (:addon-summary-list db) [])]
+      (info (format "refreshing \"%s\", this may take a minute ..." path))
+      (doseq [user-addon (get-state :user-catalogue :addon-summary-list)]
+        (refresh-user-catalogue-item user-addon full-catalogue))
+      (write-user-catalogue!)
+      (info (format "\"%s\" has been refreshed" path))))
+  nil)
+
+(defn-spec refresh-user-catalogue? boolean?
+  "predicate, returns `true` if the user-catalogue needs a refresh."
+  [keep-user-catalogue-updated? boolean?, catalogue-datestamp (s/nilable ::sp/inst)]
+  (and keep-user-catalogue-updated?
+       catalogue-datestamp
+       (utils/older-than? catalogue-datestamp constants/max-user-catalogue-age :days)))
+
+(defn-spec scheduled-user-catalogue-refresh nil?
+  "checks the loaded database and calls `refresh-user-catalogue` if it's considered too old."
+  []
+  (when (refresh-user-catalogue? (get-state :cfg :preferences :keep-user-catalogue-updated)
+                                 (get-state :user-catalogue :datestamp))
+    (info (format "user-catalogue not updated in the last %s days, automatic refresh triggered." constants/max-user-catalogue-age))
+    (refresh-user-catalogue)))
+
+;;
+
 (defn-spec init-dirs nil?
   "ensure all directories in `generate-path-map` exist and are writable, creating them if necessary.
   this logic depends on paths that are not generated until the application has been started."
@@ -1507,6 +1619,8 @@
    ;; seems like a good place to preserve the etag-db
   (save-settings!)
 
+  (scheduled-user-catalogue-refresh)
+
   nil)
 
 (defn refresh-check