project.clj, moves dev/test dependencies into :repl/:dev profiles.

moves 'test' from 'main' into repl/strongbox/user.clj. core.clj, replaces bz2 compressed catalogue with a simple string. two reasons: 1. it wasn't working at compile time like I thought. 2. regular strings are more compressible ultimately when building an AppImage.
ogri-la · Apr 2, 2023 · 54eb7fb · 54eb7fb
1 parent ced25d7
commit 54eb7fb
Show file tree

Hide file tree

Showing 8 changed files with 103 additions and 125 deletions.
diff --git a/TODO.md b/TODO.md
@@ -8,13 +8,33 @@ see CHANGELOG.md for a more formal list of changes by release
 
 ## todo
 
-* gui, raw data, 'key' column too small for 'supported-game-tracks'
+* 'downloading strongbox data' shouldn't be blocking the gui from starting
+    - is the version check happening async too?
+
+* user catalogue, what is happening now that regular, non-github, addons can be favourited?
+    - do they need to have their details refreshed?
+    - also, we have a github catalogue now, I bet the majority of these updates can be pulled directly from catalogues.
+        - it looks like refreshing the user catalogue is actually finding the addon in the catalogue, expandinging it and then installing it
+            - I don't think it should be installed!
+                - at least, finding and installing should be separate steps
+                    - for example, import-addon should find-addon then install-addon
+                    - and refresh-user-catalogue-item should be find-addon and then update-catalogue
+                        - and doesn't the catalogue it's looing in already contain the user-catalogue?
+
+* user catalogue, schedule refreshes
+    - ensure the user catalogue doesn't get too stale and perform an update in the background if it looks like it's very old
+        - update README
+    - perhaps a preference?
+
+* user catalogue, refreshing may guarantee exceeding github limit.
+    - if we know this, add a warning? refuse?
+
+* user catalogue, add a 'add to user-catalogue' option to make an addon always available despite selected catalogue
+    - done, via favouriting, but! it's not available on the installed addon pane page
 
 * search, a 'clear' button
     - resets favourited, search input, tags, etc
 
-* 'downloading strongbox data' shouldn't be blocking the gui from starting
-
 * size of addon on disk
     - I'd like to see a column with 'size on disk'
         - "1024KiB", "1MiB", "1.04MB"
@@ -35,20 +55,7 @@ see CHANGELOG.md for a more formal list of changes by release
 * display github requests remaining
     - ...
 
-* user catalogue, schedule refreshes
-    - ensure the user catalogue doesn't get too stale and perform an update in the background if it looks like it's very old
-        - update README
-    - perhaps a preference?
-
-* user catalogue, what is happening now that regular, non-github, addons can be favourited?
-    - do they need to have their details refreshed?
-    - also, we have a github catalogue now, I bet the majority of these updates can be pulled directly from catalogues.
-
-* user catalogue, refreshing my guarantee exceeding github limit.
-    - if we know this, add a warning? refuse?
-
-* user catalogue, add a 'add to user-catalogue' option to make an addon always available despite selected catalogue
-    - done, via favouriting, but! it's not available on the installed addon pane page
+* gui, raw data, 'key' column too small for 'supported-game-tracks'
 
 ## todo bucket (no particular order)
 

diff --git a/project.clj b/project.clj
@@ -25,11 +25,9 @@
                  [org.flatland/ordered "1.5.9"] ;; better ordered map
                  [clojure.java-time "0.3.3"] ;; date/time handling library, https://github.com/dm3/clojure.java-time
                  [envvar "1.1.2"] ;; environment variable wrangling
-                 [gui-diff "0.6.7" :exclusions [net.cgrant/parsley]] ;; pops up a graphical diff for test results
-                 [com.taoensso/tufte "2.2.0"] ;; profiling
                  [tolitius/lasync "0.1.23"] ;; better parallel processing
 
-                 [cljfx "1.7.19" :exclusions [org.openjfx/javafx-web
+                 [cljfx "1.7.22" :exclusions [org.openjfx/javafx-web
                                               org.openjfx/javafx-media]]
                  [cljfx/css "1.1.0"]
 
@@ -47,7 +45,6 @@
 
                  ;; GPLv3 compatible dependencies.
                  ;; these don't need an exception in LICENCE.txt
-                 [org.apache.commons/commons-compress "1.21"] ;; Apache 2.0 licenced, bz2 compression/decompression of static catalogue
                  [org.ocpsoft.prettytime/prettytime "5.0.2.Final"] ;; Apache 2.0 licenced, pretty date formatting
                  [org.controlsfx/controlsfx "11.1.1"] ;; BSD-3
 
@@ -66,16 +63,22 @@
 
   :resource-paths ["resources"]
 
-  :profiles {:dev {:plugins [[lein-ancient "0.7.0"]]
+  :profiles {:repl {:source-paths ["repl"]}
+             :dev {:plugins [[lein-ancient "0.7.0"]]
                    :resource-paths ["dev-resources" "resources"] ;; dev-resources take priority
                    :dependencies [[clj-http-fake "1.0.3"] ;; fake http responses for testing
+                                  [gui-diff "0.6.7" :exclusions [net.cgrant/parsley]] ;; pops up a graphical diff for test results
                                   ]}
 
              :uberjar {:aot :all
                        ;; fixes hanging issue:
                        ;; - https://github.com/cljfx/cljfx/issues/17
                        :injections [(javafx.application.Platform/exit)]
-                       }}
+                       :jvm-opts ["-Dclojure.compiler.disable-locals-clearing=true"
+                                  "-Dclojure.compiler.elide-meta=[:doc :file :line :added]"
+                                  "-Dclojure.compiler.direct-linking=true"]}}
+
+  :repl-options {:init-ns strongbox.user} ;; see repl/strongbox/user.clj
 
   ;; debug output from JavaFX about which GTK it is looking for. 
   ;; was useful in figuring out why javafx was failing to initialise even with xvfb.
@@ -97,5 +100,6 @@
                            ;;:unused-fn-args ;; prefer to keep for readability
                            ;;:keyword-typos ;; bugged with spec?
                            ]
-             :only-modified true}
+             :only-modified true
+             }
   )
diff --git a/repl/strongbox/user.clj b/repl/strongbox/user.clj
@@ -0,0 +1,59 @@
+(ns strongbox.user
+  (:refer-clojure :rename {test clj-test})
+  (:require
+   [taoensso.timbre :as timbre :refer [spy info warn error report]]
+   [clojure.test]
+   [clojure.tools.namespace.repl :as tn :refer [refresh]]
+   [strongbox
+    [main :as main]
+    [catalogue :as catalogue]
+    [http :as http]
+    [core :as core]
+    [utils :as utils :refer [in?]]
+    ;;[cli :as cli]
+    ]
+   [gui.diff :refer [with-gui-diff]]
+   ))
+
+(defn test
+  [& [ns-kw fn-kw]]
+  (main/stop)
+  (clojure.tools.namespace.repl/refresh) ;; reloads all namespaces, including strongbox.whatever-test ones
+  (utils/instrument true) ;; always test with spec checking ON
+
+  (try
+    ;; note! remember to update `cloverage.clj` with any new bindings
+    (with-redefs [core/*testing?* true
+                  http/*default-pause* 1 ;; ms
+                  http/*default-attempts* 1
+                  ;; don't pause while testing. nothing should depend on that pause happening.
+                  ;; note! this is different to `joblib/tick-delay` not delaying when `joblib/*tick*` is unbound.
+                  ;; tests still bind `joblib/*tick*` and run things in parallel.
+                  ;;joblib/tick-delay joblib/*tick*
+                  ;;main/*spec?* true
+                  ;;cli/install-update-these-in-parallel cli/install-update-these-serially
+                  ;;core/check-for-updates core/check-for-updates-serially
+                  ;; for testing purposes, no addon host is disabled
+                  catalogue/host-disabled? (constantly false)]
+      (core/reset-logging!)
+
+      (if ns-kw
+        (if (some #{ns-kw} [:main :utils :http
+                            :core :toc :nfo :zip :config :catalogue :addon :logging :joblib
+                            :cli :gui :jfx
+                            :curseforge-api :wowinterface-api :gitlab-api :github-api :tukui-api
+                            :release-json])
+          (with-gui-diff
+            (if fn-kw
+              ;; `test-vars` will run the test but not give feedback if test passes OR test not found
+              ;; slightly better than nothing
+              (clojure.test/test-vars [(resolve (symbol (str "strongbox." (name ns-kw) "-test") (name fn-kw)))])
+              (clojure.test/run-all-tests (re-pattern (str "strongbox." (name ns-kw) "-test"))))
+            )
+          (error "unknown test file:" ns-kw))
+        (clojure.test/run-all-tests #"strongbox\..*-test")))
+    (finally
+      ;; use case: we run the tests from the repl and afterwards we call `restart` to start the app.
+      ;; `stop` inside `restart` will be outside of `with-redefs` and still have logging `:min-level` set to `:debug`
+      ;; it will dump a file and yadda yadda.
+      (core/reset-logging!))))
diff --git a/src/strongbox/catalogue.clj b/src/strongbox/catalogue.clj
@@ -3,7 +3,6 @@
    [clojure.spec.alpha :as s]
    [orchestra.core :refer [defn-spec]]
    [taoensso.timbre :as log :refer [debug info warn error spy]]
-   [taoensso.tufte :as tufte :refer [p]]
    [java-time]
    [strongbox
     [constants :as constants]
@@ -130,8 +129,7 @@
 (defn-spec format-catalogue-data :catalogue/catalogue
   "returns a correctly formatted, ordered, catalogue given a list of addons and a datestamp"
   [addon-list :addon/summary-list, datestamp ::sp/ymd-dt]
-  (let [addon-list (p :cat/sort-addons
-                      (sort-by :name addon-list))]
+  (let [addon-list (sort-by :name addon-list)]
     {:spec {:version 2}
      :datestamp datestamp
      :total (count addon-list)
@@ -160,8 +158,7 @@
 (defn validate
   "validates the given data as a `:catalogue/catalogue`, returning nil if data is invalid"
   [catalogue]
-  (p :catalogue:validate
-     (sp/valid-or-nil :catalogue/catalogue catalogue)))
+  (sp/valid-or-nil :catalogue/catalogue catalogue))
 
 (defn-spec write-catalogue (s/or :ok ::sp/extant-file, :error nil?)
   "write catalogue to given `output-file` as JSON. returns path to output file"

diff --git a/src/strongbox/core.clj b/src/strongbox/core.clj
@@ -32,7 +32,7 @@
 (def static-catalogue
   "a bz2 compressed copy of the full catalogue used when the remote catalogue is unavailable or corrupt.
   from this we can do per-host filtering as well as shortening to generate the other catalogues."
-  (utils/compressed-slurp "full-catalogue.json"))
+  (utils/compile-time-slurp "full-catalogue.json"))
 
 (defn generate-path-map
   "filesystem paths whose location may vary based on the current working directory, environment variables, etc.
@@ -674,7 +674,7 @@
   "derives the requested catalogue from the static catalogue."
   [catalogue-location :catalogue/location]
   (let [opts {}
-        catalogue (catalogue/read-catalogue (.getBytes (utils/decompress-bytes static-catalogue)) opts)
+        catalogue (catalogue/read-catalogue (.getBytes static-catalogue) opts)
         catalogue (assoc catalogue :emergency? true)]
 
     (warn (utils/message-list (format "the remote catalogue is unreachable or corrupt: %s" (:source catalogue-location))

diff --git a/src/strongbox/main.clj b/src/strongbox/main.clj
@@ -1,24 +1,17 @@
 (ns strongbox.main
-  (:refer-clojure :rename {test clj-test})
   (:require
    [taoensso.timbre :as timbre :refer [spy info warn error report]]
-   [clojure.test]
    [clojure.tools.cli]
-   [clojure.tools.namespace.repl :as tn :refer [refresh]]
    [clojure.string :refer [lower-case]]
    [me.raynes.fs :as fs]
    [strongbox
-    [catalogue :as catalogue]
-    [http :as http]
-    ;;[joblib :as joblib]
     [core :as core]
     [utils :as utils :refer [in?]]
     ;; warning! requiring cljfx starts the javafx application thread.
     ;; this is a pita for exiting a non-javafx UI (cli) and aot as it just 'hangs'.
     ;; hanging aot is handled in project.clj, but dynamic inclusion of jfx is handled here.
     ;;[jfx :as jfx] 
-    [cli :as cli]]
-   [gui.diff :refer [with-gui-diff]])
+    [cli :as cli]])
   (:gen-class))
 
 (Thread/setDefaultUncaughtExceptionHandler
@@ -78,56 +71,12 @@
   (Thread/sleep 750) ;; gives me time to switch panes
   (start cli-opts))
 
-(defn test
-  [& [ns-kw fn-kw]]
-  (stop)
-  (clojure.tools.namespace.repl/refresh) ;; reloads all namespaces, including strongbox.whatever-test ones
-  (utils/instrument true) ;; always test with spec checking ON
-
-  (try
-    ;; note! remember to update `cloverage.clj` with any new bindings
-    (with-redefs [core/*testing?* true
-                  http/*default-pause* 1 ;; ms
-                  http/*default-attempts* 1
-                  ;; don't pause while testing. nothing should depend on that pause happening.
-                  ;; note! this is different to `joblib/tick-delay` not delaying when `joblib/*tick*` is unbound.
-                  ;; tests still bind `joblib/*tick*` and run things in parallel.
-                  ;;joblib/tick-delay joblib/*tick*
-                  ;;main/*spec?* true
-                  ;;cli/install-update-these-in-parallel cli/install-update-these-serially
-                  ;;core/check-for-updates core/check-for-updates-serially
-                  ;; for testing purposes, no addon host is disabled
-                  catalogue/host-disabled? (constantly false)]
-      (core/reset-logging!)
-
-      (if ns-kw
-        (if (some #{ns-kw} [:main :utils :http
-                            :core :toc :nfo :zip :config :catalogue :addon :logging :joblib
-                            :cli :gui :jfx
-                            :curseforge-api :wowinterface-api :gitlab-api :github-api :tukui-api
-                            :release-json])
-          (with-gui-diff
-            (if fn-kw
-              ;; `test-vars` will run the test but not give feedback if test passes OR test not found
-              ;; slightly better than nothing
-              (clojure.test/test-vars [(resolve (symbol (str "strongbox." (name ns-kw) "-test") (name fn-kw)))])
-              (clojure.test/run-all-tests (re-pattern (str "strongbox." (name ns-kw) "-test")))))
-          (error "unknown test file:" ns-kw))
-        (clojure.test/run-all-tests #"strongbox\..*-test")))
-    (finally
-      ;; use case: we run the tests from the repl and afterwards we call `restart` to start the app.
-      ;; `stop` inside `restart` will be outside of `with-redefs` and still have logging `:min-level` set to `:debug`
-      ;; it will dump a file and yadda yadda.
-      (core/reset-logging!))))
-
 ;;
 
 (defn usage
   [parsed-opts]
   (str "Usage: ./strongbox [--action] [--addons-dir]\n\n" (:summary parsed-opts)))
 
-;;
-
 (def cli-options
   [["-h" "--help"]
 

diff --git a/src/strongbox/utils.clj b/src/strongbox/utils.clj
@@ -794,27 +794,7 @@
     (or (get constants/releases major-minor)
         (get constants/releases major))))
 
-(defn-spec compressed-slurp (s/or :ok bytes?, :no-resource nil?)
-  "returns the bz2 compressed bytes of the given resource file `resource`.
-  returns `nil` if the file can't be found."
-  [resource string?]
-  (let [input-file (clojure.java.io/resource resource)]
-    (when input-file
-      (with-open [out (java.io.ByteArrayOutputStream.)]
-        (with-open [cos (.createCompressorOutputStream (CompressorStreamFactory.) CompressorStreamFactory/BZIP2, out)]
-          (clojure.java.io/copy (clojure.java.io/input-stream input-file) cos))
-        ;; compressed output stream (cos) needs to be closed to flush any remaining bytes
-        (.toByteArray out)))))
-
-(defn-spec decompress-bytes (s/or :ok string?, :nil-or-empty-bytes nil?)
-  "decompresses the given `bz2-bytes` as bz2, returning a string.
-  if bytes are empty or `nil`, returns `nil`.
-  if bytes are not bz2 compressed, throws an `IOException`."
-  [bz2-bytes (s/nilable bytes?)]
-  (when-not (empty? bz2-bytes)
-    (with-open [is (clojure.java.io/input-stream bz2-bytes)]
-      (try
-        (with-open [cin (.createCompressorInputStream (CompressorStreamFactory.) CompressorStreamFactory/BZIP2, is)]
-          (slurp cin))
-        (catch CompressorException ce
-          (throw (.getCause ce)))))))
+(defmacro compile-time-slurp
+  [resource]
+  `(slurp (clojure.java.io/resource ~resource)))
+
diff --git a/test/strongbox/utils_test.clj b/test/strongbox/utils_test.clj
@@ -564,21 +564,3 @@
     (doseq [[given expected] cases]
       (is (= expected (utils/patch-name given))))))
 
-(deftest compressed-slurp
-  (let [expected "foo!"]
-    (is (= expected (utils/decompress-bytes (utils/compressed-slurp "foo.txt"))))))
-
-(deftest compressed-slurp--not-found
-  (is (nil? (utils/compressed-slurp "file-that-definitely-does-not-exist.txt"))))
-
-(deftest decompress-bytes--empty
-  (is (nil? (utils/decompress-bytes (.getBytes ""))))
-  (is (nil? (utils/decompress-bytes nil))))
-
-(deftest decompress-bytes--not-compressed
-  (let [result (try
-                 (utils/decompress-bytes (.getBytes "!"))
-                 (catch java.io.IOException ioe
-                   ioe))]
-    (is (instance? java.io.IOException result))
-    (is (= "Stream is not in the BZip2 format" (.getMessage result)))))