cata support

TODO.md

@@ -19,7 +19,11 @@
 * move github catalogue generation from strongbox
     - having multiple catalogues will help the design adjust itself
 
-# todo 0.0.3 release
+# todo 
+
+* new example of all game tracks supports
+    - https://www.wowinterface.com/downloads/info12373
+
 
 * commit changes to catalogue and addon json once a day
     - need to be able to invoke a task like in strongbox

project.clj

@@ -32,11 +32,12 @@
                          ;; see `gui-diff` exclusion
                          [net.cgrand/parsley "0.9.3"]]
 
-  :repl-options {:init-ns scb.user}
+  :repl-options {:init-ns scb.user} ;; see repl/scb/user.clj
 
   :main scb.main
 
   ;;:profiles {:user {:plugins [[venantius/yagni "0.1.7"]]}}
+  :profiles {:repl {:source-paths ["repl"]}}
   :plugins [[jonase/eastwood "0.9.9"]
             [lein-cljfmt "0.8.0"]
             [lein-cloverage "1.2.4"]

repl/scb/user.clj

@@ -0,0 +1,151 @@
+(ns scb.user
+  (:refer-clojure :exclude [test])
+  (:require
+   [taoensso.timbre :as timbre :refer [debug info warn error spy]]
+   [clojure.test]
+   [gui.diff :refer [with-gui-diff]]
+   [scb
+    [main :as main]
+    [utils :as utils]
+    [http :as http]
+    [core :as core]])
+  (:import
+   [java.util.concurrent LinkedBlockingQueue]))
+
+(def ns-list [:core :utils
+              :tags
+              :wowi :github])
+
+(defn test
+  [& [ns-kw fn-kw]]
+  (core/stop)
+  ;;(clojure.tools.namespace.repl/refresh) ;; reloads all namespaces, including strongbox.whatever-test ones
+  (utils/instrument true) ;; always test with spec checking ON
+
+  (try
+    ;; note! remember to update `cloverage.clj` with any new bindings
+    (with-redefs [core/testing? true
+                  http/delay-between-requests 0
+                  ;;http/*default-pause* 1 ;; ms
+                  ;;http/*default-attempts* 1
+                  ;; don't pause while testing. nothing should depend on that pause happening.
+                  ;; note! this is different to `joblib/tick-delay` not delaying when `joblib/*tick*` is unbound.
+                  ;; tests still bind `joblib/*tick*` and run things in parallel.
+                  ;;joblib/tick-delay joblib/*tick*
+                  ;;main/spec? true
+                  ;;cli/install-update-these-in-parallel cli/install-update-these-serially
+                  ;;core/check-for-updates core/check-for-updates-serially
+                  ;; for testing purposes, no addon host is disabled
+                  ;;catalogue/host-disabled? (constantly false)
+                  ]
+      ;;(core/reset-logging!)
+
+      (core/init-logging)
+
+      (timbre/with-merged-config {:min-level :debug
+                                  :appenders {:println {:min-level :debug}
+                                              :spit {:enabled? false}}}
+        (if ns-kw
+          (if-not (some #{ns-kw} ns-list)
+            (error "unknown test file:" ns-kw)
+            (with-gui-diff
+              (if fn-kw
+                ;; `test-vars` will run the test but not give feedback if test passes OR test not found
+                ;; slightly better than nothing
+                (clojure.test/test-vars [(resolve (symbol (str "scb." (name ns-kw) "-test") (name fn-kw)))])
+                (clojure.test/run-all-tests (re-pattern (str "scb." (name ns-kw) "-test"))))))
+          (clojure.test/run-all-tests #"scb\..*-test"))))
+    (finally
+      ;; use case: we run the tests from the repl and afterwards we call `restart` to start the app.
+      ;; `stop` inside `restart` will be outside of `with-redefs` and still have logging `:min-level` set to `:debug`
+      ;; it will dump a file and yadda yadda.
+      ;;(core/reset-logging!)
+      nil)))
+
+(comment "this is probably the 'proper' way to do it, but it means I can't target specific tests/ns"
+         (defn -test
+           []
+           (try
+             (with-redefs [;;core/testing? true
+                  ;;http/*default-pause* 1 ;; ms
+                  ;;http/*default-attempts* 1
+                           ]
+               (with-gui-diff
+                 (clojure.test/run-all-tests #"scb\..*-test")))
+             (finally
+               nil)))
+
+         (defn test
+           []
+           (core/stop)
+           (clojure.tools.namespace.repl/refresh :after 'scb.user/-test)) ;; reloads all namespaces, including strongbox.whatever-test ones
+         )
+
+;; ---
+
+#_(defn wowi-html-landing
+    []
+    (clojure.pprint/pprint (->> "test/fixtures/wowinterface--landing.html" fs/absolute fs/normalized str wowi/to-html wowi/parse-category-group-page)))
+
+#_(defn wowi-html-listing-page
+    []
+    (let [html-snippet (->> "test/fixtures/wowinterface--listing.html" fs/absolute fs/normalized str slurp)
+          downloaded-item {:url "https://www.wowinterface.com/downloads/index.php?cid=100&sb=dec_date&so=desc&pt=f&page=1"
+                           :label "The Burning Crusade Classic"
+                           :response {:headers {}
+                                      :body html-snippet}}]
+      (wowi/parse-category-listing downloaded-item)))
+
+#_(defn wowi-html-addon-detail
+    []
+    (clojure.pprint/pprint
+     (->> "test/fixtures/wowinterface--addon-detail--multiple-downloads--no-tabber.html" fs/absolute fs/normalized str wowi/to-html wowi/parse-addon-detail-page)))
+
+#_(defn wowi-html-addon-detail-2
+    []
+    (clojure.pprint/pprint
+     (wowi/parse-addon-detail-page
+      {:url "https://www.wowinterface.com/downloads/info24155"
+       :response {:body (->> "test/fixtures/wowinterface--addon-detail--multiple-downloads--tabber.html"
+                             fs/absolute fs/normalized str slurp)}})))
+
+#_(defn wowi-api-addon-list
+    []
+    (->> (wowi/parse-api-file-list {:url wowi/api-file-list
+                                    :response (http/download wowi/api-file-list {})})
+         :parsed
+         (take 100)))
+
+#_(defn wowi-api-addon-detail
+    []
+    (let [resp (wowi/parse-api-addon-detail {:url "https://api.mmoui.com/v4/game/WOW/filedetails/5119.json"
+                                             :response (http/download "https://api.mmoui.com/v4/game/WOW/filedetails/5119.json" {})})]
+      (clojure.pprint/pprint resp)
+      (println "-----")
+      resp))
+
+;; ---
+
+(def stop core/stop)
+(def start core/start)
+(def restart core/restart)
+
+(defn status
+  []
+  (if-not (core/started?)
+    (warn "start app first: `(core/start)`")
+    (run! (fn [q-kw]
+            (println (format "%s items in %s" (.size ^LinkedBlockingQueue (core/get-state q-kw)) q-kw)))
+          core/queue-list)))
+
+#_(defn to-addon-summary
+    [source source-id]
+    (core/to-addon-summary (core/find-read-addon-data source source-id)))
+
+(defn parse-url-content
+  "downloads and parses the given URL, bypasses queues.
+  allows us to grab the exception object with `*e`"
+  [url]
+  (let [resp {:url url
+              :response (http/download url {})}]
+    (core/parse-content resp)))

src/scb/constants.clj

@@ -1,7 +1,7 @@
 (ns scb.constants)
 
-;; https://wow.gamepedia.com/Wowpedia
+;; https://warcraft.wiki.gg/wiki/Expansion#World_of_Warcraft
 (def release-of-previous-expansion
   "'Shadowlands', released November 23rd 2020. Used to shorten the 'full' catalogue.
-  https://en.wikipedia.org/wiki/World_of_Warcraft#Expansions"
+  https://warcraft.wiki.gg/wiki/Expansion#World_of_Warcraft"
   "2020-11-23T00:00:00Z")

src/scb/core.clj

@@ -327,30 +327,6 @@
 
 ;; ---
 
-(defn drain-queues
-  "drains all queues from a map of BlockingQueues into a map of vectors"
-  []
-  (info "draining queues")
-  (into {} (mapv (fn [qn]
-                   [qn (drain-queue (get-state qn))]) queue-list)))
-
-(defn freeze-state
-  [path]
-  (info "freezing state:" path)
-  (let [queue-state (drain-queues)]
-    (spit path queue-state)))
-
-(defn thaw-state
-  [path]
-  (when (fs/exists? path)
-    (info "thawing state:" path)
-    (let [old-state (read-string (slurp path))
-          lists-to-queues (into {}
-                                (mapv (fn [qn]
-                                        [qn (new-queue (get old-state qn))]) queue-list))]
-      (swap! state merge lists-to-queues))
-    nil))
-
 (defn run-worker
   [worker-fn]
   (let [f (future
@@ -478,7 +454,6 @@
   (alter-var-root #'state (constantly (-start)))
   (init-logging)
   (init-state-dirs)
-  ;;(thaw-state (paths :state-file-path))
   (run-worker download-worker)
   (run-many-workers parser-worker 5)
   (run-many-workers write-content-worker 5)
@@ -496,9 +471,7 @@
   (info "stopping")
   (if (nil? state)
     (info "(not started)")
-    (cleanup)
-    ;;(freeze-state (paths :state-file-path))
-    )
+    (cleanup))
   nil)
 
 (defn restart

src/scb/github.clj

@@ -12,7 +12,7 @@
 (defn-spec build-catalogue (s/or :ok :addon/summary-list, :error nil?)
   "converts a CSV list of addons to a strongbox-compatible catalogue of addon summaries."
   []
-  (let [url "https://raw.githubusercontent.com/ogri-la/github-wow-addon-catalogue/develop/addons.csv"
+  (let [url "https://raw.githubusercontent.com/ogri-la/github-wow-addon-catalogue-go/master/addons.csv"
         result (-> url
                    http/download-with-backoff
                    http/sink-error

src/scb/http.clj

@@ -17,9 +17,10 @@
    [java.io File FileInputStream FileOutputStream]
    [org.apache.commons.io IOUtils]))
 
-(def expiry-offset-hours 9999) ;; doesn't matter too much at this stage.
-(def delay-between-requests 1000) ;; ms, 1sec
+(def expiry-offset-hours 24)
+(def delay-between-requests 250) ;; ms
 
+;; retry logic. initial pause is *default-pause* and is doubled for each attempt.
 (def ^:dynamic *default-pause* 1000) ;; ms, 1sec
 (def ^:dynamic *default-attempts* 3)