support multiple 'interface' values in .toc files (#430)

* toc.clj, introduces 'interface-version-list' field.
 it contains all possible interface versions parsed from the 'interface' field.
* toc.clj, split 'parse-addon-toc' into '-parse-addon-toc'.
  new private fn is less strict, interpreting and extrapolating values, warning on bad values etc.
  new fn does validation and discards data if it's invalid.
  this was done so we don't have to repeat parsing steps elsewhere.
* toc.clj, the templated '## # interface' value is now considered and included in the interface-version-list.
* github-api.clj, removed duplicated parsing of :interface values in favour of that in toc.clj
* github-api.clj, order of extracted gametrack values is now deterministic.
* utils, fixed bug in interface-version-to-game-version
 - '110000' was returning '1.0.000' instead of '11.0.0'
* toc.clj, fixed bug where data would fail to validate and be discarded.
 - it was possible for multiple distinct interface-version values to become duplicate game tracks values (like [:retail :retail])
* jfx.clj, the 'WoW' (interface version) column now supports multiple values.
 - default column width values increased to accommodate them.
* specs.clj, replaces toc :interface-version with :interface-version-list
* core.clj, fixed a typo in the import/export logic and the test helper that was using :toc/game-track instead of :-toc/game-track
* toc.clj, replaced :-toc/game-track with :-toc/game-track-list
* addon.clj, a single set of toc data can now belong to many game tracks
 - this means the first instance of 'classic' toc data (for example) will be used even if multiple toc files support classic. crazy.
* tukui, curseforge, removed their 'expand-summary' implementations as they were using :interface-version
* jfx.clj, addon data detail pane now accommodates many interface values.
* toc.clj, removed a warning when the presence of a game track in the filename doesn't match the game tracks parsed from the data.
 - the user can't really do anything about it, I had at least one false-positive and with many game tracks to now check it seems needlessly expensive.
* curseforge, tukui, removes test fixtures referenced by removed code.
* catalogue.clj, fixed possibilty of multiple game tracks in toc2summary
* specs, source-updates, removes 'release-label' and 'interface-version'.
 these appear to have been introduced for curseforge and only used there.
* linted
* review feedback
* CHANGELOG

CHANGELOG.md

@@ -8,12 +8,33 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+* added support for parsing multiple interface version values from .toc file.
+    - a single .toc file may now yield multiple possible game tracks (retail, classic, wrath, etc)
+        - including duplicate game tracks, which has been accounted for.
+
 ### Changed
 
+* the WoW column in the GUI now supports displaying multiple game versions (like 8.0, 9.0, 10.0).
+    - these are dervied from interface versions (80000, 90000, 100000)
+* the raw data GUI widget in the addon details pane now supports displaying multiple interface versions
+
 ### Fixed
 
+* fixed a typo in a field name in the import/export logic.
+    - almost certainly didn't affect anybody.
+* fixed a bug in the interface-version => game-version logic where 110000 was returned as '1.0.00' instead of '11.0.0'.
+
 ### Removed
 
+* removed curseforge and tukui test fixtures and the fetching of test fixtures.
+* removed support for 'release labels' attached to addon source updates.
+    - these were only supported by curseforge and provided a friendlier label a release than the version number.
+* removed more curseforge and tukui logic that was disabled many releases ago
+    - including test fixtures for this removed logic.
+* removed support for interface versions contianing non-numeric values.
+    - for example, there is an interface version "30008a" which should return "3.0.8a"
+    - this is an exceedingly rare (and possibly an invalid) case and nobody should be affected.
+
 ## 7.3.0 - 2024-04-23
 
 ### Added

TODO.md

@@ -6,9 +6,12 @@ see CHANGELOG.md for a more formal list of changes by release
 
 ## done
 
+* support multiple interface values
+
 ## todo
 
 ## todo bucket (no particular order)
+* remove support for parsing templated .toc keyvals: '# ## Key: Value'
 
 * gui, 'set-icon' is taking a long time to do it's thing.
 

manage.sh

@@ -74,15 +74,6 @@ elif test "$cmd" = "update-test-fixtures"; then
         echo "$fname"
     }
 
-    # curseforge
-    dl "https://www.curseforge.com/wow/addons?filter-sort=name&page=1" "curseforge-addon-summary-listing.html"
-
-    # curseforge api
-    dl "https://addons-ecs.forgesvc.net/api/v2/addon/327019" "curseforge-api-addon--everyaddon.json"
-    cp "test/fixtures/curseforge-api-addon--everyaddon.json" "test/fixtures/curseforge-api-addon--everyotheraddon.json"
-    ## one search result
-    dl "https://addons-ecs.forgesvc.net/api/v2/addon/search?gameId=1&index=0&pageSize=1&searchFilter=&sort=2" "curseforge-api-search--truncated.json"
-
     # wowinterface
     dl "https://wowinterface.com/addons.php" "wowinterface-category-list.html"
     dl "https://www.wowinterface.com/downloads/cat19.html" "wowinterface-category-page.html"
@@ -107,12 +98,6 @@ elif test "$cmd" = "update-test-fixtures"; then
     dl "https://gitlab.com/api/v4/projects/thing-engineering%2Fwowthing%2Fwowthing-collector/repository/blobs/125c899d813d2e11c976879f28dccc2a36fd207b" "gitlab-repo-blobs--wowthing.json"
     dl "https://gitlab.com/api/v4/projects/woblight%2Fnitro/repository/tree" "gitlab-repo-tree--woblight-nitro.json"
 
-    # tukui api
-    dl "https://www.tukui.org/api.php?addon=98" "tukui--addon-details.json"
-    dl "https://www.tukui.org/api.php?classic-addon=13" "tukui--classic-addon-details.json"
-    dl "https://www.tukui.org/api.php?ui=tukui" "tukui--tukui-addon-proper.json"
-    dl "https://www.tukui.org/api.php?ui=elvui" "tukui--elvui-addon-proper.json"
-
     # user-catalogue
     dl "https://api.github.com/repos/Stanzilla/AdvancedInterfaceOptions/releases" "user-catalogue--github.json"
 

src/strongbox/addon.clj

@@ -175,13 +175,13 @@
   (logging/with-addon {:dirname (-> addon-dir fs/base-name str)}
     (let [toc-data-list (toc/parse-addon-toc-guard addon-dir)]
       (if (= 1 (count toc-data-list))
-        ;; whatever toc data we have, we only have 1 of it (normal case), so return that
-        (-> toc-data-list first (dissoc :-toc/game-track))
+        ;; we only have 1 set of .toc data, so return that
+        (-> toc-data-list first (dissoc :-toc/game-track-list))
 
-        ;; we have multiple sets of toc data to choose from. which to choose?
+        ;; we have multiple sets of .toc data to choose from. which to choose?
         ;; prefer the one for the given `game-track`, if it exists, otherwise do as we do with
         ;; the catalogue and use a list of priorities.
-        (let [grouped-toc-data (group-by :-toc/game-track toc-data-list)
+        (let [grouped-toc-data (utils/group-by-coll :-toc/game-track-list toc-data-list)
               safe-fallback [game-track]
               priorities (get constants/game-track-priority-map game-track safe-fallback)
               group (utils/first-nn #(get grouped-toc-data %) priorities)]
@@ -193,7 +193,7 @@
                      (not (apply = group)))
             (debug (format "multiple sets of different toc data found for %s. using first." game-track)))
 
-          (-> group first (dissoc :-toc/game-track)))))))
+          (-> group first (dissoc :-toc/game-track-list)))))))
 
 (defn-spec load-all-installed-addons :addon/toc-list
   "reads and merges the toc data and the nfo data from *all* addons in the given `install-dir`, groups them and returns the grouped mooshed data."

src/strongbox/catalogue.clj

@@ -21,11 +21,11 @@
   does *not* support ignoring disabled hosts, see `expand-summary`.
   returns `nil` when no release found."
   [addon :addon/expandable, game-track ::sp/game-track]
-  (let [dispatch-map {"curse" curseforge-api/expand-summary
+  (let [dispatch-map {;;"curse" curseforge-api/expand-summary ;; 2024-07-15: removed
                       "wowin" wowinterface-api/expand-summary
                       "gitla" gitlab-api/expand-summary
                       "githu" github-api/expand-summary
-                      "tukui" tukui-api/expand-summary
+                      ;;"tukui" tukui-api/expand-summary ;; 2024-07-15: removed
                       nil (fn [_ _] (error "malformed addon:" (utils/pprint addon)))}
         key (utils/safe-subs (:source addon) 5)]
     (try
@@ -38,9 +38,7 @@
                                (strongbox.addon/find-pinned-release (assoc addon :release-list release-list)))
               source-updates (or pinned-release latest-release)]
           (when source-updates
-            (-> addon
-                (merge source-updates {:release-list release-list})
-                (dissoc :release-label)))))
+            (merge addon source-updates {:release-list release-list}))))
       (catch Exception e
         (error e (utils/reportable-error "unexpected error attempting to expand addon summary"))))))
 
@@ -104,7 +102,15 @@
           syn (if (= (:source toc) "wowinterface")
                 (cond
                   (:installed-game-track toc) (assoc syn :game-track-list [(:installed-game-track toc)])
-                  (:interface-version toc) (assoc syn :game-track-list [(utils/interface-version-to-game-track (:interface-version toc))])
+
+                  (not (empty? (:interface-version-list toc)))
+                  (assoc syn :game-track-list
+                         (->> toc
+                              :interface-version-list
+                              (map utils/interface-version-to-game-track)
+                              distinct
+                              vec))
+
                   :else sink)
                 syn)
 

src/strongbox/core.clj

@@ -1593,8 +1593,8 @@
                  ;; 2020-06: dirname must be a non-empty string
                  ;; todo: why is dirname needed here?
                  :dirname addon/dummy-dirname
-                 :interface-version constants/default-interface-version
-                 :toc/game-track :retail
+                 :interface-version-list [constants/default-interface-version]
+                 :-toc/game-track-list [:retail]
                  :supported-game-tracks []
                  :installed-version "0"}
         addon-list (map #(merge padding %) addon-list)

src/strongbox/curseforge_api.clj

@@ -1,175 +1,16 @@
 (ns strongbox.curseforge-api
   (:require
    [strongbox
-    [http :as http]
-    [specs :as sp]
-    [utils :as utils :refer [to-json join]]]
+    [specs :as sp]]
    [clojure.spec.alpha :as s]
-   [me.raynes.fs :as fs]
-   [orchestra.core :refer [defn-spec]]
-   [taoensso.timbre :as log :refer [debug info warn error spy]]))
+   [orchestra.core :refer [defn-spec]]))
 
 (def curseforge-api "https://addons-ecs.forgesvc.net/api/v2")
 
 (defn-spec api-url ::sp/url
   [path string?, & args (s/* any?)]
   (str curseforge-api (apply format path args)))
 
-;; addon expansion
-
-(defn-spec release-download-url (s/or :ok ::sp/url, :error nil?)
-  "returns a URL to the release file.
-  'https://edge.forgecdn.net/files/1234/567/Addon-v7.8.9.zip'"
-  [project-file-id int?, project-file-name string?]
-  (try
-    (let [project-file-id (str project-file-id) ;; "3164017"
-          offset (- (count project-file-id) 3) ;; "4"
-          bit1 (-> project-file-id (.substring 0 offset)) ;; "3164"
-          bit2 (-> project-file-id (.substring offset) (utils/ltrim "0"))] ;; 17 (strips leading zeroes)
-      (format "https://edge.forgecdn.net/files/%s/%s/%s" bit1 bit2 project-file-name))
-    (catch java.lang.StringIndexOutOfBoundsException e
-      (warn (format "failed to build a download url for release '%s'" project-file-name)))))
-
-(defn-spec rename-identical-releases ::sp/list-of-maps
-  "releases must have unique names otherwise we can't find them in the list of available releases.
-  This function assigns each release a `:-unique-name` that is either `:projectFileName` or
-  `:projectFileName` + `projectFileId`."
-  [release-list ::sp/list-of-maps]
-  (let [count-occurances (fn [accumulator-m m]
-                           (let [key :projectFileName]
-                             (update accumulator-m (get m key) (fn [x] (inc (or x 0))))))
-        occurances (reduce count-occurances {} release-list) ;; {"Foo-v1.zip" 1, "Foo-v2.zip 1, "Foo.zip" 5}
-        get* #(get %2 %1)
-        rename-release (fn [release]
-                         (let [[name _] (fs/split-ext (:projectFileName release))
-                               pid (:projectFileId release)]
-                           (assoc release :-unique-name
-                                  (if (-> release :projectFileName (get* occurances) (> 1))
-                                    (format "%s--%s" name pid) ;; "Foo--3084724"
-                                    name)))) ;; "Foo"
-        ]
-    (mapv rename-release release-list)))
-
-(defn-spec game-version-flavor-to-game-track ::sp/game-track
-  "given a curseforge 'gameVersionFlavor' value, returns the equivalent strongbox game track"
-  [game-version-flavor (s/nilable string?)]
-  (case game-version-flavor
-    "wow_retail" :retail
-    "wow_classic" :classic
-    "wow_burning_crusade" :classic-tbc
-
-    ;; else
-    (do (warn (format "unexpected game track '%s', falling back to 'retail'" game-version-flavor))
-        :retail)))
-
-(defn-spec older-releases :addon/release-list
-  "these are releases under `:gameVersionLatestFiles` that that are the most recent release by game/interface version (8.0.3 etc).
-  returns only stable releases."
-  [gameVersionLatestFiles ::sp/list-of-maps]
-  (let [stable 1 ;; 2 is beta, 3 is alpha
-        stable-releases #(-> % :fileType (= stable))
-        pad-release (fn [release]
-                      (when-let [download-url (release-download-url (:projectFileId release) (:projectFileName release))]
-                        {:download-url download-url
-                         :version (:-unique-name release) ;; see `rename-identical-releases`
-                         :release-label (format "[WoW %s] %s" (:gameVersion release) (:-unique-name release))
-                         :interface-version (utils/game-version-to-interface-version (:gameVersion release))
-                         :game-track (game-version-flavor-to-game-track (:gameVersionFlavor release))}))]
-
-    (->> gameVersionLatestFiles
-         (filter stable-releases)
-         rename-identical-releases
-         (map pad-release)
-         (remove nil?)
-         vec)))
-
-(defn-spec extract-release :addon/release-list
-  "for each release, set the correct value for `:gameVersionFlavor` and `:gameVersion`.
-  if `:gameVersion` is an empty list, use the value from `:gameVersionFlavor` to come up with a value.
-  return multiple instances of the release if necessary."
-  [release map?]
-  (let [;; "wow_retail", "wow_classic" => :retail, :classic
-        fallback (game-version-flavor-to-game-track (:gameVersionFlavor release))
-
-        ;; if the `:gameVersion` value is missing we assume this addon supports the latest version of `:retail`
-        ;; if we know the `gameVersionFlavor` we can switch that assumption to the latest version of `:classic`
-        release (if (empty? (:gameVersion release))
-                  (assoc release :gameVersion [(utils/game-track-to-latest-game-version fallback)])
-                  release)
-
-        ;; generate a release per-gametrack
-        pad-release (fn [game-version]
-                      (let [;; api value is empty in some cases (carbonite, improved loot frames, skada damage meter).
-                            ;; this value overrides the one found in .toc files, so if it can't be scraped, use the .toc version.
-                            interface-version (utils/game-version-to-interface-version game-version)
-                            interface-version (when interface-version
-                                                {:interface-version interface-version})
-                            [name _] (fs/split-ext (get release :fileName "null"))]
-                        (merge {:download-url (:downloadUrl release)
-                                :version (:displayName release)
-                                :release-label (format "[WoW %s] %s" (first (:gameVersion release)) name)
-                                :game-track (utils/game-version-to-game-track game-version)}
-                               interface-version)))
-
-        padded-release-list (mapv pad-release (:gameVersion release))
-
-        ;; 2021-04-02: curseforge is using a `:gameVersionFlavor` of `null` and `2.5.1` as the `:gameVersion` for WoW Classic (BC).
-        ;; this fn must return a valid `:addon/release-list` and that means `:game-track` cannot be anything other than `:classic` or `:retail`.
-        ;; so lets remove these releases until we can support them.
-        unknown-game-track (comp nil? :game-track)]
-    (vec (remove unknown-game-track padded-release-list))))
-
-(defn-spec prune-leading-duplicates (s/or :ok :addon/release-list, :garbage-in nil?)
-  "curseforge may produce the same release under `:latestFiles` and `:gameVersionLatestFiles`.
-  remove any `:gameVersionLatestFiles` releases in favour of releases from `:latestFiles`.
-  because we're duplicating releases by `:game-track`, assume two different releases may share 
-  the same download URL and only call this *after* a game track has been selected."
-  [release-list (s/nilable :addon/release-list)]
-  (if (empty? release-list)
-    nil
-    (let [[latest-release latest-release-by-game-version] release-list]
-      (if (= (:download-url latest-release)
-             (:download-url latest-release-by-game-version))
-        (concat [latest-release] (rest (rest release-list)))
-        release-list))))
-
-(defn-spec group-releases map?
-  "given a curseforge api result, returns a map of release data keyed by `game-track`."
-  [api-result (s/nilable map?)]
-
-  ;; issue #63: curseforge actually allow a release to be on both retail and classic game tracks.
-  ;; the single value under `gameVersionFlavor` is *inaccurate and misleading* and we can't trust it.
-  ;; instead we look at the `gameVersion` list and convert the versions we find there into game tracks.
-  ;; `8.2.0` and `8.2.5` => `retail`
-  ;; `1.13.2` => `classic`
-
-  ;; however! `gameVersion` is occasionally *empty* (see Adibags) and we have to guess which game track
-  ;; this release supports. In these cases we fall back to `:gameVersionFlavor`.
-
-  (let [;; results appear sorted, but lets be sure as we'll be taking the first
-        desc (comp - compare) ;; most to least recent (desc)
-        stable 1 ;; 2 is beta, 3 is alpha
-        stable-release #(-> % :releaseType (= stable))
-        concat* #(concat %2 %1)
-        more-releases (when-let [gameVersionLatestFiles (:gameVersionLatestFiles api-result)]
-                        (older-releases gameVersionLatestFiles))]
-    (->> api-result
-         :latestFiles
-         (sort-by :fileDate desc)
-         (filter stable-release)
-         (remove :exposeAsAlternative) ;; no alternative versions, for now
-         (map extract-release)
-         flatten
-         (concat* more-releases)
-         (group-by :game-track))))
-
-(defn-spec expand-summary (s/or :ok :addon/release-list, :error nil?)
-  "fetches a list of releases from the addon host for the given `addon-summary`"
-  [addon-summary :addon/expandable, game-track ::sp/game-track]
-  (let [url (api-url "/addon/%s" (:source-id addon-summary))
-        result (some-> url http/download-with-backoff http/sink-error utils/from-json)]
-    (-> result group-releases (get game-track) prune-leading-duplicates)))
-
 ;; catalogue building
 
 (defn-spec parse-user-string (s/or :ok ::sp/url, :error nil?)

src/strongbox/github_api.clj

@@ -192,20 +192,11 @@
 (defn-spec -find-gametracks-toc-data (s/or :ok ::sp/game-track-list, :error nil?)
   "returns a set of game tracks after inspecting .toc file contents"
   [toc-data map?]
-  (->> (-> toc-data
-           ;; hrm: this only allows for two possible game tracks, one normal and one hiding in the template area
-           ;; 2021-06-10: see release.json
-           (select-keys [:interface :#interface])
-           vals)
-
-       (map utils/to-int)
-       (map utils/interface-version-to-game-track)
-
-       ;; 2021-05-02: unknown game versions of 2.x (that are now considered "Classic (TBC)") were returning `nil` as the game track.
-       (remove nil?)
-       set
-       vec
-       utils/nilable))
+  (let [use-defaults false]
+    (-> toc-data
+        (toc/-parse-addon-toc use-defaults)
+        :supported-game-tracks
+        utils/nilable)))
 
 (defn-spec find-gametracks-toc-data (s/or :ok ::sp/game-track-list, :error nil?)
   "returns a set of game tracks after inspecting the .toc file contents"

src/strongbox/gitlab_api.clj

@@ -190,20 +190,18 @@
   "attempts to guess the game track of a [filename blob-url] pair.
   if the game track can't be guessed from the filename, it downlads the blob and inspects the interface version."
   [[filename blob-url] (s/coll-of string?)]
-  (if-let [game-track (utils/guess-game-track filename)]
-    [game-track]
-    (do (debug "couldn't guess game track, downloading toc file and inspecting interface version:" filename)
-        (some->> blob-url
-                 download-decode-blob
-                 (select-keys* [:interface :#interface])
-                 vals
-                 (map utils/to-int)
-                 (mapv utils/interface-version-to-game-track)))))
+  (let [game-track (utils/guess-game-track filename)
+        use-defaults false]
+    (if game-track
+      [game-track]
+      (do (debug "couldn't guess game track, downloading toc file and inspecting interface version:" filename)
+          (some-> blob-url
+                  download-decode-blob
+                  (toc/-parse-addon-toc use-defaults)
+                  :supported-game-tracks)))))
 
 (defn-spec guess-game-track-list (s/or :ok ::sp/game-track-list, :error nil?)
   "attempts to guess the game tracks an addon may support.
-  if multiple toc files exist it assumes they are being used for classic versions of the game.
-  if only a single toc file exists, it downloads and inspects the `:interface` value in the toc file.
   if no toc files are found it returns `nil`."
   [source-id :addon/source-id]
   (->> source-id