Merge branch 'develop'

controlled_vocabulary/management/commands/vocab.py

@@ -68,7 +68,10 @@ def handle(self, *args, **options):
         action_method = getattr(self, "action_" + action, None)
         if action_method:
             show_help = False
-            action_method()
+            res = action_method()
+            if res not in [None, True]:
+                import sys
+                sys.exit(1)
 
         if show_help:
             self.stdout.write(self.help)
@@ -107,25 +110,37 @@ def action_managers(self):
     def action_update(self):
         vocs = self.app.write_vocabulary_records_from_managers()
         if self.options["verbosity"] > 0:
-            for voc in vocs.values():
-                self.stdout.write(voc.__module__)
+            if vocs:
+                for voc in vocs.values():
+                    self.stdout.write(voc.__module__)
+            else:
+                self.stdout.write('Please run django migrate.')
 
     def action_refetch(self):
-        self.action_fetch(True)
+        return self.action_fetch(True)
 
     def action_fetch(self, overwrite=False):
+        ret = True
+
         for voc in self._get_vocabularies():
             download_method = getattr(voc, "download", None)
             if download_method:
                 if self.options["verbosity"] > 0:
                     self.stdout.write(voc.prefix)
                 url, filepath, size, downloaded = download_method(overwrite=overwrite)
                 if self.options["verbosity"] > 0:
-                    self.stdout.write(
-                        "\t{}\n\t{}\n\t{:.3f}MB".format(
-                            url, filepath, size / 1024 / 1024
+                    if size > 0:
+                        self.stdout.write(
+                            "\t{}\n\t{}\n\t{:.3f}MB".format(
+                                url, filepath, size / 1024 / 1024
+                            )
                         )
-                    )
+                    else:
+                        self.stdout.write(f"ERROR: vocabulary download failed {url}.")
+                if size < 1:
+                    ret = False
+
+        return ret
 
     def action_init(self):
         self.action_update()

controlled_vocabulary/vocabularies/fast_topic.py

@@ -1,6 +1,7 @@
-from .base_http import VocabularyHTTP
 import re
 
+from .base_http import VocabularyHTTP
+
 
 class VocabularyFastTopic(VocabularyHTTP):
     # https://www.oclc.org/developer/develop/web-services/fast-api/linked-data.en.html
@@ -12,7 +13,7 @@ class VocabularyFastTopic(VocabularyHTTP):
     description = "Topic list from the Faceted Application of Subject Terminology"
     source = {
         # 'url': 'http://fast.oclc.org/searchfast/fastsuggest?query={query}&fl=suggest50&rows=10',
-        "url": "https://fast.oclc.org/searchfast/fastsuggest?&query={pattern}&queryIndex=suggest50&queryReturn=suggest50,id&sort=usage desc&suggest=fastSuggest",
+        "url": "https://fast.oclc.org/searchfast/fastsuggest?&query={pattern}&queryIndex=suggest50&queryReturn=suggest50,id&sort=usage desc&suggest=fastSuggest&rows=20",
         # https://www.oclc.org/research/themes/data-science/fast/download.html
         # 'url': 'https://researchworks.oclc.org/researchdata/fast/FASTTopical.nt.zip',
     }
@@ -30,5 +31,4 @@ def parse_search_response(self, res):
 
         for doc in res["response"]["docs"]:
             ret.append([self._get_clean_id(doc["id"]), doc["suggest50"]])
-
         return ret

controlled_vocabulary/vocabularies/iso15924.py

@@ -12,8 +12,7 @@ class VocabularyISO15924(VocabularyBaseCSV):
     concept = "wikidata:Q8192:writing system"
     description = "Codes for the Representation of names of scripts"
     source = {
-        "url": "https://www.unicode.org/iso15924/iso15924.txt.zip",
-        "extract": "iso15924-utf8-*.txt",
+        "url": "https://www.unicode.org/iso15924/iso15924.txt",
         "processed": "iso15924-utf8.txt",
         'delimiter': ';',
     }