Remove LSF, convert staticdata files to split files

pyfa-org · Jun 28, 2020 · 5c7e3fe · 5c7e3fe
1 parent 546a3b6
commit 5c7e3fe
Show file tree

Hide file tree

Showing 44 changed files with 3,923,788 additions and 69 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -33,6 +33,4 @@ pyfa.py     text eol=lf
 *.jpg binary
 *.icns binary
 *.ico binary
-staticdata/fsd_binary/* filter=lfs diff=lfs merge=lfs -text
-staticdata/fsd_lite/* filter=lfs diff=lfs merge=lfs -text
-staticdata/phobos/* filter=lfs diff=lfs merge=lfs -text
+
diff --git a/db_update.py b/db_update.py
@@ -93,13 +93,25 @@ def update_db():
     eos.db.gamedata_meta.create_all()
 
     def _readData(minerName, jsonName, keyIdName=None):
-        with open(os.path.join(JSON_DIR, minerName, '{}.json'.format(jsonName)), encoding='utf-8') as f:
-            rawData = json.load(f)
+        compiled_data = None
+        for i in itertools.count(0):
+            try:
+                with open(os.path.join(JSON_DIR, minerName, '{}.{}.json'.format(jsonName, i)), encoding='utf-8') as f:
+                    rawData = json.load(f)
+                    if i == 0:
+                        compiled_data = {} if type(rawData) == dict else []
+                    if type(rawData) == dict:
+                        compiled_data.update(rawData)
+                    else:
+                        compiled_data.extend(rawData)
+            except FileNotFoundError:
+                break
+
         if not keyIdName:
             return rawData
         # IDs in keys, rows in values
         data = []
-        for k, v in rawData.items():
+        for k, v in compiled_data.items():
             row = {}
             row.update(v)
             if keyIdName not in row:
@@ -325,24 +337,27 @@ def convertSection(sectionData):
 
         newData = []
         for row in data:
-            newRow = {
-                'typeID': row['typeID'],
-            }
-            for (k, v) in eos.config.translation_mapping.items():
-                if v == '':
-                    v = '_en-us'
-                typeLines = []
-                traitData = row['traits{}'.format(v)]
-                for skillData in sorted(traitData.get('skills', ()), key=lambda i: i['header']):
-                    typeLines.append(convertSection(skillData))
-                if 'role' in traitData:
-                    typeLines.append(convertSection(traitData['role']))
-                if 'misc' in traitData:
-                    typeLines.append(convertSection(traitData['misc']))
-                traitLine = '<br />\n<br />\n'.join(typeLines)
-                newRow['traitText{}'.format(v)] = traitLine
-
-            newData.append(newRow)
+            try:
+                newRow = {
+                    'typeID': row['typeID'],
+                }
+                for (k, v) in eos.config.translation_mapping.items():
+                    if v == '':
+                        v = '_en-us'
+                    typeLines = []
+                    traitData = row['traits{}'.format(v)]
+                    for skillData in sorted(traitData.get('skills', ()), key=lambda i: i['header']):
+                        typeLines.append(convertSection(skillData))
+                    if 'role' in traitData:
+                        typeLines.append(convertSection(traitData['role']))
+                    if 'misc' in traitData:
+                        typeLines.append(convertSection(traitData['misc']))
+                    traitLine = '<br />\n<br />\n'.join(typeLines)
+                    newRow['traitText{}'.format(v)] = traitLine
+
+                newData.append(newRow)
+            except:
+                pass
         _addRows(newData, eos.gamedata.Traits, fieldMap={'traitText_en-us': 'display'})
 
     def processMetadata():