Merge pull request #138 from sfinlon/master

fix timestamp parsing to move to arrow > 0.15.2

csirtg_indicator/format/zstix.py

@@ -42,15 +42,15 @@ def _sha256(keypair):
             return f
 
         def _address_ipv4(address):
-            if re.search('^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}', address):
+            if re.search(r'^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?).){3}', address):
                 return 1
 
         def _address_fqdn(address):
-            if re.search('^[a-zA-Z0-9.\-_]+\.[a-z]{2,6}$', address):
+            if re.search(r'^[a-zA-Z0-9.-_]+.[a-z]{2,6}$', address):
                 return 1
 
         def _address_url(address):
-            if re.search('^(ftp|https?):\/\/', address):
+            if re.search(r'^(ftp|https?)://', address):
                 return 1
 
         def _address(keypair):

csirtg_indicator/utils/ztime.py

@@ -34,16 +34,28 @@ def parse_timestamp(ts):
                 ts = str(ts)
             if len(ts) == 8:
                 ts = '{}T00:00:00Z'.format(ts)
-                t = arrow.get(ts, 'YYYYMMDDTHH:mm:ss')
+                t = arrow.get(ts, 'YYYYMMDDTHH:mm:ssZ')
 
             if t.year < 1970:
-                raise RuntimeError('invalid timestamp: %s' % ts)
+                raise RuntimeError('a invalid timestamp: %s' % ts)
+
+        return t
+    except arrow.parser.ParserError as e:
+        t = arrow.get(ts, ['YYYY-MM-DD HH:mm:ss ZZZ', 'ddd, DD MMM YYYY HH:mm:ss Z'])
+        if t.year < 1980:
+            if type(ts) == datetime:
+                ts = str(ts)
+            if len(ts) == 8:
+                ts = '{}T00:00:00Z'.format(ts)
+                t = arrow.get(ts, 'YYYYMMDDTHH:mm:ssZ')
 
+            if t.year < 1970:
+                raise RuntimeError('invalid timestamp: %s' % ts)
         return t
 
-    except ValueError as e:
+    except arrow.parser.ParserError as e:
         if len(ts) == 14:
-            match = re.search('^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})$', ts)
+            match = re.search(r'^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})$', ts)
             if match:
                 ts = '{}-{}-{}T{}:{}:{}Z'.format(match.group(1), match.group(2), match.group(3), match.group(4),
                                                  match.group(5), match.group(6))
@@ -53,7 +65,7 @@ def parse_timestamp(ts):
                 raise RuntimeError('Invalid Timestamp: %s' % ts)
         if len(ts) == 16:
             # 20160219T224322Z
-            match = re.search('^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$', ts)
+            match = re.search(r'^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$', ts)
             if match:
                 ts = '{}-{}-{}T{}:{}:{}Z'.format(match.group(1), match.group(2), match.group(3), match.group(4),
                                                  match.group(5), match.group(6))
@@ -70,4 +82,4 @@ def parse_timestamp(ts):
         return t
 
     else:
-        raise RuntimeError('Invalid Timestamp: %s' % ts)
+        raise RuntimeError('Invalid Timestamp: %s' % ts)

requirements.txt

@@ -1,4 +1,4 @@
-arrow==0.14.2
+arrow>=0.15.2
 pytricia>=0.9.0
 ipaddress>=1.0.16
 pendulum>=0.5.2

setup.py

@@ -46,7 +46,7 @@
     author_email="[email protected]",
     packages=find_packages(),
     install_requires=[
-        'arrow==0.14.2',
+        'arrow>=0.15.2',
         'pytricia>=0.9.0',
         'ipaddress>=1.0.16',
         'pendulum==2.0.5',