avoided dummy metadata upon reading non-meta component file (#196)

* avoided dummy metadata upon reading non-meta component file

* cleanup testdata by generating non-metadata file within test

* fixes in formatting of MIDD and PERD lines of component file

hatyan/components.py

@@ -149,8 +149,8 @@ def write_components(comp, filename):
     tstart = metadata.pop('tstart')
     tstop = metadata.pop('tstop')
     tzone_min = metadata.pop('tzone')._minutes
-    tstart_str = tstart.strftime("%Y%m%d  %H%M%S")
-    tstop_str = tstop.strftime("%Y%m%d  %H%M%S")
+    tstart_str = tstart.strftime("%Y%m%d  %H%M")
+    tstop_str = tstop.strftime("%Y%m%d  %H%M")
 
     if 'A0' in comp.index.tolist():
         midd = comp.loc['A0','A']*100
@@ -205,7 +205,7 @@ def write_components(comp, filename):
         f.write(f'STAT  {station}    {grootheid}    {vertref}    {unit}    {waarnemingssoort}\n')
         f.write(f'PERD  {tstart_str}  {tstop_str}     {tzone_min}\n')
         f.write( 'CODE      3\n')
-        f.write(f'MIDD {midd:9.2f}\n')
+        f.write(f'MIDD {midd:9.3f}\n')
         f.write(f'NCOM {ncomp:5d}\n')
         for compname in comp.index.tolist():
             comp_one = comp.loc[compname]
@@ -368,7 +368,8 @@ def read_components(filename):
     print('reading file: %s'%(filename))
 
     line_compstart = None
-    stat_perd_available = False
+    stat_available = False
+    perd_available = False
     with open(filename) as f:
         for i, line in enumerate(f):
             if line.startswith('STAT'):
@@ -377,13 +378,13 @@ def read_components(filename):
                 vertref = line.split()[3]
                 eenheid = line.split()[4]
                 # waarnemingssoort = int(line.split()[5])
-                stat_perd_available = True
+                stat_available = True
             elif line.startswith('PERD'):
                 dateline = line.split()
                 tstart = pd.Timestamp(dateline[1]+' '+dateline[2])
                 tstop = pd.Timestamp(dateline[3]+' '+dateline[4])
                 tzone = pytz.FixedOffset(int(dateline[5]))
-                stat_perd_available = True
+                perd_available = True
             elif line.startswith('MIDD'):
                 A0_cm = float(line.split()[1])
             elif line.startswith('COMP'):
@@ -409,19 +410,12 @@ def read_components(filename):
     comp_pd = pd.DataFrame({'A': Aphi_datapd_raw['A'].values/100, 'phi_deg': Aphi_datapd_raw['phi'].values}, index=Aphi_datapd_raw['name'].values)
 
     # add metadata
-    if not stat_perd_available:
-        warnings.warn(UserWarning("No metadata available in component file (STAT/PERD lines), "
-                                  "you are probably using a component file generated with hatyan 2.7.0 or older. "
-                                  "Using dummy values for station/grootheid/vertref/tstart/tstop/tzone "
-                                  "to avoid issues in the hatyan process."))
-        station = "UNKNOWN"
-        grootheid = "WATHTE" # 'UNKNOWN' #TODO: this is not guaranteed, problem?
-        eenheid = "cm"
-        vertref = "NAP" # 'UNKNOWN' #TODO: this is not guaranteed, problem?
-        tstart = pd.Timestamp('9999-01-01')
-        tstop = pd.Timestamp('9999-01-01')
-        tzone = pytz.FixedOffset(60)
-
+    if not (stat_available & perd_available):
+        raise KeyError("No STAT/PERD metadata available in component file, "
+                       "you are probably using a component file generated with hatyan 2.7.0 or older. "
+                       "Add metadata to the header of the component file like this:\n"
+                       "STAT  DENHDR    WATHTE     NAP     cm    1\n"
+                       "PERD  20090101  0000  20121231  2300     60")
     metadata = {'station':station,
                 'grootheid':grootheid, 'eenheid':eenheid,
                 'vertref':vertref,

tests/test_components.py

@@ -8,10 +8,8 @@
 import os
 import pytest
 import numpy as np
-import pandas as pd
-import pytz
 import hatyan
-from hatyan.metadata import metadata_from_obj, metadata_add_to_obj
+from hatyan.metadata import metadata_from_obj
 
 dir_tests = os.path.dirname(__file__) #F9 doesnt work, only F5 (F5 also only method to reload external definition scripts)
 dir_testdata = os.path.join(dir_tests,'data_unitsystemtests')
@@ -39,26 +37,31 @@ def test_read_write_components():
 @pytest.mark.unittest
 def test_read_write_components_nometadata():
     """
-    test for component files written with hatyan 2.7.0 or lower
-    these files lack essential metadata for STAT, PERD and CODE lines
-    Dummy values are added in read_components, but in general these component files cannot be used for much things.
+    Test for component files written with hatyan 2.7.0 or lower,
+    these files lack essential metadata for STAT, PERD and CODE lines.
+    This tests tests whether the correct exception is raised.
     """
 
-    file_orig = os.path.join(dir_testdata,'DENHDR_ana_nometadata.txt')
-    file_new = "temp_comp_dummymetadata.txt"
-    file_dia_new = "temp_dia_dummymetadata.txt"
-    comp_orig = hatyan.read_components(file_orig)
-    assert len(comp_orig) == 95
-
-    hatyan.write_components(comp_orig, file_new)
-    comp_new = hatyan.read_components(filename=file_new)
-
-    ts_pred = hatyan.prediction(comp=comp_orig, times=slice("2020-01-01","2020-01-20",10))
-    hatyan.write_tsdia(ts=ts_pred, filename=file_dia_new)
-
-    assert np.allclose(comp_orig, comp_new)
-    os.remove(file_new)
-    os.remove(file_dia_new)
+    file_orig = os.path.join(dir_testdata,'DENHDR_ana.txt')
+    file_nometa = "temp_comp_dummymetadata.txt"
+    with open(file_orig, "r") as f:
+        data = f.readlines()
+    with open(file_nometa, "w") as f:
+        for line in data:
+            if line.startswith("STAT"):
+                continue
+            if line.startswith("PERD"):
+                continue
+            if line.startswith("CODE"):
+                continue
+            f.write(line)
+
+    try:
+        _ = hatyan.read_components(file_nometa)
+    except KeyError as e:
+        assert "No STAT/PERD metadata available in component file" in str(e)
+
+    os.remove(file_nometa)
 
 
 @pytest.mark.unittest