diff --git a/README.md b/README.md index ebccf9d..478863d 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ with engines['csv_timeseries'].open( filename=TEST_FILE, filters={'countries': {include=['NO']}} ) as ts: + ts.read() for var in ts.variables(): # stations ts.data(var).stations @@ -79,6 +80,7 @@ import pyaro.timeseries TEST_FILE = "csvReader_testdata.csv" engine = pyaro.list_timeseries_engines()["csv_timeseries"] ts = engine.open(TEST_FILE, filters=[], fill_country_flag=False) +ts.read() print(ts.variables()) # stations ts.data('SOx').stations diff --git a/setup.cfg b/setup.cfg index 326e9b3..a4281eb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = pyaro -version = 0.0.13.dev0 +version = 0.0.14.dev0 author = Heiko Klein, Daniel Heinesen author_email = Heiko.Klein@met.no description = pyaro py-aerocom reader objects diff --git a/src/pyaro/csvreader/CSVTimeseriesReader.py b/src/pyaro/csvreader/CSVTimeseriesReader.py index 07120f8..ae569f1 100644 --- a/src/pyaro/csvreader/CSVTimeseriesReader.py +++ b/src/pyaro/csvreader/CSVTimeseriesReader.py @@ -2,11 +2,12 @@ import glob import logging import os +from datetime import datetime import numpy as np import pyaro.timeseries.AutoFilterReaderEngine -from pyaro.timeseries import Data, Flag, NpStructuredData, Station +from pyaro.timeseries import Data, NpStructuredData, Station logger = logging.getLogger(__name__) @@ -35,26 +36,26 @@ class CSVTimeseriesReader(pyaro.timeseries.AutoFilterReaderEngine.AutoFilterRead ) def __init__( - self, - filename, - columns={ - "variable": 0, - "station": 1, - "longitude": 2, - "latitude": 3, - "value": 4, - "units": 5, - "start_time": 6, - "end_time": 7, - "altitude": "0", - "country": "NO", - "standard_deviation": "NaN", - "flag": "0", - }, - variable_units={"SOx": "Gg", "NOx": "Mg"}, - country_lookup=False, - csvreader_kwargs={"delimiter": ","}, - filters=[], + self, + filename, + columns={ + "variable": 0, + "station": 1, + "longitude": 2, + "latitude": 3, + "value": 4, + "units": 5, + "start_time": 6, + "end_time": 7, + "altitude": "0", + "country": "NO", + "standard_deviation": "NaN", + "flag": "0", + }, + variable_units={"SOx": "Gg", "NOx": "Mg"}, + country_lookup=False, + csvreader_kwargs={"delimiter": ","}, + filters=[], ): """open a new csv timeseries-reader @@ -86,17 +87,41 @@ def __init__( self._set_filters(filters) self._extra_metadata = tuple(set(columns.keys()) - set(self.col_keys())) if country_lookup: - lookupISO2 = _lookup_function() + self._lookupISO2 = _lookup_function() else: - lookupISO2 = None + self._lookupISO2 = None + self._filename = filename + self._columns = columns + self._variable_units = variable_units + self._csvreader_kwargs = csvreader_kwargs + + def read(self): + """read method""" + for path in self._file_iterator: - logger.debug("%s: %s", filename, path) + logger.debug("%s: %s", self._filename, path) self._read_single_file( - path, columns, variable_units, lookupISO2, csvreader_kwargs + path, self._columns, self._variable_units, self._lookupISO2, self._csvreader_kwargs ) + def read_revisiondate(self, filename): + """quick way of getting the revision date""" + if os.path.isdir(filename): + filename = "glob:" + filename + "/*.csv" + if filename.startswith("glob:"): + _file_iterator = glob.iglob(filename[5:], recursive=True) + else: + return datetime.fromtimestamp(os.path.getmtime(filename)) + + newest_date = datetime.fromtimestamp(0) + for path in _file_iterator: + filedate = datetime.fromtimestamp(os.path.getmtime(path)) + if filedate > newest_date: + newest_date = filedate + return newest_date + def _read_single_file( - self, filename, columns, variable_units, country_lookup, csvreader_kwargs + self, filename, columns, variable_units, country_lookup, csvreader_kwargs ): with open(filename, newline="") as csvfile: crd = csv.reader(csvfile, **csvreader_kwargs) @@ -115,11 +140,11 @@ def _read_single_file( extra_metadata[t] = row[columns[t]] for t in ( - "value", - "latitude", - "longitude", - "altitude", - "standard_deviation", + "value", + "latitude", + "longitude", + "altitude", + "standard_deviation", ): r[t] = float(r[t]) for t in ("start_time", "end_time"): @@ -208,3 +233,9 @@ def description(self): def url(self): return "https://github.com/metno/pyaro" + + def read(self): + return self.reader_class().read(*args, **kwargs) + + def read_revisiondate(self, filename): + return self.reader_class().read_revisiondate(self, filename) diff --git a/src/pyaro/timeseries/Engine.py b/src/pyaro/timeseries/Engine.py index 6610293..d81b181 100644 --- a/src/pyaro/timeseries/Engine.py +++ b/src/pyaro/timeseries/Engine.py @@ -14,6 +14,16 @@ def open(self, filename_or_obj_or_url, *, filters=None): """ pass + @abc.abstractmethod + def read(self): + """read-method of the timeseries + + :return pyaro.timeseries.Reader + :raises UnknownFilterException + """ + # yield self + pass + @property @abc.abstractmethod def args(self) -> list[str]: diff --git a/src/pyaro/timeseries/Reader.py b/src/pyaro/timeseries/Reader.py index b2e6da0..c3e542f 100644 --- a/src/pyaro/timeseries/Reader.py +++ b/src/pyaro/timeseries/Reader.py @@ -1,14 +1,14 @@ import abc + from .Data import Data from .Station import Station -from .Filter import Filter, filters class Reader(abc.ABC): """Baseclass for timeseries. This can be used with a context manager""" @abc.abstractmethod - def __init__(self, filename_or_obj_or_url, *, filters=None): + def __init__(self, filename_or_obj_or_url, filters=None, **kwargs): """Initialize the reader. This function is usually called from the Engine's open function. @@ -19,6 +19,18 @@ def __init__(self, filename_or_obj_or_url, *, filters=None): """ pass + @abc.abstractmethod + def read(self): + """define read method. All needed parameters should be put into self + by the __init__ method + + This function is usually called after the Engine's open function. + Should implement context manager + """ + # yield self + pass + + @abc.abstractmethod def metadata(self) -> dict[str, str]: """Metadata set by the datasource. diff --git a/src/pyaro/timeseries/Wrappers.py b/src/pyaro/timeseries/Wrappers.py index 809f808..efa67f8 100644 --- a/src/pyaro/timeseries/Wrappers.py +++ b/src/pyaro/timeseries/Wrappers.py @@ -1,10 +1,9 @@ from .Reader import Reader -from .Data import Data class VariableNameChangingReader(Reader): """A pyaro.timeseries.Reader wrapper taking a real Reader implementation and - changing variable names in the original reader. Exampel: + changing variable names in the original reader. Example: with VariableNameChangingReader(pyaro.open_timeseries(file, filters=[]), {'SOx': 'oxidised_sulphur'}) as ts: @@ -14,7 +13,7 @@ class VariableNameChangingReader(Reader): """ - def __init__(self, reader: Reader, reader_to_new: dict[str, str]): + def __init__(self, reader: Reader, reader_to_new: dict[str, str], **kwargs, ): """Initialize the variable name changes of Reader :param reader: The Reader instance to change variable names on @@ -45,6 +44,15 @@ def data(self, varname): data._set_variable(varname) return data + def metadata(self): + """Get the metadata from the reader + NOT changing the variable name to the newly given ones for the moment + + :return: metadata from the original reader class + """ + metadata = self._reader.metadata() + return metadata + def stations(self): return self._reader.stations() @@ -57,3 +65,11 @@ def variables(self): def close(self): self._reader.close() + + def read(self, ): + """define read method. All needed parameters should be put into self + by the __init__ method + + This method is called after the Engine's open function. + """ + return self._reader.read() diff --git a/tests/test_CSVTimeSeriesReader.py b/tests/test_CSVTimeSeriesReader.py index 81f14e7..705b86a 100644 --- a/tests/test_CSVTimeSeriesReader.py +++ b/tests/test_CSVTimeSeriesReader.py @@ -1,10 +1,10 @@ import datetime import logging +import os import sys import unittest -import os +from datetime import datetime -import numpy as np import pyaro import pyaro.timeseries from pyaro.timeseries.Filter import FilterException @@ -57,6 +57,7 @@ def test_init(self): engine.description() engine.args() with engine.open(self.file, filters=[]) as ts: + ts.read() count = 0 for var in ts.variables(): count += len(ts.data(var)) @@ -70,6 +71,7 @@ def test_init_multifile(self): engine.description() engine.args() with engine.open(self.multifile, filters=[]) as ts: + ts.read() count = 0 for var in ts.variables(): count += len(ts.data(var)) @@ -83,6 +85,7 @@ def test_init_directory(self): engine.description() engine.args() with engine.open(self.multifile_dir, filters=[]) as ts: + ts.read() count = 0 for var in ts.variables(): count += len(ts.data(var)) @@ -91,14 +94,22 @@ def test_init_directory(self): def test_init2(self): with pyaro.open_timeseries( - "csv_timeseries", *[self.file], **{"filters": []} + "csv_timeseries", *[self.file], **{"filters": []} ) as ts: + ts.read() count = 0 for var in ts.variables(): count += len(ts.data(var)) self.assertEqual(count, 208) self.assertEqual(len(ts.stations()), 2) + def test_revision_date(self): + engine = pyaro.list_timeseries_engines()["csv_timeseries"] + bla = engine.read_revisiondate(self.file) + blubb = engine.read_revisiondate(self.multifile) + self.assertGreater(bla, datetime.fromtimestamp(0)) + self.assertGreater(blubb, datetime.fromtimestamp(0)) + def test_init_extra_columns(self): columns = { "variable": 0, @@ -116,8 +127,9 @@ def test_init_extra_columns(self): "area_classification": 8, } with pyaro.open_timeseries( - "csv_timeseries", *[self.file], **{"filters": [], "columns": columns} + "csv_timeseries", *[self.file], **{"filters": [], "columns": columns} ) as ts: + ts.read() areas = ["Rural", "Urban"] stations = ts.stations() self.assertEqual(stations["station1"]["area_classification"], areas[0]) @@ -125,17 +137,19 @@ def test_init_extra_columns(self): def test_metadata(self): with pyaro.open_timeseries( - "csv_timeseries", *[self.file], **{"filters": []} + "csv_timeseries", *[self.file], **{"filters": []} ) as ts: + ts.read() self.assertIsInstance(ts.metadata(), dict) self.assertIn("path", ts.metadata()) def test_data(self): engines = pyaro.list_timeseries_engines() with engines["csv_timeseries"].open( - filename=self.file, - filters=[pyaro.timeseries.filters.get("countries", include=["NO"])], + filename=self.file, + filters=[pyaro.timeseries.filters.get("countries", include=["NO"])], ) as ts: + ts.read() for var in ts.variables(): # stations ts.data(var).stations @@ -158,9 +172,10 @@ def test_data(self): def test_append_data(self): engines = pyaro.list_timeseries_engines() with engines["csv_timeseries"].open( - filename=self.file, - filters={"countries": {"include": ["NO"]}}, + filename=self.file, + filters={"countries": {"include": ["NO"]}}, ) as ts: + ts.read() var = next(iter(ts.variables())) data = ts.data(var) old_size = len(data) @@ -178,13 +193,14 @@ def test_append_data(self): standard_deviation=data.standard_deviations, ) self.assertEqual( - (2**rounds) * old_size, len(data), "data append by array" + (2 ** rounds) * old_size, len(data), "data append by array" ) def test_stationfilter(self): engine = pyaro.list_timeseries_engines()["csv_timeseries"] sfilter = pyaro.timeseries.filters.get("stations", exclude=["station1"]) with engine.open(self.file, filters=[sfilter]) as ts: + ts.read() count = 0 for var in ts.variables(): count += len(ts.data(var)) @@ -202,6 +218,7 @@ def test_boundingboxfilter(self): ) self.assertEqual(sfilter.init_kwargs()["include"][0][3], 0) with engine.open(self.file, filters=[sfilter]) as ts: + ts.read() count = 0 for var in ts.variables(): count += len(ts.data(var)) @@ -212,6 +229,7 @@ def test_boundingboxfilter(self): ) self.assertEqual(sfilter.init_kwargs()["exclude"][0][3], -180) with engine.open(self.file, filters=[sfilter]) as ts: + ts.read() count = 0 for var in ts.variables(): count += len(ts.data(var)) @@ -239,6 +257,7 @@ def test_timebounds(self): self.assertIsInstance(dt1, datetime.datetime) self.assertIsInstance(dt2, datetime.datetime) with engine.open(self.file, filters=[tfilter]) as ts: + ts.read() count = 0 for var in ts.variables(): count += len(ts.data(var)) @@ -258,6 +277,7 @@ def test_flagfilter(self): ffilter.init_kwargs()["include"][0], pyaro.timeseries.Flag.VALID ) with engine.open(self.file, filters=[ffilter]) as ts: + ts.read() count = 0 for var in ts.variables(): count += len(ts.data(var)) @@ -268,6 +288,7 @@ def test_flagfilter(self): "flags", include=[pyaro.timeseries.Flag.INVALID] ) with engine.open(self.file, filters=[ffilter]) as ts: + ts.read() count = 0 for var in ts.variables(): count += len(ts.data(var)) @@ -288,6 +309,7 @@ def test_variable_time_station_filter(self): ) engine = pyaro.list_timeseries_engines()["csv_timeseries"] with engine.open(self.file, filters=[vtsfilter]) as ts: + ts.read() count = 0 for var in ts.variables(): count += len(ts.data(var)) @@ -311,6 +333,7 @@ def test_variable_time_station_filter_csv(self): ) engine = pyaro.list_timeseries_engines()["csv_timeseries"] with engine.open(self.file, filters=[vtsfilter]) as ts: + ts.read() count = 0 for var in ts.variables(): count += len(ts.data(var)) @@ -321,8 +344,9 @@ def test_wrappers(self): engine = pyaro.list_timeseries_engines()["csv_timeseries"] newsox = "oxidised_sulphur" with VariableNameChangingReader( - engine.open(self.file, filters=[]), {"SOx": newsox} + engine.open(self.file, filters=[]), {"SOx": newsox} ) as ts: + ts.read() self.assertEqual(ts.data(newsox).variable, newsox) pass @@ -333,45 +357,50 @@ def test_variables_filter(self): "variables", reader_to_new={"SOx": newsox} ) with engine.open(self.file, filters=[vfilter]) as ts: + ts.read() self.assertEqual(ts.data(newsox).variable, newsox) pass def test_duplicate_filter(self): engine = pyaro.list_timeseries_engines()["csv_timeseries"] with engine.open( - self.multifile_dir + "/csvReader_testdata2.csv", - filters={"duplicates": {"duplicate_keys": None}}, + self.multifile_dir + "/csvReader_testdata2.csv", + filters={"duplicates": {"duplicate_keys": None}}, ) as ts: + ts.read() self.assertEqual(len(ts.data("NOx")), 8) with engine.open( - self.multifile_dir + "/csvReader_testdata2.csv", - filters={ - "duplicates": {"duplicate_keys": ["stations", "start_times", "values"]} - }, + self.multifile_dir + "/csvReader_testdata2.csv", + filters={ + "duplicates": {"duplicate_keys": ["stations", "start_times", "values"]} + }, ) as ts: + ts.read() self.assertEqual(len(ts.data("NOx")), 10) def test_time_resolution_filter(self): engine = pyaro.list_timeseries_engines()["csv_timeseries"] with self.assertRaises(FilterException): with engine.open( - self.file, - filters={"time_resolution": {"resolutions": ["ldjf4098"]}}, + self.file, + filters={"time_resolution": {"resolutions": ["ldjf4098"]}}, ) as ts: pass with engine.open( - self.file, - filters={"time_resolution": {"resolutions": ["1 day"]}}, + self.file, + filters={"time_resolution": {"resolutions": ["1 day"]}}, ) as ts: + ts.read() count = 0 for var in ts.variables(): count += len(ts.data(var)) self.assertEqual(count, 208) for resolution in "1 minute, 1 hour, 1week, 1month, 3year".split(","): with engine.open( - self.file, - filters={"time_resolution": {"resolutions": ["1 hour"]}}, + self.file, + filters={"time_resolution": {"resolutions": ["1 hour"]}}, ) as ts: + ts.read() count = 0 for var in ts.variables(): count += len(ts.data(var)) @@ -384,9 +413,10 @@ def test_filterFactory(self): def test_filterCollection(self): with pyaro.open_timeseries( - "csv_timeseries", - filename=self.file, + "csv_timeseries", + filename=self.file, ) as ts: + ts.read() filters = pyaro.timeseries.FilterCollection( { "countries": {"include": ["NO"]}, @@ -400,8 +430,9 @@ def test_filterCollection(self): @unittest.skipUnless(has_pandas, "no pandas installed") def test_timeseries_data_to_pd(self): with pyaro.open_timeseries( - "csv_timeseries", *[self.file], **{"filters": []} + "csv_timeseries", *[self.file], **{"filters": []} ) as ts: + ts.read() count = 0 vars = list(ts.variables()) data = ts.data(vars[0]) @@ -413,8 +444,9 @@ def test_timeseries_data_to_pd(self): @unittest.skipUnless(has_geocode, "geocode-reverse-natural-earth not available") def test_country_lookup(self): with pyaro.open_timeseries( - "csv_timeseries", *[self.file], **{"filters": [], "country_lookup": True} + "csv_timeseries", *[self.file], **{"filters": [], "country_lookup": True} ) as ts: + ts.read() count = 0 vars = list(ts.variables()) data = ts.data(vars[0]) @@ -423,89 +455,95 @@ def test_country_lookup(self): def test_altitude_filter_1(self): engines = pyaro.list_timeseries_engines() with engines["csv_timeseries"].open( - filename=self.elevation_file, - filters=[pyaro.timeseries.filters.get("altitude", max_altitude=150)], - columns={ - "variable": 0, - "station": 1, - "longitude": 2, - "latitude": 3, - "value": 4, - "units": 5, - "start_time": 6, - "end_time": 7, - "altitude": 9, - "country": "NO", - "standard_deviation": "NaN", - "flag": "0", - } + filename=self.elevation_file, + filters=[pyaro.timeseries.filters.get("altitude", max_altitude=150)], + columns={ + "variable": 0, + "station": 1, + "longitude": 2, + "latitude": 3, + "value": 4, + "units": 5, + "start_time": 6, + "end_time": 7, + "altitude": 9, + "country": "NO", + "standard_deviation": "NaN", + "flag": "0", + } ) as ts: + ts.read() self.assertEqual(len(ts.stations()), 1) def test_altitude_filter_2(self): engines = pyaro.list_timeseries_engines() with engines["csv_timeseries"].open( - filename=self.elevation_file, - filters=[pyaro.timeseries.filters.get("altitude", min_altitude=250)], - columns={ - "variable": 0, - "station": 1, - "longitude": 2, - "latitude": 3, - "value": 4, - "units": 5, - "start_time": 6, - "end_time": 7, - "altitude": 9, - "country": "NO", - "standard_deviation": "NaN", - "flag": "0", - } + filename=self.elevation_file, + filters=[pyaro.timeseries.filters.get("altitude", min_altitude=250)], + columns={ + "variable": 0, + "station": 1, + "longitude": 2, + "latitude": 3, + "value": 4, + "units": 5, + "start_time": 6, + "end_time": 7, + "altitude": 9, + "country": "NO", + "standard_deviation": "NaN", + "flag": "0", + } ) as ts: + ts.read() self.assertEqual(len(ts.stations()), 1) def test_altitude_filter_3(self): engines = pyaro.list_timeseries_engines() with engines["csv_timeseries"].open( - filename=self.elevation_file, - filters=[pyaro.timeseries.filters.get("altitude", min_altitude=150, max_altitude=250)], - columns={ - "variable": 0, - "station": 1, - "longitude": 2, - "latitude": 3, - "value": 4, - "units": 5, - "start_time": 6, - "end_time": 7, - "altitude": 9, - "country": "NO", - "standard_deviation": "NaN", - "flag": "0", - } + filename=self.elevation_file, + filters=[pyaro.timeseries.filters.get("altitude", min_altitude=150, max_altitude=250)], + columns={ + "variable": 0, + "station": 1, + "longitude": 2, + "latitude": 3, + "value": 4, + "units": 5, + "start_time": 6, + "end_time": 7, + "altitude": 9, + "country": "NO", + "standard_deviation": "NaN", + "flag": "0", + } ) as ts: + ts.read() self.assertEqual(len(ts.stations()), 1) def test_relaltitude_filter_emep_1(self): engines = pyaro.list_timeseries_engines() with engines["csv_timeseries"].open( - filename=self.elevation_file, - filters=[pyaro.timeseries.filters.get("relaltitude", topo_file = "./tests/testdata/datadir_elevation/topography.nc", rdiff=0)], - columns={ - "variable": 0, - "station": 1, - "longitude": 2, - "latitude": 3, - "value": 4, - "units": 5, - "start_time": 6, - "end_time": 7, - "altitude": 9, - "country": "NO", - "standard_deviation": "NaN", - "flag": "0", - } + filename=self.elevation_file, + filters=[pyaro.timeseries.filters.get("relaltitude", + topo_file="./tests/testdata/datadir_elevation/topography.nc", + rdiff=0)], + columns={ + "variable": 0, + "station": 1, + "longitude": 2, + "latitude": 3, + "value": 4, + "units": 5, + "start_time": 6, + "end_time": 7, + "altitude": 9, + "country": "NO", + "standard_deviation": "NaN", + "flag": "0", + } ) as ts: + ts.read() # Altitudes in test dataset: # Station | Alt_obs | Modeobs | rdiff | # Station 1 | 100 | 12.2554 | 87.7446 | @@ -517,118 +555,132 @@ def test_relaltitude_filter_emep_1(self): def test_relaltitude_filter_emep_2(self): engines = pyaro.list_timeseries_engines() with engines["csv_timeseries"].open( - filename=self.elevation_file, - filters=[pyaro.timeseries.filters.get("relaltitude", topo_file = "./tests/testdata/datadir_elevation/topography.nc", rdiff=90)], - columns={ - "variable": 0, - "station": 1, - "longitude": 2, - "latitude": 3, - "value": 4, - "units": 5, - "start_time": 6, - "end_time": 7, - "altitude": 9, - "country": "NO", - "standard_deviation": "NaN", - "flag": "0", - } + filename=self.elevation_file, + filters=[pyaro.timeseries.filters.get("relaltitude", + topo_file="./tests/testdata/datadir_elevation/topography.nc", + rdiff=90)], + columns={ + "variable": 0, + "station": 1, + "longitude": 2, + "latitude": 3, + "value": 4, + "units": 5, + "start_time": 6, + "end_time": 7, + "altitude": 9, + "country": "NO", + "standard_deviation": "NaN", + "flag": "0", + } ) as ts: + ts.read() # At rdiff = 90, only the first station should be included. self.assertEqual(len(ts.stations()), 1) def test_relaltitude_filter_emep_3(self): engines = pyaro.list_timeseries_engines() with engines["csv_timeseries"].open( - filename=self.elevation_file, - filters=[pyaro.timeseries.filters.get("relaltitude", topo_file = "./tests/testdata/datadir_elevation/topography.nc", rdiff=300)], - columns={ - "variable": 0, - "station": 1, - "longitude": 2, - "latitude": 3, - "value": 4, - "units": 5, - "start_time": 6, - "end_time": 7, - "altitude": 9, - "country": "NO", - "standard_deviation": "NaN", - "flag": "0", - } + filename=self.elevation_file, + filters=[pyaro.timeseries.filters.get("relaltitude", + topo_file="./tests/testdata/datadir_elevation/topography.nc", + rdiff=300)], + columns={ + "variable": 0, + "station": 1, + "longitude": 2, + "latitude": 3, + "value": 4, + "units": 5, + "start_time": 6, + "end_time": 7, + "altitude": 9, + "country": "NO", + "standard_deviation": "NaN", + "flag": "0", + } ) as ts: + ts.read() # Since rdiff=300, all stations should be included. self.assertEqual(len(ts.stations()), 3) def test_relaltitude_filter_1(self): engines = pyaro.list_timeseries_engines() with engines["csv_timeseries"].open( - filename=self.elevation_file, - filters=[pyaro.timeseries.filters.get("relaltitude", topo_file = "./tests/testdata/datadir_elevation/topography.nc", rdiff=0)], - columns={ - "variable": 0, - "station": 1, - "longitude": 2, - "latitude": 3, - "value": 4, - "units": 5, - "start_time": 6, - "end_time": 7, - "altitude": 9, - "country": "NO", - "standard_deviation": "NaN", - "flag": "0", - } + filename=self.elevation_file, + filters=[pyaro.timeseries.filters.get("relaltitude", + topo_file="./tests/testdata/datadir_elevation/topography.nc", + rdiff=0)], + columns={ + "variable": 0, + "station": 1, + "longitude": 2, + "latitude": 3, + "value": 4, + "units": 5, + "start_time": 6, + "end_time": 7, + "altitude": 9, + "country": "NO", + "standard_deviation": "NaN", + "flag": "0", + } ) as ts: + ts.read() self.assertEqual(len(ts.stations()), 0) def test_relaltitude_filter_2(self): engines = pyaro.list_timeseries_engines() with engines["csv_timeseries"].open( - filename=self.elevation_file, - filters=[pyaro.timeseries.filters.get("relaltitude", topo_file = "./tests/testdata/datadir_elevation/topography.nc", rdiff=90)], - columns={ - "variable": 0, - "station": 1, - "longitude": 2, - "latitude": 3, - "value": 4, - "units": 5, - "start_time": 6, - "end_time": 7, - "altitude": 9, - "country": "NO", - "standard_deviation": "NaN", - "flag": "0", - } + filename=self.elevation_file, + filters=[pyaro.timeseries.filters.get("relaltitude", + topo_file="./tests/testdata/datadir_elevation/topography.nc", + rdiff=90)], + columns={ + "variable": 0, + "station": 1, + "longitude": 2, + "latitude": 3, + "value": 4, + "units": 5, + "start_time": 6, + "end_time": 7, + "altitude": 9, + "country": "NO", + "standard_deviation": "NaN", + "flag": "0", + } ) as ts: + ts.read() # At rdiff = 90, only the first station should be included. self.assertEqual(len(ts.stations()), 1) def test_relaltitude_filter_3(self): engines = pyaro.list_timeseries_engines() with engines["csv_timeseries"].open( - filename=self.elevation_file, - filters=[pyaro.timeseries.filters.get("relaltitude", topo_file = "./tests/testdata/datadir_elevation/topography.nc", rdiff=300)], - columns={ - "variable": 0, - "station": 1, - "longitude": 2, - "latitude": 3, - "value": 4, - "units": 5, - "start_time": 6, - "end_time": 7, - "altitude": 9, - "country": "NO", - "standard_deviation": "NaN", - "flag": "0", - } + filename=self.elevation_file, + filters=[pyaro.timeseries.filters.get("relaltitude", + topo_file="./tests/testdata/datadir_elevation/topography.nc", + rdiff=300)], + columns={ + "variable": 0, + "station": 1, + "longitude": 2, + "latitude": 3, + "value": 4, + "units": 5, + "start_time": 6, + "end_time": 7, + "altitude": 9, + "country": "NO", + "standard_deviation": "NaN", + "flag": "0", + } ) as ts: + ts.read() # Since rdiff=300, all stations should be included. self.assertEqual(len(ts.stations()), 3) - if __name__ == "__main__": unittest.main()