From a9657b708fd1fdc4117afb3ef3c09b52ab75657e Mon Sep 17 00:00:00 2001 From: kkappler Date: Wed, 6 Dec 2023 19:16:00 -0800 Subject: [PATCH] invert filters applied booleans --- mth5/io/phoenix/readers/receiver_metadata.py | 6 ++-- mth5/io/zen/zen.py | 2 +- mth5/timeseries/channel_ts.py | 7 ++-- tests/clients/test_geomag_client.py | 4 +-- tests/io/nims/test_read_nims.py | 18 +++++----- tests/io/phoenix/test_base_reader.py | 2 +- .../phoenix/test_read_phoenix_continuous.py | 2 +- tests/io/phoenix/test_read_phoenix_native.py | 2 +- .../io/phoenix/test_read_phoenix_segmented.py | 2 +- tests/io/usgs_ascii/test_header.py | 10 +++--- tests/io/zen/test_z3d.py | 4 +-- tests/timeseries/test_remove_response.py | 33 ++++++++++++------- tests/version_1/test_make_mth5_geomag.py | 8 ++--- tests/version_2/test_make_mth5_geomag.py | 8 ++--- 14 files changed, 60 insertions(+), 48 deletions(-) diff --git a/mth5/io/phoenix/readers/receiver_metadata.py b/mth5/io/phoenix/readers/receiver_metadata.py index 2235dcd8..0353836c 100644 --- a/mth5/io/phoenix/readers/receiver_metadata.py +++ b/mth5/io/phoenix/readers/receiver_metadata.py @@ -140,7 +140,7 @@ def _to_electric_metadata(self, tag): f"{self.lp_filter_base_name}_{int(ch.lp)}hz_low_pass", f"dipole_{int(c.dipole_length)}m", ] - c.filter.applied = [False, False] + c.filter.applied = [True, True] return c def _to_magnetic_metadata(self, tag): @@ -167,10 +167,10 @@ def _to_magnetic_metadata(self, tag): c.filter.name = [ f"{self.lp_filter_base_name}_{int(ch.lp)}hz_low_pass" ] - c.filter.applied = [False] + c.filter.applied = [True] if c.sensor.id is not None: c.filter.name.append(f"coil_{c.sensor.id}_response") - c.filter.applied.append(False) + c.filter.applied.append(True) return c diff --git a/mth5/io/zen/zen.py b/mth5/io/zen/zen.py index 90b31db0..a099c779 100644 --- a/mth5/io/zen/zen.py +++ b/mth5/io/zen/zen.py @@ -427,7 +427,7 @@ def channel_metadata(self): ch.units = "digital counts" ch.channel_number = self.channel_number ch.filter.name = self.channel_response.names - ch.filter.applied = [False] * len(self.channel_response.names) + ch.filter.applied = [True] * len(self.channel_response.names) return ch diff --git a/mth5/timeseries/channel_ts.py b/mth5/timeseries/channel_ts.py index 3a7a0298..4b368945 100644 --- a/mth5/timeseries/channel_ts.py +++ b/mth5/timeseries/channel_ts.py @@ -1039,7 +1039,7 @@ def channel_response_filter(self, value): for f_name in self._channel_response.names: self.channel_metadata.filter.name.append(f_name) - self.channel_metadata.filter.applied = [False] * len( + self.channel_metadata.filter.applied = [True] * len( self.channel_metadata.filter.name ) @@ -1137,9 +1137,10 @@ def remove_instrument_response(self, **kwargs): calibration_operation, calibrated_units = self.get_response_correction_operation_and_units() calibrated_ts.ts = remover.remove_instrument_response(operation=calibration_operation) - # change applied booleans + # change applied booleans + # TODO use invert on bool, instead of direct assignement to False applied_filters = calibrated_ts.channel_metadata.filter.applied - calibrated_ts.channel_metadata.filter.applied = [True] * len( + calibrated_ts.channel_metadata.filter.applied = [False] * len( self.channel_metadata.filter.applied ) diff --git a/tests/clients/test_geomag_client.py b/tests/clients/test_geomag_client.py index d0047576..7b1a7c3e 100644 --- a/tests/clients/test_geomag_client.py +++ b/tests/clients/test_geomag_client.py @@ -247,7 +247,7 @@ def test_hx_metadata(self): ("channel_number", 0), ("component", "hx"), ("data_quality.rating.value", 0), - ("filter.applied", [False]), + ("filter.applied", [True]), ("filter.name", []), ("location.elevation", 331.0), ("location.latitude", 37.091), @@ -279,7 +279,7 @@ def test_hy_metadata(self): ("channel_number", 0), ("component", "hy"), ("data_quality.rating.value", 0), - ("filter.applied", [False]), + ("filter.applied", [True]), ("filter.name", []), ("location.elevation", 331.0), ("location.latitude", 37.091), diff --git a/tests/io/nims/test_read_nims.py b/tests/io/nims/test_read_nims.py index 2dbc3de9..b158f08f 100644 --- a/tests/io/nims/test_read_nims.py +++ b/tests/io/nims/test_read_nims.py @@ -254,7 +254,7 @@ def test_ex_metadata(self): ("component", "ex"), ("data_quality.rating.value", 0), ("dipole_length", 109.0), - ("filter.applied", [False, False, False, False, False, False]), + ("filter.applied", [True, True, True, True, True, True]), ( "filter.name", [ @@ -302,7 +302,7 @@ def test_ey_metadata(self): ("component", "ey"), ("data_quality.rating.value", 0), ("dipole_length", 101.0), - ("filter.applied", [False, False, False, False, False, False]), + ("filter.applied", [True, True, True, True, True, True]), ( "filter.name", [ @@ -349,7 +349,7 @@ def test_hx_metadata(self): ("channel_number", 1), ("component", "hx"), ("data_quality.rating.value", 0), - ("filter.applied", [False, False, False]), + ("filter.applied", [True, True, True]), ( "filter.name", [ @@ -387,7 +387,7 @@ def test_hy_metadata(self): ("channel_number", 2), ("component", "hy"), ("data_quality.rating.value", 0), - ("filter.applied", [False, False, False]), + ("filter.applied", [True, True, True]), ( "filter.name", [ @@ -425,7 +425,7 @@ def test_hz_metadata(self): ("channel_number", 3), ("component", "hz"), ("data_quality.rating.value", 0), - ("filter.applied", [False, False, False]), + ("filter.applied", [True, True, True]), ( "filter.name", [ @@ -463,7 +463,7 @@ def test_temperature_metadata(self): ("channel_number", 6), ("component", "temperature"), ("data_quality.rating.value", 0), - ("filter.applied", [False]), + ("filter.applied", [True]), ("filter.name", []), ("location.elevation", 0.0), ("location.latitude", 0.0), @@ -500,7 +500,7 @@ def test_calibrate(self): self.assertEqual(ch.channel_metadata.units, "nT") with self.subTest("applied"): self.assertListEqual( - ch.channel_metadata.filter.applied, [True, True, True] + ch.channel_metadata.filter.applied, [False, False, False] ) for comp in ["ex", "ey"]: @@ -510,7 +510,7 @@ def test_calibrate(self): with self.subTest("applied"): self.assertListEqual( ch.channel_metadata.filter.applied, - [True, True, True, True, True, True], + [False, False, False, False, False, False], ) ch = getattr(calibrated_run, "temperature") @@ -519,7 +519,7 @@ def test_calibrate(self): with self.subTest("applied"): self.assertListEqual( ch.channel_metadata.filter.applied, - [True], + [False], ) diff --git a/tests/io/phoenix/test_base_reader.py b/tests/io/phoenix/test_base_reader.py index 7efc7ad6..90dbdb05 100644 --- a/tests/io/phoenix/test_base_reader.py +++ b/tests/io/phoenix/test_base_reader.py @@ -120,7 +120,7 @@ def test_channel_metadata(self): ("channel_number", 0), ("component", "h2"), ("data_quality.rating.value", 0), - ("filter.applied", [False, False]), + ("filter.applied", [True, True]), ( "filter.name", ["mtu-5c_rmt03_10128_10000hz_low_pass", "coil_0_response"], diff --git a/tests/io/phoenix/test_read_phoenix_continuous.py b/tests/io/phoenix/test_read_phoenix_continuous.py index b4768c57..34b6085b 100644 --- a/tests/io/phoenix/test_read_phoenix_continuous.py +++ b/tests/io/phoenix/test_read_phoenix_continuous.py @@ -149,7 +149,7 @@ def test_to_channel_ts(self): ("channel_number", 0), ("component", "h2"), ("data_quality.rating.value", 0), - ("filter.applied", [False, False]), + ("filter.applied", [True, True]), ( "filter.name", ["mtu-5c_rmt03-j_666_h2_10000hz_lowpass", "v_to_mv"], diff --git a/tests/io/phoenix/test_read_phoenix_native.py b/tests/io/phoenix/test_read_phoenix_native.py index 5ef0e5fd..ec0fc5b8 100644 --- a/tests/io/phoenix/test_read_phoenix_native.py +++ b/tests/io/phoenix/test_read_phoenix_native.py @@ -170,7 +170,7 @@ def test_metadata(self): ("channel_number", 0), ("component", "h2"), ("data_quality.rating.value", 0), - ("filter.applied", [False, False]), + ("filter.applied", [True, True]), ( "filter.name", ["mtu-5c_rmt03-j_666_h2_10000hz_lowpass", "v_to_mv"], diff --git a/tests/io/phoenix/test_read_phoenix_segmented.py b/tests/io/phoenix/test_read_phoenix_segmented.py index ba212798..eee34c9f 100644 --- a/tests/io/phoenix/test_read_phoenix_segmented.py +++ b/tests/io/phoenix/test_read_phoenix_segmented.py @@ -163,7 +163,7 @@ def test_to_channel_ts(self): ("channel_number", 0), ("component", "h2"), ("data_quality.rating.value", 0), - ("filter.applied", [False, False]), + ("filter.applied", [True, True]), ( "filter.name", ["mtu-5c_rmt03-j_666_h2_10000hz_lowpass", "v_to_mv"], diff --git a/tests/io/usgs_ascii/test_header.py b/tests/io/usgs_ascii/test_header.py index 0a6f7bf1..9e966c8a 100644 --- a/tests/io/usgs_ascii/test_header.py +++ b/tests/io/usgs_ascii/test_header.py @@ -52,7 +52,7 @@ def test_ex(self): ("component", "ex"), ("data_quality.rating.value", 0), ("dipole_length", 100.0), - ("filter.applied", [False]), + ("filter.applied", [True]), ("filter.name", []), ("measurement_azimuth", 9.0), ("measurement_tilt", 0.0), @@ -86,7 +86,7 @@ def test_ey(self): ("component", "ey"), ("data_quality.rating.value", 0), ("dipole_length", 102.0), - ("filter.applied", [False]), + ("filter.applied", [True]), ("filter.name", []), ("measurement_azimuth", 99.0), ("measurement_tilt", 0.0), @@ -119,7 +119,7 @@ def test_hx(self): ("channel_number", 31), ("component", "hx"), ("data_quality.rating.value", 0), - ("filter.applied", [False]), + ("filter.applied", [True]), ("filter.name", []), ("location.elevation", 0.0), ("location.latitude", 0.0), @@ -146,7 +146,7 @@ def test_hy(self): ("channel_number", 33), ("component", "hy"), ("data_quality.rating.value", 0), - ("filter.applied", [False]), + ("filter.applied", [True]), ("filter.name", []), ("location.elevation", 0.0), ("location.latitude", 0.0), @@ -173,7 +173,7 @@ def test_hz(self): ("channel_number", 35), ("component", "hz"), ("data_quality.rating.value", 0), - ("filter.applied", [False]), + ("filter.applied", [True]), ("filter.name", []), ("location.elevation", 0.0), ("location.latitude", 0.0), diff --git a/tests/io/zen/test_z3d.py b/tests/io/zen/test_z3d.py index 5d3274e4..dc501cd5 100644 --- a/tests/io/zen/test_z3d.py +++ b/tests/io/zen/test_z3d.py @@ -160,7 +160,7 @@ def test_channel_metadata(self): ("dc.end", 0.019371436521409924), ("dc.start", 0.019130984313785026), ("dipole_length", 56.0), - ("filter.applied", [False, False]), + ("filter.applied", [True, True]), ( "filter.name", ["dipole_56.00m", "zen_counts2mv"], @@ -571,7 +571,7 @@ def test_channel_metadata(self): ("channel_number", 2), ("component", "hy"), ("data_quality.rating.value", 0), - ("filter.applied", [False, False]), + ("filter.applied", [True, True]), ( "filter.name", [ diff --git a/tests/timeseries/test_remove_response.py b/tests/timeseries/test_remove_response.py index 626512a3..34f2f1c2 100644 --- a/tests/timeseries/test_remove_response.py +++ b/tests/timeseries/test_remove_response.py @@ -20,29 +20,40 @@ class TestRemoveResponse(unittest.TestCase): """ - Test remove response, make a fake signal add some trends, + Test remove response, make a fake signal add some trends, + ToDo: + - Add more than one filter to the test. + - Utilize the is_channel_response attributes """ @classmethod def setUpClass(self): # pole zero filter - pz = PoleZeroFilter( + pz1 = PoleZeroFilter( units_in="volts", units_out="nanotesla", name="instrument_response" ) - pz.poles = [ + pz1.poles = [ (-6.283185 + 10.882477j), (-6.283185 - 10.882477j), (-12.566371 + 0j), ] - pz.zeros = [] - pz.normalization_factor = 18244400 - + pz1.zeros = [] + pz1.normalization_factor = 18244400 + pz2 = PoleZeroFilter( + units_in="nanotesla", units_out="volts", name="instrument_response2" + ) + pz2.poles = [] + pz2.zeros = [] + pz2.normalization_factor = 10 + pz2.gain = 12 # channel properties self.channel = ChannelTS() - self.channel.channel_metadata.filter.applied = [False] - self.channel.channel_metadata.filter.name = ["instrument_response"] + self.channel.channel_metadata.filter.applied = [True, ]#False] + self.channel.channel_metadata.filter.name = ["instrument_response",]# "instrument_response2"] self.channel.channel_metadata.component = "hx" - self.channel.channel_response_filter.filters_list.append(pz) + self.channel.channel_metadata.units = "digital counts" + self.channel.channel_response_filter.filters_list.append(pz1) +# self.channel.channel_response_filter.filters_list.append(pz2) self.channel.sample_rate = 1 n_samples = 4096 self.t = np.arange(n_samples) * self.channel.sample_interval @@ -61,7 +72,7 @@ def setUpClass(self): # multiply by filter response f = np.fft.rfftfreq(self.t.size, self.channel.sample_interval) response_ts = np.fft.irfft( - np.fft.rfft(self.example_ts) * pz.complex_response(f)[::-1] + np.fft.rfft(self.example_ts) * pz1.complex_response(f)[::-1] ) # add in a linear trend @@ -74,7 +85,7 @@ def test_return_type(self): def test_applied(self): self.assertTrue( - (np.array(self.calibrated_ts.channel_metadata.filter.applied) == True).all() + (np.array(self.calibrated_ts.channel_metadata.filter.applied) == False).all() ) def test_returned_metadata(self): diff --git a/tests/version_1/test_make_mth5_geomag.py b/tests/version_1/test_make_mth5_geomag.py index ce8f1223..c7320e89 100644 --- a/tests/version_1/test_make_mth5_geomag.py +++ b/tests/version_1/test_make_mth5_geomag.py @@ -202,7 +202,7 @@ def test_fresno_run_001_hx_metadata(self): ("channel_number", 0), ("component", "hx"), ("data_quality.rating.value", 0), - ("filter.applied", [False]), + ("filter.applied", [True]), ("filter.name", []), ("hdf5_reference", ""), ("location.elevation", 331.0), @@ -235,7 +235,7 @@ def test_fresno_run_001_hy_metadata(self): ("channel_number", 0), ("component", "hy"), ("data_quality.rating.value", 0), - ("filter.applied", [False]), + ("filter.applied", [True]), ("filter.name", []), ("hdf5_reference", ""), ("location.elevation", 331.0), @@ -375,7 +375,7 @@ def test_ottowa_run_001_hx_metadata(self): ("channel_number", 0), ("component", "hx"), ("data_quality.rating.value", 0), - ("filter.applied", [False]), + ("filter.applied", [True]), ("filter.name", []), ("hdf5_reference", ""), ("location.elevation", 0.0), @@ -408,7 +408,7 @@ def test_ottowa_run_001_hy_metadata(self): ("channel_number", 0), ("component", "hy"), ("data_quality.rating.value", 0), - ("filter.applied", [False]), + ("filter.applied", [True]), ("filter.name", []), ("hdf5_reference", ""), ("location.elevation", 0.0), diff --git a/tests/version_2/test_make_mth5_geomag.py b/tests/version_2/test_make_mth5_geomag.py index 487307cc..a08172fa 100644 --- a/tests/version_2/test_make_mth5_geomag.py +++ b/tests/version_2/test_make_mth5_geomag.py @@ -203,7 +203,7 @@ def test_fresno_run_001_hx_metadata(self): ("channel_number", 0), ("component", "hx"), ("data_quality.rating.value", 0), - ("filter.applied", [False]), + ("filter.applied", [True]), ("filter.name", []), ("hdf5_reference", ""), ("location.elevation", 331.0), @@ -236,7 +236,7 @@ def test_fresno_run_001_hy_metadata(self): ("channel_number", 0), ("component", "hy"), ("data_quality.rating.value", 0), - ("filter.applied", [False]), + ("filter.applied", [True]), ("filter.name", []), ("hdf5_reference", ""), ("location.elevation", 331.0), @@ -376,7 +376,7 @@ def test_ottowa_run_001_hx_metadata(self): ("channel_number", 0), ("component", "hx"), ("data_quality.rating.value", 0), - ("filter.applied", [False]), + ("filter.applied", [True]), ("filter.name", []), ("hdf5_reference", ""), ("location.elevation", 0.0), @@ -409,7 +409,7 @@ def test_ottowa_run_001_hy_metadata(self): ("channel_number", 0), ("component", "hy"), ("data_quality.rating.value", 0), - ("filter.applied", [False]), + ("filter.applied", [True]), ("filter.name", []), ("hdf5_reference", ""), ("location.elevation", 0.0),