Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix mt metadata issue 173 #174

Merged
merged 28 commits into from
Dec 15, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
5686957
Move handling of units into a class method
kkappler Nov 24, 2023
ae841ab
fix missing return argument
kkappler Nov 24, 2023
105e4a1
add doc
kkappler Nov 24, 2023
54183f8
update mt_metadata to fix_issue_173 branch
kkappler Nov 24, 2023
aae83fb
add usage of inverse_filters
kkappler Nov 25, 2023
a9657b7
invert filters applied booleans
kkappler Dec 7, 2023
c4997e9
Fix Error
kkappler Dec 7, 2023
f027611
Update unit checking
kkappler Dec 7, 2023
ac16486
Improve remove_instrument_response
kkappler Dec 7, 2023
655948c
Test now uses units
kkappler Dec 7, 2023
afd66fb
Towards mt_metadata issue 173
kkappler Dec 8, 2023
efb3a7b
Handle filter booleans
kkappler Dec 8, 2023
277aef5
Update units after calibration
kkappler Dec 8, 2023
0b1860d
add logger warnings
kkappler Dec 8, 2023
e9acdd5
fix logging message to f-string
kkappler Dec 9, 2023
fd55eed
ChannelResponseFilter-->ChannelResponse
kkappler Dec 9, 2023
5a60542
ChannelResponseFilter-->ChannelResponse
kkappler Dec 9, 2023
47d6539
channel_response_filter --> channel_response
kkappler Dec 9, 2023
c668344
update returned args from get_list_of_filters_to_remove to be one val…
kkappler Dec 9, 2023
29dff5e
remove references to filter correction operation
kkappler Dec 9, 2023
78cd366
Calibration is only applied via division
kkappler Dec 9, 2023
6484d96
add test for codecov
kkappler Dec 9, 2023
09c6fac
add calibration operation option back into remove instrument response
kkappler Dec 12, 2023
3a74131
Merge pull request #177 from kujaku11/fix_mt_metadata_issue_173a
kkappler Dec 12, 2023
38f8625
update mt_metadata branch
kkappler Dec 12, 2023
407469f
tidy and add doc to _validate_channel_metadata
kkappler Dec 13, 2023
e49b903
Merge branch 'master' into fix_mt_metadata_issue_173
kkappler Dec 15, 2023
28c7495
add warning about issue #169
kkappler Dec 15, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ jobs:
conda install pytest=7.1.2
conda install pytest-cov
conda install pytest-subtests
pip install git+https://github.com/kujaku11/mt_metadata.git@patches
pip install git+https://github.com/kujaku11/mt_metadata.git@fix_issue_173

- name: Install Our Package
shell: bash
Expand Down
10 changes: 5 additions & 5 deletions mth5/groups/channel_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
from mt_metadata import timeseries as metadata
from mt_metadata.utils.mttime import MTime
from mt_metadata.base import Base
from mt_metadata.timeseries.filters import ChannelResponseFilter
from mt_metadata.timeseries.filters import ChannelResponse

from mth5 import CHANNEL_DTYPE
from mth5.utils.exceptions import MTH5Error
Expand Down Expand Up @@ -248,7 +248,7 @@ def survey_id(self):
return self.hdf5_dataset.parent.parent.parent.parent.attrs["id"]

@property
def channel_response_filter(self):
def channel_response(self):
# get the filters to make a channel response
filters_group = FiltersGroup(
self.hdf5_dataset.parent.parent.parent.parent["Filters"]
Expand All @@ -261,7 +261,7 @@ def channel_response_filter(self):
except KeyError:
self.logger.warning(f"Could not locate filter {name}")
continue
return ChannelResponseFilter(filters_list=f_list)
return ChannelResponse(filters_list=f_list)

@property
def start(self):
Expand Down Expand Up @@ -686,7 +686,7 @@ def to_channel_ts(self):
run_metadata=self.run_metadata.copy(),
station_metadata=self.station_metadata.copy(),
survey_metadata=self.survey_metadata.copy(),
channel_response_filter=self.channel_response_filter,
channel_response=self.channel_response,
)

def to_xarray(self):
Expand Down Expand Up @@ -1079,7 +1079,7 @@ def time_slice(
self.metadata.type,
data=self.hdf5_dataset[regional_ref],
channel_metadata={self.metadata.type: meta_dict},
channel_response_filter=self.channel_response_filter,
channel_response=self.channel_response,
)
else:
msg = "return_type not understood, must be [ pandas | numpy | channel_ts ]"
Expand Down
6 changes: 3 additions & 3 deletions mth5/groups/filters.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
Need to make a group for FAP and FIR filters.


:copyright:
:copyright:
Jared Peacock ([email protected])

:license: MIT
Expand Down Expand Up @@ -143,8 +143,8 @@ def to_filter_object(self, name):
try:
f_type = self.filter_dict[name]["type"]
except KeyError:
msg = "Could not find %s in the filter dictionary"
self.logger.error(msg, name)
msg = f"Could not find {name} in the filter dictionary"
self.logger.error(msg)
raise KeyError(msg, name)
if f_type in ["zpk"]:
return self.zpk_group.to_object(name)
Expand Down
4 changes: 2 additions & 2 deletions mth5/groups/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -678,11 +678,11 @@ def from_channel_ts(self, channel_ts_obj):
self.logger.error(msg)
raise MTH5Error(msg)
## Need to add in the filters
if channel_ts_obj.channel_response_filter.filters_list != []:
if channel_ts_obj.channel_response.filters_list != []:
from mth5.groups import FiltersGroup

fg = FiltersGroup(self.hdf5_group.parent.parent.parent["Filters"])
for ff in channel_ts_obj.channel_response_filter.filters_list:
for ff in channel_ts_obj.channel_response.filters_list:
fg.add_filter(ff)
ch_obj = self.add_channel(
channel_ts_obj.component,
Expand Down
10 changes: 5 additions & 5 deletions mth5/io/nims/nims.py
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,7 @@ def hx(self):
channel_metadata=self.hx_metadata,
run_metadata=self.run_metadata,
station_metadata=self.station_metadata,
channel_response_filter=self.get_channel_response("hx"),
channel_response=self.get_channel_response("hx"),
)
return None

Expand Down Expand Up @@ -357,7 +357,7 @@ def hy(self):
channel_metadata=self.hy_metadata,
run_metadata=self.run_metadata,
station_metadata=self.station_metadata,
channel_response_filter=self.get_channel_response("hy"),
channel_response=self.get_channel_response("hy"),
)
return None

Expand Down Expand Up @@ -394,7 +394,7 @@ def hz(self):
channel_metadata=self.hz_metadata,
run_metadata=self.run_metadata,
station_metadata=self.station_metadata,
channel_response_filter=self.get_channel_response("hz"),
channel_response=self.get_channel_response("hz"),
)
return None

Expand Down Expand Up @@ -432,7 +432,7 @@ def ex(self):
channel_metadata=self.ex_metadata,
run_metadata=self.run_metadata,
station_metadata=self.station_metadata,
channel_response_filter=self.get_channel_response("ex", self.ex_length),
channel_response=self.get_channel_response("ex", self.ex_length),
)
return None

Expand Down Expand Up @@ -470,7 +470,7 @@ def ey(self):
channel_metadata=self.ey_metadata,
run_metadata=self.run_metadata,
station_metadata=self.station_metadata,
channel_response_filter=self.get_channel_response("ey", self.ey_length),
channel_response=self.get_channel_response("ey", self.ey_length),
)
return None

Expand Down
6 changes: 3 additions & 3 deletions mth5/io/nims/response_filters.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
PoleZeroFilter,
TimeDelayFilter,
CoefficientFilter,
ChannelResponseFilter,
ChannelResponse,
)

# =============================================================================
Expand Down Expand Up @@ -273,12 +273,12 @@ def get_channel_response(self, channel, dipole_length=1):
"""

if channel[0] in ["e"]:
return ChannelResponseFilter(
return ChannelResponse(
filters_list=self._get_electric_filter(channel, dipole_length)
)

elif channel[0] in ["b", "h"]:
return ChannelResponseFilter(
return ChannelResponse(
filters_list=self._get_magnetic_filter(channel)
)

Expand Down
10 changes: 5 additions & 5 deletions mth5/io/phoenix/readers/base.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
"""
Module to read and parse native Phoenix Geophysics data formats of the
Module to read and parse native Phoenix Geophysics data formats of the
MTU-5C Family.

This module implements Streamed readers for segmented-decimated continuus-decimated
and native sampling rate time series formats of the MTU-5C family.

:author: Jorge Torres-Solis

Revised 2022 by J. Peacock
Revised 2022 by J. Peacock
"""

# =============================================================================
Expand All @@ -21,7 +21,7 @@

from mt_metadata.timeseries.filters import (
CoefficientFilter,
ChannelResponseFilter,
ChannelResponse,
)

from loguru import logger
Expand Down Expand Up @@ -463,7 +463,7 @@ def get_v_to_mv_filter(self):

return conversion

def get_channel_response_filter(self, rxcal_fn=None, scal_fn=None):
def get_channel_response(self, rxcal_fn=None, scal_fn=None):
"""
Get the channel response filter

Expand All @@ -489,4 +489,4 @@ def get_channel_response_filter(self, rxcal_fn=None, scal_fn=None):
if ch_metadata.type in ["electric"]:
filter_list.append(self.get_dipole_filter())

return ChannelResponseFilter(filters_list=filter_list)
return ChannelResponse(filters_list=filter_list)
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ def to_channel_ts(self, rxcal_fn=None, scal_fn=None):
channel_metadata=self.channel_metadata,
run_metadata=self.run_metadata,
station_metadata=self.station_metadata,
channel_response_filter=self.get_channel_response_filter(
channel_response=self.get_channel_response(
rxcal_fn=rxcal_fn, scal_fn=scal_fn
),
)
2 changes: 1 addition & 1 deletion mth5/io/phoenix/readers/native/native_reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -288,7 +288,7 @@ def to_channel_ts(self, rxcal_fn=None, scal_fn=None):
channel_metadata=ch_metadata,
run_metadata=self.run_metadata,
station_metadata=self.station_metadata,
channel_response_filter=self.get_channel_response_filter(
channel_response=self.get_channel_response(
rxcal_fn=rxcal_fn, scal_fn=scal_fn
),
)
6 changes: 3 additions & 3 deletions mth5/io/phoenix/readers/receiver_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ def _to_electric_metadata(self, tag):
f"{self.lp_filter_base_name}_{int(ch.lp)}hz_low_pass",
f"dipole_{int(c.dipole_length)}m",
]
c.filter.applied = [False, False]
c.filter.applied = [True, True]
return c

def _to_magnetic_metadata(self, tag):
Expand All @@ -167,10 +167,10 @@ def _to_magnetic_metadata(self, tag):
c.filter.name = [
f"{self.lp_filter_base_name}_{int(ch.lp)}hz_low_pass"
]
c.filter.applied = [False]
c.filter.applied = [True]
if c.sensor.id is not None:
c.filter.name.append(f"coil_{c.sensor.id}_response")
c.filter.applied.append(False)
c.filter.applied.append(True)

return c

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ def to_channel_ts(self, rxcal_fn=None, scal_fn=None):
channel_metadata=ch_metadata,
run_metadata=self.run_metadata,
station_metadata=self.station_metadata,
channel_response_filter=self.get_channel_response_filter(
channel_response=self.get_channel_response(
rxcal_fn=rxcal_fn, scal_fn=scal_fn
),
)
Expand Down Expand Up @@ -331,7 +331,7 @@ def to_channel_ts(self, rxcal_fn=None, scal_fn=None):
channel_metadata=ch_metadata,
run_metadata=self.run_metadata,
station_metadata=self.station_metadata,
channel_response_filter=self.get_channel_response_filter(
channel_response=self.get_channel_response(
rxcal_fn=rxcal_fn, scal_fn=scal_fn
),
)
Expand Down
10 changes: 5 additions & 5 deletions mth5/io/zen/zen.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
* Tools for reading and writing files for Zen and processing software
* Tools for copying data from SD cards
* Tools for copying schedules to SD cards

Created on Tue Jun 11 10:53:23 2013
Updated August 2020 (JP)

Expand All @@ -30,7 +30,7 @@

from mt_metadata.utils.mttime import MTime
from mt_metadata.timeseries.filters import (
ChannelResponseFilter,
ChannelResponse,
FrequencyResponseTableFilter,
CoefficientFilter,
)
Expand Down Expand Up @@ -427,7 +427,7 @@ def channel_metadata(self):
ch.units = "digital counts"
ch.channel_number = self.channel_number
ch.filter.name = self.channel_response.names
ch.filter.applied = [False] * len(self.channel_response.names)
ch.filter.applied = [True] * len(self.channel_response.names)

return ch

Expand Down Expand Up @@ -596,7 +596,7 @@ def channel_response(self):
filter_list.append(self.dipole_filter)

filter_list.append(self.counts2mv_filter)
return ChannelResponseFilter(filters_list=filter_list)
return ChannelResponse(filters_list=filter_list)

@property
def dipole_filter(self):
Expand Down Expand Up @@ -1127,7 +1127,7 @@ def to_channelts(self):
channel_metadata=self.channel_metadata,
station_metadata=self.station_metadata,
run_metadata=self.run_metadata,
channel_response_filter=self.channel_response,
channel_response=self.channel_response,
)


Expand Down
Loading