From c11dc4694c796cf1e6c46a16a85398f50b9c4d36 Mon Sep 17 00:00:00 2001 From: JP Date: Sun, 18 Aug 2024 14:15:18 -0700 Subject: [PATCH] replacing RunSummary and KDS with mtpy versions --- .github/workflows/tests.yml | 3 ++- data/synthetic/emtf_results/test1.zss | 2 +- docs/examples/operate_aurora.ipynb | 6 +++-- docs/tutorials/processing_configuration.ipynb | 5 ++-- .../tutorials/synthetic_data_processing.ipynb | 5 ++-- tests/cas04/02b_process_cas04_mth5.py | 13 ++++++--- tests/parkfield/test_process_parkfield_run.py | 5 ++-- .../test_process_parkfield_run_rr.py | 14 +++++++--- .../test_compare_aurora_vs_archived_emtf.py | 4 +-- tests/synthetic/test_fourier_coefficients.py | 26 +++++++++++++----- .../test_metadata_values_set_correctly.py | 2 +- tests/synthetic/test_multi_run.py | 18 +++++-------- tests/synthetic/test_processing.py | 27 ++++++++++++++----- tests/synthetic/test_stft_methods_agree.py | 11 +++++--- 14 files changed, 93 insertions(+), 48 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ef027e5b..48b42097 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,7 +36,8 @@ jobs: conda install -c conda-forge pytest pytest-cov certifi">=2017.4.17" pandoc pip install -r requirements-dev.txt pip install git+https://github.com/kujaku11/mt_metadata.git@main - pip install git+https://github.com/kujaku11/mth5.git@add_aurora_tools + pip install git+https://github.com/kujaku11/mth5.git@add_aurora_tools + pip install git+https://github.com/MTgeophysics/mtpy-v2.git@aurora - name: Install Our Package run: | diff --git a/data/synthetic/emtf_results/test1.zss b/data/synthetic/emtf_results/test1.zss index a9d08b67..bc25b995 100644 --- a/data/synthetic/emtf_results/test1.zss +++ b/data/synthetic/emtf_results/test1.zss @@ -2,7 +2,7 @@ ********** WITH FULL ERROR COVARINCE********** Robust Single station station :test1 -coordinate 1007.996 0.000 declination 0.00 +coordinate 0 0.000 declination 0.00 number of channels 5 number of frequencies 25 orientations and tilts of each channel 1 0.00 0.00 tes Hx diff --git a/docs/examples/operate_aurora.ipynb b/docs/examples/operate_aurora.ipynb index 2a1482e6..1895a152 100644 --- a/docs/examples/operate_aurora.ipynb +++ b/docs/examples/operate_aurora.ipynb @@ -65,8 +65,10 @@ "from aurora.config import BANDS_DEFAULT_FILE\n", "from aurora.config.config_creator import ConfigCreator\n", "from aurora.pipelines.process_mth5 import process_mth5\n", - "from aurora.transfer_function.kernel_dataset import KernelDataset\n", - "from aurora.pipelines.run_summary import RunSummary\n", + "\n", + "from mtpy.processing.run_summary import RunSummary\n", + "from mtpy.processing.kernel_dataset import KernelDataset\n", + "\n", "\n", "warnings.filterwarnings('ignore')" ] diff --git a/docs/tutorials/processing_configuration.ipynb b/docs/tutorials/processing_configuration.ipynb index b3c9b7fa..b5799b6a 100644 --- a/docs/tutorials/processing_configuration.ipynb +++ b/docs/tutorials/processing_configuration.ipynb @@ -152,9 +152,10 @@ "metadata": {}, "outputs": [], "source": [ - "from aurora.pipelines.run_summary import RunSummary\n", "from aurora.test_utils.synthetic.paths import SyntheticTestPaths\n", - "from aurora.transfer_function.kernel_dataset import KernelDataset\n", + "\n", + "from mtpy.processing.run_summary import RunSummary\n", + "from mtpy.processing.kernel_dataset import KernelDataset\n", "\n", "synthetic_test_paths = SyntheticTestPaths()\n", "MTH5_PATH = synthetic_test_paths.mth5_path" diff --git a/docs/tutorials/synthetic_data_processing.ipynb b/docs/tutorials/synthetic_data_processing.ipynb index 80ad878f..ce0ca83e 100644 --- a/docs/tutorials/synthetic_data_processing.ipynb +++ b/docs/tutorials/synthetic_data_processing.ipynb @@ -45,10 +45,11 @@ "\n", "from aurora.config.config_creator import ConfigCreator\n", "from aurora.pipelines.process_mth5 import process_mth5\n", - "from aurora.pipelines.run_summary import RunSummary\n", "from aurora.test_utils.synthetic.make_mth5_from_asc import create_test12rr_h5\n", "from aurora.test_utils.synthetic.paths import DATA_PATH\n", - "from aurora.transfer_function.kernel_dataset import KernelDataset\n", + "\n", + "from mtpy.processing.run_summary import RunSummary\n", + "from mtpy.processing.kernel_dataset import KernelDataset\n", "\n", "warnings.filterwarnings('ignore')" ] diff --git a/tests/cas04/02b_process_cas04_mth5.py b/tests/cas04/02b_process_cas04_mth5.py index f2fa6541..999e34f9 100644 --- a/tests/cas04/02b_process_cas04_mth5.py +++ b/tests/cas04/02b_process_cas04_mth5.py @@ -57,9 +57,10 @@ from aurora.config.config_creator import ConfigCreator from aurora.general_helper_functions import get_test_path from aurora.pipelines.process_mth5 import process_mth5 -from aurora.pipelines.run_summary import RunSummary from aurora.transfer_function.plot.comparison_plots import compare_two_z_files -from aurora.transfer_function.kernel_dataset import KernelDataset + +from mtpy.processing.run_summary import RunSummary +from mtpy.processing.kernel_dataset import KernelDataset from loguru import logger @@ -125,7 +126,9 @@ def z_file_name(self, target_dir): return out_file -def process_station_runs(local_station_id, remote_station_id="", station_runs={}): +def process_station_runs( + local_station_id, remote_station_id="", station_runs={} +): """ Parameters @@ -152,7 +155,9 @@ def process_station_runs(local_station_id, remote_station_id="", station_runs={} # Pass the run_summary to a Dataset class kernel_dataset = KernelDataset() - kernel_dataset.from_run_summary(run_summary, local_station_id, remote_station_id) + kernel_dataset.from_run_summary( + run_summary, local_station_id, remote_station_id + ) # reduce station_runs_dict to only relevant stations diff --git a/tests/parkfield/test_process_parkfield_run.py b/tests/parkfield/test_process_parkfield_run.py index 7b0068b6..f3f9ad45 100644 --- a/tests/parkfield/test_process_parkfield_run.py +++ b/tests/parkfield/test_process_parkfield_run.py @@ -1,10 +1,11 @@ from aurora.config.config_creator import ConfigCreator from aurora.pipelines.process_mth5 import process_mth5 -from aurora.pipelines.run_summary import RunSummary from aurora.test_utils.parkfield.make_parkfield_mth5 import ensure_h5_exists from aurora.test_utils.parkfield.path_helpers import PARKFIELD_PATHS -from aurora.transfer_function.kernel_dataset import KernelDataset from aurora.transfer_function.plot.comparison_plots import compare_two_z_files + +from mtpy.processing.run_summary import RunSummary +from mtpy.processing.kernel_dataset import KernelDataset from loguru import logger from mth5.helpers import close_open_files diff --git a/tests/parkfield/test_process_parkfield_run_rr.py b/tests/parkfield/test_process_parkfield_run_rr.py index 37d1a741..bd77364c 100644 --- a/tests/parkfield/test_process_parkfield_run_rr.py +++ b/tests/parkfield/test_process_parkfield_run_rr.py @@ -1,13 +1,15 @@ from aurora.config.config_creator import ConfigCreator from aurora.pipelines.process_mth5 import process_mth5 -from aurora.pipelines.run_summary import RunSummary from aurora.sandbox.mth5_channel_summary_helpers import ( channel_summary_to_make_mth5, ) from aurora.test_utils.parkfield.make_parkfield_mth5 import ensure_h5_exists from aurora.test_utils.parkfield.path_helpers import PARKFIELD_PATHS -from aurora.transfer_function.kernel_dataset import KernelDataset from aurora.transfer_function.plot.comparison_plots import compare_two_z_files + +from mtpy.processing.run_summary import RunSummary +from mtpy.processing.kernel_dataset import KernelDataset + from loguru import logger from mth5.mth5 import MTH5 from mth5.helpers import close_open_files @@ -90,7 +92,9 @@ def test(): test_processing(z_file_path=z_file_path) # COMPARE WITH ARCHIVED Z-FILE - auxilliary_z_file = PARKFIELD_PATHS["emtf_results"].joinpath("PKD_272_00.zrr") + auxilliary_z_file = PARKFIELD_PATHS["emtf_results"].joinpath( + "PKD_272_00.zrr" + ) if z_file_path.exists(): compare_two_z_files( z_file_path, @@ -104,7 +108,9 @@ def test(): xlims=[0.05, 500], ) else: - logger.error("Z-File not found - Parkfield tests failed to generate output") + logger.error( + "Z-File not found - Parkfield tests failed to generate output" + ) logger.warning("NCEDC probably not returning data") close_open_files() diff --git a/tests/synthetic/test_compare_aurora_vs_archived_emtf.py b/tests/synthetic/test_compare_aurora_vs_archived_emtf.py index 941b84f4..10763dc9 100644 --- a/tests/synthetic/test_compare_aurora_vs_archived_emtf.py +++ b/tests/synthetic/test_compare_aurora_vs_archived_emtf.py @@ -1,5 +1,5 @@ from aurora.pipelines.process_mth5 import process_mth5 -from aurora.pipelines.run_summary import RunSummary +from mtpy.processing.run_summary import RunSummary from aurora.sandbox.io_helpers.zfile_murphy import read_z_file from mth5.data.make_mth5_from_asc import create_test1_h5 from mth5.data.make_mth5_from_asc import create_test2_h5 @@ -14,7 +14,7 @@ from aurora.transfer_function.emtf_z_file_helpers import ( merge_tf_collection_to_match_z_file, ) -from aurora.transfer_function.kernel_dataset import KernelDataset +from mtpy.processing.kernel_dataset import KernelDataset from plot_helpers_synthetic import plot_rho_phi from loguru import logger from mth5.helpers import close_open_files diff --git a/tests/synthetic/test_fourier_coefficients.py b/tests/synthetic/test_fourier_coefficients.py index 252f4989..ae9b7960 100644 --- a/tests/synthetic/test_fourier_coefficients.py +++ b/tests/synthetic/test_fourier_coefficients.py @@ -5,14 +5,19 @@ from aurora.pipelines.fourier_coefficients import fc_decimations_creator from aurora.pipelines.fourier_coefficients import read_back_fcs from aurora.pipelines.process_mth5 import process_mth5 -from aurora.pipelines.run_summary import RunSummary + from aurora.test_utils.synthetic.make_mth5_from_asc import create_test1_h5 from aurora.test_utils.synthetic.make_mth5_from_asc import create_test2_h5 from aurora.test_utils.synthetic.make_mth5_from_asc import create_test3_h5 from aurora.test_utils.synthetic.make_mth5_from_asc import create_test12rr_h5 -from aurora.test_utils.synthetic.make_processing_configs import create_test_run_config +from aurora.test_utils.synthetic.make_processing_configs import ( + create_test_run_config, +) from aurora.test_utils.synthetic.paths import SyntheticTestPaths -from aurora.transfer_function.kernel_dataset import KernelDataset + +from mtpy.processing.run_summary import RunSummary +from mtpy.processing.kernel_dataset import KernelDataset + from loguru import logger from mth5.helpers import close_open_files @@ -55,7 +60,12 @@ def setUpClass(self): mth5_path_2 = create_test2_h5(file_version=self.file_version) mth5_path_3 = create_test3_h5(file_version=self.file_version) mth5_path_12rr = create_test12rr_h5(file_version=self.file_version) - self.mth5_paths = [mth5_path_1, mth5_path_2, mth5_path_3, mth5_path_12rr] + self.mth5_paths = [ + mth5_path_1, + mth5_path_2, + mth5_path_3, + mth5_path_12rr, + ] def test_123(self): """ @@ -85,7 +95,9 @@ def test_123(self): ]: station_id = mth5_path.stem tfk_dataset.from_run_summary(run_summary, station_id) - processing_config = create_test_run_config(station_id, tfk_dataset) + processing_config = create_test_run_config( + station_id, tfk_dataset + ) elif mth5_path.stem in [ "test3", ]: @@ -135,7 +147,9 @@ def test_create_then_use_stored_fcs_for_processing(self): tf1 = process_synthetic_2( force_make_mth5=True, z_file_path=z_file_path_1, save_fc=True ) - tf2 = process_synthetic_2(force_make_mth5=False, z_file_path=z_file_path_2) + tf2 = process_synthetic_2( + force_make_mth5=False, z_file_path=z_file_path_2 + ) assert tf1 == tf2 diff --git a/tests/synthetic/test_metadata_values_set_correctly.py b/tests/synthetic/test_metadata_values_set_correctly.py index 219fc530..a7cd32b0 100644 --- a/tests/synthetic/test_metadata_values_set_correctly.py +++ b/tests/synthetic/test_metadata_values_set_correctly.py @@ -7,7 +7,7 @@ import pandas as pd import unittest -from aurora.pipelines.run_summary import RunSummary +from mtpy.processing.run_summary import RunSummary from mth5.data.make_mth5_from_asc import create_test3_h5 from mth5.data.station_config import make_station_03 from mth5.helpers import close_open_files diff --git a/tests/synthetic/test_multi_run.py b/tests/synthetic/test_multi_run.py index e4708361..40819266 100644 --- a/tests/synthetic/test_multi_run.py +++ b/tests/synthetic/test_multi_run.py @@ -2,10 +2,11 @@ import unittest from aurora.config.config_creator import ConfigCreator from aurora.pipelines.process_mth5 import process_mth5 -from aurora.pipelines.run_summary import RunSummary + from aurora.test_utils.synthetic.make_mth5_from_asc import create_test3_h5 from aurora.test_utils.synthetic.paths import SyntheticTestPaths -from aurora.transfer_function.kernel_dataset import KernelDataset +from mtpy.processing.run_summary import RunSummary +from mtpy.processing.kernel_dataset import KernelDataset from mth5.helpers import close_open_files @@ -137,13 +138,8 @@ def test_works_with_truncated_run(self): tf_cls.write(fn=xml_file_name, file_type="emtfxml") -def main(): - # tmp = TestMultiRunProcessing() - # tmp.setUp() - # tmp.test_works_with_truncated_run() - # tmp.test_all_runs() - unittest.main() - - +# ============================================================================= +# run +# ============================================================================= if __name__ == "__main__": - main() + unittest.main() diff --git a/tests/synthetic/test_processing.py b/tests/synthetic/test_processing.py index 167411dc..5f3db8dc 100644 --- a/tests/synthetic/test_processing.py +++ b/tests/synthetic/test_processing.py @@ -2,7 +2,7 @@ import unittest from aurora.pipelines.process_mth5 import process_mth5 -from aurora.pipelines.run_summary import RunSummary + from aurora.test_utils.synthetic.make_mth5_from_asc import create_test1_h5 from aurora.test_utils.synthetic.make_mth5_from_asc import create_test2_h5 from aurora.test_utils.synthetic.make_mth5_from_asc import create_test12rr_h5 @@ -10,9 +10,12 @@ create_test_run_config, ) from aurora.test_utils.synthetic.paths import SyntheticTestPaths -from aurora.transfer_function.kernel_dataset import KernelDataset + from mth5.helpers import close_open_files +from mtpy.processing.run_summary import RunSummary +from mtpy.processing.kernel_dataset import KernelDataset + synthetic_test_paths = SyntheticTestPaths() synthetic_test_paths.mkdirs() AURORA_RESULTS_PATH = synthetic_test_paths.aurora_results_path @@ -60,20 +63,26 @@ def test_can_output_tf_class_and_write_tf_xml(self): def test_can_use_channel_nomenclature(self): channel_nomencalture = "LEMI12" - z_file_path = AURORA_RESULTS_PATH.joinpath(f"syn1-{channel_nomencalture}.zss") + z_file_path = AURORA_RESULTS_PATH.joinpath( + f"syn1-{channel_nomencalture}.zss" + ) tf_cls = process_synthetic_1( z_file_path=z_file_path, file_version=self.file_version, channel_nomenclature=channel_nomencalture, ) - xml_file_base = f"syn1_mth5-{self.file_version}_{channel_nomencalture}.xml" + xml_file_base = ( + f"syn1_mth5-{self.file_version}_{channel_nomencalture}.xml" + ) xml_file_name = AURORA_RESULTS_PATH.joinpath(xml_file_base) tf_cls.write(fn=xml_file_name, file_type="emtfxml") def test_can_use_mth5_file_version_020(self): file_version = "0.2.0" z_file_path = AURORA_RESULTS_PATH.joinpath(f"syn1-{file_version}.zss") - tf_cls = process_synthetic_1(z_file_path=z_file_path, file_version=file_version) + tf_cls = process_synthetic_1( + z_file_path=z_file_path, file_version=file_version + ) xml_file_base = f"syn1_mth5v{file_version}.xml" xml_file_name = AURORA_RESULTS_PATH.joinpath(xml_file_base) tf_cls.write(fn=xml_file_name, file_type="emtfxml") @@ -106,7 +115,9 @@ def test_can_use_scale_factor_dictionary(self): ) def test_simultaneous_regression(self): - z_file_path = AURORA_RESULTS_PATH.joinpath("syn1_simultaneous_estimate.zss") + z_file_path = AURORA_RESULTS_PATH.joinpath( + "syn1_simultaneous_estimate.zss" + ) tf_cls = process_synthetic_1( z_file_path=z_file_path, simultaneous_regression=True ) @@ -228,7 +239,9 @@ def process_synthetic_1( if return_collection: z_figure_name = z_file_path.name.replace("zss", "png") for xy_or_yx in ["xy", "yx"]: - ttl_str = f"{xy_or_yx} component, test_scale_factor = {test_scale_factor}" + ttl_str = ( + f"{xy_or_yx} component, test_scale_factor = {test_scale_factor}" + ) out_png_name = f"{xy_or_yx}_{z_figure_name}" tf_result.rho_phi_plot( xy_or_yx=xy_or_yx, diff --git a/tests/synthetic/test_stft_methods_agree.py b/tests/synthetic/test_stft_methods_agree.py index eef0586f..0563f444 100644 --- a/tests/synthetic/test_stft_methods_agree.py +++ b/tests/synthetic/test_stft_methods_agree.py @@ -2,9 +2,9 @@ See aurora issue #3. This test confirms that the internal aurora stft method returns the same array as scipy.signal.spectrogram """ + import numpy as np -from aurora.pipelines.run_summary import RunSummary from aurora.pipelines.time_series_helpers import prototype_decimate from aurora.pipelines.time_series_helpers import run_ts_to_stft from aurora.pipelines.time_series_helpers import run_ts_to_stft_scipy @@ -12,7 +12,10 @@ from aurora.test_utils.synthetic.make_processing_configs import ( create_test_run_config, ) -from aurora.transfer_function.kernel_dataset import KernelDataset + +from mtpy.processing.run_summary import RunSummary +from mtpy.processing.kernel_dataset import KernelDataset + from loguru import logger from mth5.mth5 import MTH5 from mth5.helpers import close_open_files @@ -65,7 +68,9 @@ def test_stft_methods_agree(): run_ts = run_obj.to_runts(start=None, end=None) local_run_xrts = run_ts.dataset else: - local_run_xrts = prototype_decimate(dec_config.decimation, local_run_xrts) + local_run_xrts = prototype_decimate( + dec_config.decimation, local_run_xrts + ) dec_config.extra_pre_fft_detrend_type = "" local_stft_obj = run_ts_to_stft(dec_config, local_run_xrts)