diff --git a/.github/workflows/wait-for-pypi.py b/.github/workflows/wait-for-pypi.py old mode 100644 new mode 100755 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 70f433b..8b37a02 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ ci: autofix_prs: false # Comment "pre-commit.ci autofix" on a PR to trigger default_language_version: - python: python3.9 + python: python3.10 repos: - repo: https://github.com/gitleaks/gitleaks diff --git a/batcher/harmony/cli.py b/batcher/harmony/cli.py index 5d7504c..ca46570 100644 --- a/batcher/harmony/cli.py +++ b/batcher/harmony/cli.py @@ -25,6 +25,7 @@ # either express or implied. See the License for the specific language governing permissions and # limitations under the License. """A Harmony CLI wrapper around the concatenate-batcher""" + from argparse import ArgumentParser import harmony_service_lib @@ -45,7 +46,8 @@ def main(config: harmony_service_lib.util.Config = None) -> None: None """ parser = ArgumentParser( - prog="Pre-concatenate-batching", description="Run the pre-concatenate-batching service" + prog="Pre-concatenate-batching", + description="Run the pre-concatenate-batching service", ) harmony_service_lib.setup_cli(parser) args = parser.parse_args() diff --git a/batcher/harmony/service_adapter.py b/batcher/harmony/service_adapter.py index 72926b6..251d2f2 100644 --- a/batcher/harmony/service_adapter.py +++ b/batcher/harmony/service_adapter.py @@ -70,7 +70,9 @@ def invoke(self): # Message-only support is being depreciated in Harmony, so we should expect to # only see requests with catalogs when invoked with a newer Harmony instance # https://github.com/nasa/harmony-service-lib-py/blob/21bcfbda17caf626fb14d2ac4f8673be9726b549/harmony/adapter.py#L71 - raise RuntimeError("Invoking Batchee without a STAC catalog is not supported") + raise RuntimeError( + "Invoking Batchee without a STAC catalog is not supported" + ) return self.message, self.process_catalog(self.catalog) @@ -108,7 +110,9 @@ def process_catalog(self, catalog: pystac.Catalog) -> list[pystac.Catalog]: # and each Catalog holds multiple Items (which represent each granule). catalogs = [] for batch_id, batch_items in grouped.items(): - self.logger.info(f"constructing new pystac.Catalog for batch_id==={batch_id}.") + self.logger.info( + f"constructing new pystac.Catalog for batch_id==={batch_id}." + ) # Initialize a new, empty Catalog batch_catalog = catalog.clone() batch_catalog.id = str(uuid4()) @@ -135,7 +139,9 @@ def process_catalog(self, catalog: pystac.Catalog) -> list[pystac.Catalog]: ) batch_catalog.add_item(output_item) - self.logger.info("STAC catalog creation for batch_id==={batch_id} complete.") + self.logger.info( + "STAC catalog creation for batch_id==={batch_id} complete." + ) catalogs.append(batch_catalog) self.logger.info("All STAC catalogs are complete.") diff --git a/batcher/harmony/util.py b/batcher/harmony/util.py index 8f9055e..69d1aee 100644 --- a/batcher/harmony/util.py +++ b/batcher/harmony/util.py @@ -25,6 +25,7 @@ # either express or implied. See the License for the specific language governing permissions and # limitations under the License. """Misc utility functions""" + from datetime import datetime from pystac import Asset, Item @@ -104,7 +105,10 @@ def _get_output_date_range(input_items: list[Item]) -> dict[str, str]: start_datetime = min(start_datetime, new_start_datetime) end_datetime = max(end_datetime, new_end_datetime) - return {"start_datetime": start_datetime.isoformat(), "end_datetime": end_datetime.isoformat()} + return { + "start_datetime": start_datetime.isoformat(), + "end_datetime": end_datetime.isoformat(), + } def _get_item_date_range(item: Item) -> tuple[datetime, datetime]: diff --git a/batcher/tempo_filename_parser.py b/batcher/tempo_filename_parser.py index ceb7c8a..74c4d79 100644 --- a/batcher/tempo_filename_parser.py +++ b/batcher/tempo_filename_parser.py @@ -25,6 +25,7 @@ # either express or implied. See the License for the specific language governing permissions and # limitations under the License. """Holds the logic for grouping together data files based on their filenames.""" + import logging import re from argparse import ArgumentParser @@ -44,7 +45,9 @@ ) -def get_batch_indices(filenames: list, logger: logging.Logger = default_logger) -> list[int]: +def get_batch_indices( + filenames: list, logger: logging.Logger = default_logger +) -> list[int]: """ Returns ------- @@ -59,10 +62,14 @@ def get_batch_indices(filenames: list, logger: logging.Logger = default_logger) matches = tempo_granule_filename_pattern.match(name) if matches: match_dict = matches.groupdict() - day_and_scans.append((match_dict["day_in_granule"], match_dict["daily_scan_id"])) + day_and_scans.append( + (match_dict["day_in_granule"], match_dict["daily_scan_id"]) + ) # Unique day-scans are determined (while keeping the same order). Each will be its own batch. - unique_day_scans: list[tuple[str, str]] = sorted(set(day_and_scans), key=day_and_scans.index) + unique_day_scans: list[tuple[str, str]] = sorted( + set(day_and_scans), key=day_and_scans.index + ) logger.info(f"unique_day_scans==={unique_day_scans}.") @@ -79,7 +86,8 @@ def main() -> list[list[str]]: """Main CLI entrypoint""" parser = ArgumentParser( - prog="batchee", description="Simple CLI wrapper around the granule batcher module." + prog="batchee", + description="Simple CLI wrapper around the granule batcher module.", ) parser.add_argument( "file_names", @@ -101,7 +109,9 @@ def main() -> list[list[str]]: input_filenames = args.file_names batch_indices = get_batch_indices(input_filenames) - unique_category_indices: list[int] = sorted(set(batch_indices), key=batch_indices.index) + unique_category_indices: list[int] = sorted( + set(batch_indices), key=batch_indices.index + ) logging.info(f"batch_indices = {batch_indices}") # --- Construct a STAC object based on the batch indices --- diff --git a/tests/test_harmony_adapter.py b/tests/test_harmony_adapter.py index 2726a6e..18c49fc 100644 --- a/tests/test_harmony_adapter.py +++ b/tests/test_harmony_adapter.py @@ -22,7 +22,6 @@ def test_service_invoke(self, temp_output_dir): # test with both paged catalogs and un-paged catalogs for in_catalog_name in ["catalog.json", "catalog0.json"]: - in_catalog_path = self.__harmony_path.joinpath("source", in_catalog_name) test_args = [ @@ -64,7 +63,9 @@ def test_service_invoke(self, temp_output_dir): for item_meta in out_catalog["links"]: if item_meta["rel"] == "item": - item_path = temp_output_dir.joinpath(item_meta["href"]).resolve() + item_path = temp_output_dir.joinpath( + item_meta["href"] + ).resolve() # -- Item Verification -- item = json.loads(item_path.read_text()) @@ -80,7 +81,9 @@ def test_service_invoke(self, temp_output_dir): assert data["type"] == "application/x-netcdf4" assert data["roles"] == ["data"] - batched_files[batch_index].append(Path(urlsplit(data["href"]).path).stem) + batched_files[batch_index].append( + Path(urlsplit(data["href"]).path).stem + ) # -- batch file list verification -- files_dict = {