From c2319bf60fa5e60939ed90f80afeee57cbb6cb55 Mon Sep 17 00:00:00 2001 From: annajungbluth Date: Thu, 4 Jul 2024 13:01:28 +0000 Subject: [PATCH 1/2] changed try/except --- rs_tools/_src/data/msg/download.py | 42 +++++++++++++++--------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/rs_tools/_src/data/msg/download.py b/rs_tools/_src/data/msg/download.py index aae75fe..0ebfe0f 100644 --- a/rs_tools/_src/data/msg/download.py +++ b/rs_tools/_src/data/msg/download.py @@ -150,8 +150,12 @@ def msg_download( return (files, successful_queries) def _download(time: datetime, data_product: str, save_dir: str, datastore): - products = _compile_msg_products(data_product=data_product, time=time, datastore=datastore) - sub_files_list = _msg_data_download(products=products, save_dir=save_dir) + try: + products = _compile_msg_products(data_product=data_product, time=time, datastore=datastore) + sub_files_list = _msg_data_download(products=products, save_dir=save_dir) + except Exception as error: + logger.error(f"Error downloading data: {error}") + sub_files_list = None return sub_files_list def _compile_msg_products(data_product: str, time: datetime, datastore): @@ -162,25 +166,21 @@ def _compile_msg_products(data_product: str, time: datetime, datastore): return products def _msg_data_download(products, save_dir: str): - try: - for product in products: - for entry in product.entries: - if entry.endswith(".nat") or entry.endswith(".grb"): - with product.open(entry=entry) as fsrc: - # Create a full file path for saving the file - save_path = os.path.join(save_dir, os.path.basename(fsrc.name)) - # Check if file already exists - if os.path.exists(save_path): - print(f"File {save_path} already exists. Skipping download.") - return [save_path] - else: - with open(save_path, mode='wb') as fdst: - shutil.copyfileobj(fsrc, fdst) - print(f"Successfully downloaded {entry}.") - return [save_path] - except Exception as error: - print(f"Error downloading product': '{error}'") - pass + for product in products: + for entry in product.entries: + if entry.endswith(".nat") or entry.endswith(".grb"): + with product.open(entry=entry) as fsrc: + # Create a full file path for saving the file + save_path = os.path.join(save_dir, os.path.basename(fsrc.name)) + # Check if file already exists + if os.path.exists(save_path): + print(f"File {save_path} already exists. Skipping download.") + return [save_path] + else: + with open(save_path, mode='wb') as fdst: + shutil.copyfileobj(fsrc, fdst) + print(f"Successfully downloaded {entry}.") + return [save_path] def _check_eumdac_login(eumdac_key: str, eumdac_secret: str) -> bool: """check if eumdac login is available in environment variables / as input arguments""" From 67870e621f41cba0f2b767809fc4cbd1d3c5abbc Mon Sep 17 00:00:00 2001 From: annajungbluth Date: Fri, 5 Jul 2024 08:06:47 +0000 Subject: [PATCH 2/2] removed path extension --- rs_tools/_src/data/goes/downloader_goes16.py | 2 +- rs_tools/_src/data/modis/downloader_aqua.py | 2 +- rs_tools/_src/data/modis/downloader_aqua_day.py | 2 +- rs_tools/_src/data/modis/downloader_terra.py | 2 +- rs_tools/_src/data/modis/downloader_terra_day.py | 2 +- rs_tools/_src/data/msg/download.py | 2 +- rs_tools/_src/data/msg/downloader_msg.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/rs_tools/_src/data/goes/downloader_goes16.py b/rs_tools/_src/data/goes/downloader_goes16.py index c8b0b70..603dc06 100644 --- a/rs_tools/_src/data/goes/downloader_goes16.py +++ b/rs_tools/_src/data/goes/downloader_goes16.py @@ -121,7 +121,7 @@ def download( daily_window_t0=daily_window_t0, daily_window_t1=daily_window_t1, time_step=time_step, - save_dir=Path(save_dir).joinpath("goes16"), + save_dir=Path(save_dir), #.joinpath("goes16"), ) logger.info("Downloading GOES 16 Data...") goes16_filenames = dc_goes16_download.download() diff --git a/rs_tools/_src/data/modis/downloader_aqua.py b/rs_tools/_src/data/modis/downloader_aqua.py index 8dd6c6e..2759ba9 100644 --- a/rs_tools/_src/data/modis/downloader_aqua.py +++ b/rs_tools/_src/data/modis/downloader_aqua.py @@ -91,7 +91,7 @@ def download( end_date=end_date, start_time=start_time, end_time=end_time, - save_dir=Path(save_dir).joinpath("aqua"), + save_dir=Path(save_dir), #.joinpath("aqua"), bounding_box=bounding_box, ) logger.info("Downloading AQUA Data...") diff --git a/rs_tools/_src/data/modis/downloader_aqua_day.py b/rs_tools/_src/data/modis/downloader_aqua_day.py index 36e7d37..68fdbb5 100644 --- a/rs_tools/_src/data/modis/downloader_aqua_day.py +++ b/rs_tools/_src/data/modis/downloader_aqua_day.py @@ -91,7 +91,7 @@ def download( end_date=end_date, start_time=start_time, end_time=end_time, - save_dir=Path(save_dir).joinpath("aqua"), + save_dir=Path(save_dir), #.joinpath("aqua"), bounding_box=bounding_box, ) logger.info("Downloading AQUA Data...") diff --git a/rs_tools/_src/data/modis/downloader_terra.py b/rs_tools/_src/data/modis/downloader_terra.py index ac949c3..c9f9cd9 100644 --- a/rs_tools/_src/data/modis/downloader_terra.py +++ b/rs_tools/_src/data/modis/downloader_terra.py @@ -91,7 +91,7 @@ def download( end_date=end_date, start_time=start_time, end_time=end_time, - save_dir=Path(save_dir).joinpath("terra"), + save_dir=Path(save_dir), #.joinpath("terra"), bounding_box=bounding_box, ) logger.info("Downloading TERRA Data...") diff --git a/rs_tools/_src/data/modis/downloader_terra_day.py b/rs_tools/_src/data/modis/downloader_terra_day.py index a22613e..b800555 100644 --- a/rs_tools/_src/data/modis/downloader_terra_day.py +++ b/rs_tools/_src/data/modis/downloader_terra_day.py @@ -91,7 +91,7 @@ def download( end_date=end_date, start_time=start_time, end_time=end_time, - save_dir=Path(save_dir).joinpath("terra"), + save_dir=Path(save_dir), #.joinpath("terra"), bounding_box=bounding_box, ) logger.info("Downloading TERRA Data...") diff --git a/rs_tools/_src/data/msg/download.py b/rs_tools/_src/data/msg/download.py index 0ebfe0f..2a82381 100644 --- a/rs_tools/_src/data/msg/download.py +++ b/rs_tools/_src/data/msg/download.py @@ -144,6 +144,7 @@ def msg_download( if sub_files_list is None: logger.info(f"Could not find data for time {itime}. Skipping to next time.") else: + logger.info(f"Successfully downloaded data for time {itime}.") files += sub_files_list successful_queries.append(itime) @@ -174,7 +175,6 @@ def _msg_data_download(products, save_dir: str): save_path = os.path.join(save_dir, os.path.basename(fsrc.name)) # Check if file already exists if os.path.exists(save_path): - print(f"File {save_path} already exists. Skipping download.") return [save_path] else: with open(save_path, mode='wb') as fdst: diff --git a/rs_tools/_src/data/msg/downloader_msg.py b/rs_tools/_src/data/msg/downloader_msg.py index ec81c5a..0fee14b 100644 --- a/rs_tools/_src/data/msg/downloader_msg.py +++ b/rs_tools/_src/data/msg/downloader_msg.py @@ -92,7 +92,7 @@ def download( daily_window_t0=daily_window_t0, daily_window_t1=daily_window_t1, time_step=time_step, - save_dir=Path(save_dir).joinpath("msg"), + save_dir=Path(save_dir), #.joinpath("msg"), ) logger.info("Downloading MSG Data...") msg_filenames = dc_msg_download.download()