Merge pull request #51 from spaceml-org/5-overpass-download

5 overpass download
spaceml-org · Sep 24, 2024 · 4665635 · 4665635
2 parents 52ba29f + 67870e6
commit 4665635
Show file tree

Hide file tree

Showing 7 changed files with 27 additions and 27 deletions.
diff --git a/rs_tools/_src/data/goes/downloader_goes16.py b/rs_tools/_src/data/goes/downloader_goes16.py
@@ -121,7 +121,7 @@ def download(
         daily_window_t0=daily_window_t0,
         daily_window_t1=daily_window_t1,
         time_step=time_step,
-        save_dir=Path(save_dir).joinpath("goes16"),
+        save_dir=Path(save_dir), #.joinpath("goes16"),
     )
     logger.info("Downloading GOES 16 Data...")
     goes16_filenames = dc_goes16_download.download()

diff --git a/rs_tools/_src/data/modis/downloader_aqua.py b/rs_tools/_src/data/modis/downloader_aqua.py
@@ -91,7 +91,7 @@ def download(
         end_date=end_date,
         start_time=start_time,
         end_time=end_time,
-        save_dir=Path(save_dir).joinpath("aqua"),
+        save_dir=Path(save_dir), #.joinpath("aqua"),
         bounding_box=bounding_box,
     )
     logger.info("Downloading AQUA Data...")

diff --git a/rs_tools/_src/data/modis/downloader_aqua_day.py b/rs_tools/_src/data/modis/downloader_aqua_day.py
@@ -91,7 +91,7 @@ def download(
         end_date=end_date,
         start_time=start_time,
         end_time=end_time,
-        save_dir=Path(save_dir).joinpath("aqua"),
+        save_dir=Path(save_dir), #.joinpath("aqua"),
         bounding_box=bounding_box,
     )
     logger.info("Downloading AQUA Data...")

diff --git a/rs_tools/_src/data/modis/downloader_terra.py b/rs_tools/_src/data/modis/downloader_terra.py
@@ -91,7 +91,7 @@ def download(
         end_date=end_date,
         start_time=start_time,
         end_time=end_time,
-        save_dir=Path(save_dir).joinpath("terra"),
+        save_dir=Path(save_dir), #.joinpath("terra"),
         bounding_box=bounding_box,
     )
     logger.info("Downloading TERRA Data...")

diff --git a/rs_tools/_src/data/modis/downloader_terra_day.py b/rs_tools/_src/data/modis/downloader_terra_day.py
@@ -91,7 +91,7 @@ def download(
         end_date=end_date,
         start_time=start_time,
         end_time=end_time,
-        save_dir=Path(save_dir).joinpath("terra"),
+        save_dir=Path(save_dir), #.joinpath("terra"),
         bounding_box=bounding_box,
     )
     logger.info("Downloading TERRA Data...")

diff --git a/rs_tools/_src/data/msg/download.py b/rs_tools/_src/data/msg/download.py
@@ -144,14 +144,19 @@ def msg_download(
         if sub_files_list is None:
             logger.info(f"Could not find data for time {itime}. Skipping to next time.")
         else:
+            logger.info(f"Successfully downloaded data for time {itime}.")
             files += sub_files_list
             successful_queries.append(itime)
 
     return (files, successful_queries)
 
 def _download(time: datetime, data_product: str, save_dir: str, datastore):
-    products = _compile_msg_products(data_product=data_product, time=time, datastore=datastore)
-    sub_files_list = _msg_data_download(products=products, save_dir=save_dir)
+    try:
+        products = _compile_msg_products(data_product=data_product, time=time, datastore=datastore)
+        sub_files_list = _msg_data_download(products=products, save_dir=save_dir)
+    except Exception as error:
+        logger.error(f"Error downloading data: {error}")
+        sub_files_list = None
     return sub_files_list
 
 def _compile_msg_products(data_product: str, time: datetime, datastore):
@@ -162,25 +167,20 @@ def _compile_msg_products(data_product: str, time: datetime, datastore):
     return products
 
 def _msg_data_download(products, save_dir: str):
-    try:
-        for product in products:
-            for entry in product.entries:
-                if entry.endswith(".nat") or entry.endswith(".grb"): 
-                    with product.open(entry=entry) as fsrc:
-                        # Create a full file path for saving the file
-                        save_path = os.path.join(save_dir, os.path.basename(fsrc.name))
-                        # Check if file already exists
-                        if os.path.exists(save_path):
-                            print(f"File {save_path} already exists. Skipping download.")
-                            return [save_path]
-                        else:
-                            with open(save_path, mode='wb') as fdst:
-                                shutil.copyfileobj(fsrc, fdst)
-                            print(f"Successfully downloaded {entry}.")
-                            return [save_path]
-    except Exception as error:
-        print(f"Error downloading product': '{error}'")
-        pass
+    for product in products:
+        for entry in product.entries:
+            if entry.endswith(".nat") or entry.endswith(".grb"): 
+                with product.open(entry=entry) as fsrc:
+                    # Create a full file path for saving the file
+                    save_path = os.path.join(save_dir, os.path.basename(fsrc.name))
+                    # Check if file already exists
+                    if os.path.exists(save_path):
+                        return [save_path]
+                    else:
+                        with open(save_path, mode='wb') as fdst:
+                            shutil.copyfileobj(fsrc, fdst)
+                        print(f"Successfully downloaded {entry}.")
+                        return [save_path]
 
 def _check_eumdac_login(eumdac_key: str, eumdac_secret: str) -> bool:
     """check if eumdac login is available in environment variables / as input arguments"""

diff --git a/rs_tools/_src/data/msg/downloader_msg.py b/rs_tools/_src/data/msg/downloader_msg.py
@@ -92,7 +92,7 @@ def download(
         daily_window_t0=daily_window_t0,
         daily_window_t1=daily_window_t1,
         time_step=time_step,
-        save_dir=Path(save_dir).joinpath("msg"),
+        save_dir=Path(save_dir), #.joinpath("msg"),
     )
     logger.info("Downloading MSG Data...")
     msg_filenames = dc_msg_download.download()