Merge commit '37e5634cbfefec09e159ab70daf14557c66f0f6c' into docker

src/cloudcasting_app/app.py

@@ -3,7 +3,7 @@
 
 This app expects these environmental variables to be available:
     SATELLITE_ZARR_PATH (str): The path of the input satellite data
-    OUTPUT_PREDICTION_ZARR_PATH (str): The path to save the predictions to
+    OUTPUT_PREDICTION_DIRECTORY (str): The path of the directory to save the predictions to
 """
 
 from importlib.metadata import PackageNotFoundError, version
@@ -12,6 +12,7 @@
 import yaml
 import hydra
 import typer
+import fsspec
 
 import pandas as pd
 import xarray as xr
@@ -131,7 +132,21 @@ def app(t0=None):
     ds_y_hat = da_y_hat.to_dataset(name="sat_pred")
     ds_y_hat.sat_pred.attrs.update(ds.data.attrs)
 
-    ds_y_hat.to_zarr(os.environ["OUTPUT_PREDICTION_ZARR_PATH"])
+    # Save predictions to latest path and to path with timestring
+    out_dir = os.environ["OUTPUT_PREDICTION_DIRECTORY"]
+
+    latest_zarr_path = f"{out_dir}/latest.zarr"
+    t0_string_zarr_path = t0.strftime(f"{out_dir}/%Y-%m-%dT%H:%M.zarr")
+
+    fs, _ = fsspec.core.url_to_fs(out_dir)
+    for path in [latest_zarr_path, t0_string_zarr_path]:
+
+        # Remove the path if it exists already
+        if fs.exists(path):
+            logger.info(f"Removing path: {path}")
+            fs.rm(path, recursive=True)
+
+        ds_y_hat.to_zarr(path)
 
 
 if __name__ == "__main__":

src/cloudcasting_app/data.py

@@ -105,7 +105,7 @@ def download_all_sat_data() -> bool:
 
     # download 5 minute satellite data
     sat_5_dl_path = os.environ["SATELLITE_ZARR_PATH"]
-    fs = fsspec.open(sat_5_dl_path).fs
+    fs, _ = fsspec.core.url_to_fs(sat_5_dl_path)
     if fs.exists(sat_5_dl_path):
         sat_available = True
         logger.info(f"Downloading 5-minute satellite data")

tests/test_app.py

@@ -14,15 +14,21 @@ def test_app(sat_5_data, tmp_path, test_t0):
     # In production sat zarr is zipped
     os.environ["SATELLITE_ZARR_PATH"] = "temp_sat.zarr.zip"
 
-    os.environ["OUTPUT_PREDICTION_ZARR_PATH"] = "sat_prediction.zarr"
+    os.environ["OUTPUT_PREDICTION_DIRECTORY"] = f"{tmp_path}"
 
     with zarr.storage.ZipStore("temp_sat.zarr.zip", mode="x") as store:
         sat_5_data.to_zarr(store)    
 
     app()
-
+
+    # Check the two output files have been created
+    latest_zarr_path = f"{tmp_path}/latest.zarr"
+    t0_string_zarr_path = test_t0.strftime(f"{tmp_path}/%Y-%m-%dT%H:%M.zarr")
+    assert os.path.exists(latest_zarr_path)
+    assert os.path.exists(t0_string_zarr_path)
+
     # Load the predictions and check them
-    ds_y_hat = xr.open_zarr(os.environ["OUTPUT_PREDICTION_ZARR_PATH"])
+    ds_y_hat = xr.open_zarr(latest_zarr_path)
 
     assert "sat_pred" in  ds_y_hat
     assert (