Release 2.31 (#87)

* Feature/arome pygrib compatibility fix (#83) [2022-12-05 11:00] Rebuild of the AROME Repository (And some extra components for the multi locations feature) FIXED: - The Arome repository now works again, after losing functionality due to cfgrib and other package updates. - Fixed the corresponding tests to work once again as well ADDED: - Some small improvements to types and comments - Some of the implementation of the feature for requesting multiple locations. This was deemed necessary / easier due to updates to tools used both in this and that feature. * Feature/multiple locations api support (#85) [2022-12-12 10:22] API support for multiple locations has been added CHANGES: - API "get_weather" calls for multiple locations are now supported. - Tests and comments have been adjusted accordingly. TODO: - Adjust factor selection (no factor selection currently takes place) * [2022-12-13 13:35] Fixed bug #84: wpla 2x bug precipitation always zero with knmi uurgegevens (#86) [2022-12-13 13:35] Fixed bug #84 FIXED: - Text based files would round to 2 decimals while the precipitation for uurgegevens was expressed in m/h, resulting in almost always values rounding to zero. NOTES: - The fix handles the rounding and not the base type, as that has been as it was since version 1 of the API. The third API call version will have mm/h as the default unit for precipitation.
alliander-opensource · Dec 13, 2022 · db7005f · db7005f
1 parent 5ab0eb1
commit db7005f
Show file tree

Hide file tree

Showing 22 changed files with 645 additions and 523 deletions.
diff --git a/README.md b/README.md
@@ -2,7 +2,7 @@
 SPDX-FileCopyrightText: 2019-2022 Alliander N.V.
 SPDX-License-Identifier: MPL-2.0
 -->
-[![License: MIT](https://img.shields.io/badge/License-MPL2.0-informational.svg)](https://github.com/alliander-opensource/Weather-Provider-API/blob/master/LICENSE)
+[![License: MIT](https://img.shields.io/badge/License-MPL2.0-informational.svg)](https://github.com/alliander-opensource/weather-provider-api/blob/master/LICENSE)
 [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=alliander-opensource_weather-provider-api&metric=alert_status)](https://sonarcloud.io/dashboard?id=alliander-opensource_Weather-Provider-API)
 [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=alliander-opensource_weather-provider-api&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=alliander-opensource_Weather-Provider-API)
 [![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=alliander-opensource_weather-provider-api&metric=security_rating)](https://sonarcloud.io/dashboard?id=alliander-opensource_Weather-Provider-API)

diff --git a/pyproject.toml b/pyproject.toml
@@ -11,32 +11,34 @@ include = [
 ]
 
 [tool.poetry.dependencies]
-python = ">=3.8,<4.0"
-starlette-prometheus = "^0.9.0"
-numpy = "^1.23.3"
-structlog = "^22.1.0"
-bs4 = "^0.0.1"
-netCDF4 = "^1.6.1"
-pandas = "^1.5.0"
-xarray = "^2022.9.0"
-fastapi = "^0.85.0"
+python = ">=3.8,<3.11"
+fastapi = "^0.88.0"
 requests = "^2.28.1"
-geopy = "^2.2.0"
-accept-types = "^0.4.1"
-lxml = "^4.9.1"
-uvicorn = "^0.18.3"
+geopy = "^2.3.0"
+numpy = "^1.23.5"
+structlog = "^22.3.0"
 gunicorn = "^20.1.0"
 eccodes = "^1.5.0"
-ecmwflibs = "^0.4.17"
-cfgrib = "^0.9.10.2"
+accept-types = "^0.4.1"
+lxml = "^4.9.1"
+starlette-prometheus = "^0.9.0"
+beautifulsoup4 = "^4.11.1"
+netcdf4 = "^1.6.2"
+ecmwflibs = "^0.5.0"
 tomli = "^2.0.1"
+pandas = "^1.5.2"
+xarray = "^2022.12.0"
+cfgrib = "^0.9.10.3"
+uvicorn = "^0.20.0"
 
 [tool.poetry.group.dev.dependencies]
-pytest = "^7.1.3"
+pytest = "^7.2.0"
 coverage = "^6.5.0"
 pytest-cov = "^4.0.0"
 isort = "^5.10.1"
 black = "^22.10.0"
+jupyter = "^1.0.0"
+
 
 [tool.poetry.scripts]
 wpla_update_era5sl = "weather_provider_api.scripts.update_era5sl_repository:main"

diff --git a/tests/test_knmi_arome_model.py b/tests/test_knmi_arome_model.py
@@ -13,10 +13,10 @@
 from weather_provider_api.routers.weather.sources.cds.factors import era5sl_factors
 from weather_provider_api.routers.weather.sources.cds.models.era5sl import ERA5SLModel
 from weather_provider_api.routers.weather.sources.knmi.client.arome_repository import (
-    AromeRepository,
+    HarmonieAromeRepository,
 )
 from weather_provider_api.routers.weather.sources.knmi.models.harmonie_arome import (
-    AromeModel,
+    HarmonieAromeModel,
 )
 from weather_provider_api.routers.weather.utils.geo_position import GeoPosition
 
@@ -69,12 +69,12 @@ def test_retrieve_weather(
     one_month_ago = five_days_ago - relativedelta(months=1)
 
     # Instead of returning the regular data
-    def mock_fill_dataset_with_data(self, arome_coordinates, begin, end):
+    def mock_fill_dataset_with_data(self, begin, end, coordinates):
         return mock_dataset_arome
 
-    monkeypatch.setattr(AromeRepository, "gather_period", mock_fill_dataset_with_data)
+    monkeypatch.setattr(HarmonieAromeRepository, "gather_period", mock_fill_dataset_with_data)
 
-    arome_model = AromeModel()
+    arome_model = HarmonieAromeModel()
     # The coordinates requested are those of Amsterdam and Arnhem
     ds = arome_model.get_weather(
         coords=[GeoPosition(52.3667, 4.8945), GeoPosition(51.9851, 5.8987)],

diff --git a/tests/test_knmi_arome_repository.py b/tests/test_knmi_arome_repository.py
@@ -16,21 +16,21 @@
 from dateutil.relativedelta import relativedelta
 
 from weather_provider_api.routers.weather.sources.knmi.client.arome_repository import (
-    AromeRepository,
+    HarmonieAromeRepository,
 )
 
 
 def _get_mock_prefix(dummy_date: datetime):
-    arome_repo = AromeRepository()
+    arome_repo = HarmonieAromeRepository()
     return (
-        arome_repo.file_prefix
-        + "_"
-        + str(dummy_date.year)
-        + str(dummy_date.month).zfill(2)
-        + str(dummy_date.day).zfill(2)
-        + "_"
-        + str(dummy_date.hour).zfill(2)
-        + "00.nc"
+            arome_repo.file_prefix
+            + "_"
+            + str(dummy_date.year)
+            + str(dummy_date.month).zfill(2)
+            + str(dummy_date.day).zfill(2)
+            + "_"
+            + str(dummy_date.hour).zfill(2)
+            + "00.nc"
     )
 
 
@@ -53,7 +53,7 @@ def _empty_folder(folder: Path):
 
 
 def test_arome_repository_cleanup(_get_mock_repository_dir: Path):
-    arome_repo = AromeRepository()
+    arome_repo = HarmonieAromeRepository()
     arome_repo.repository_folder = _get_mock_repository_dir
 
     # CLEANUP TEST 1:   The entire repository directory doesn't exist.
@@ -101,7 +101,7 @@ def test_arome_repository_cleanup(_get_mock_repository_dir: Path):
 
 
 def test_repo_get_month_filename(_get_mock_repository_dir: Path):
-    arome_repo = AromeRepository()
+    arome_repo = HarmonieAromeRepository()
     arome_repo.repository_folder = _get_mock_repository_dir
 
     # SETUP: Create a clean mock repo-folder with dummy-files for 2 dates
@@ -110,27 +110,24 @@ def test_repo_get_month_filename(_get_mock_repository_dir: Path):
     arome_repo.cleanup()
     existing_dates = _fill_mock_repository(arome_repo.repository_folder)
     assert (
-        len(glob.glob(str(arome_repo.repository_folder.joinpath("AROME")) + "*.*")) == 2
+            len(glob.glob(str(arome_repo.repository_folder.joinpath("AROME")) + "*.*")) == 2
     )  # Confirm file creation
 
-    # FETCHING TEST 1:  All file-types exist, direct period result request
-    # Expected result:  Only the definitive file is returned, the rest is removed from the folder
-    file_prefix = arome_repo.repository_folder.joinpath(
-        _get_mock_prefix(existing_dates[0])
-    )
-
+    # FETCHING TEST 1:  One period outside the
     result = [
         Path(file_name)
         for file_name in arome_repo._get_file_list_for_period(
-            existing_dates[1], existing_dates[0]
+            existing_dates[1] + relativedelta(months=1), existing_dates[0]
         )
     ]
     assert len(result) == 1
-    assert result == [Path(str(file_prefix))]
+    assert result[0] == Path(
+        f'{arome_repo.repository_folder.joinpath("AROME_")}{existing_dates[0].strftime("%Y%m%d_%H00")}.nc'
+    )
 
 
 def test_arome_repository_remove_file(_get_mock_repository_dir):
-    arome_repo = AromeRepository()
+    arome_repo = HarmonieAromeRepository()
     arome_repo.repository_folder = _get_mock_repository_dir
 
     # SETUP: Create a clean mock repo-folder with dummy-files for 2 dates
@@ -145,7 +142,7 @@ def test_arome_repository_remove_file(_get_mock_repository_dir):
     existing_file = files_in_folder[0]
     assert arome_repo._safely_delete_file(existing_file)
 
-    # FILE REMOVAL TEST 2:  Try to remove an non-existing file from the repo
+    # FILE REMOVAL TEST 2:  Try to remove a non-existing file from the repo
     # Expected result:      A FileNotFound Error is raised
     non_existing_file = arome_repo.repository_folder.joinpath("DEF_DOESNT_EXIST.NOPE")
     with pytest.raises(OSError) as e:

diff --git a/weather_provider_api/app_config.py b/weather_provider_api/app_config.py
@@ -7,6 +7,7 @@
 import os
 import tempfile
 from importlib import metadata as importlib_metadata
+from pathlib import Path
 
 import tomli
 
@@ -28,10 +29,14 @@ class BaseConfig(object):
 
     default_version = None
     try:
+        if Path('./pyproject.toml').exists():
+            pyproject_file = Path('./pyproject.toml')
+        elif Path('../pyproject.toml').exists():
+            pyproject_file = Path('../pyproject.toml')
+        with open(pyproject_file, mode='rb') as file:
+            default_version = tomli.load(file)['tool']['poetry']['version']
+    except Exception:
         default_version = importlib_metadata.version(__package__)
-    except importlib_metadata.PackageNotFoundError:
-        with open('/pyproject.toml', mode='rb') as pyproject_file:
-            default_version = tomli.load(pyproject_file)['tool']['poetry']['version']
     finally:
         if not default_version:
             default_version = 'Version Unidentified'

diff --git a/weather_provider_api/main.py b/weather_provider_api/main.py
@@ -50,20 +50,22 @@
 mount_api_version(app, v1)
 mount_api_version(app, v2)
 
+
 # Redirect users to the docs
 @app.get("/")
 def redirect_to_docs():
     redirect_url = "/api/v2/docs"  # replace with docs URL or use weather_provider_api.url_path_for()
     return RedirectResponse(url=redirect_url)
 
+
 logger.info(f'--------------------------------------', datetime=datetime.utcnow())
 logger.info(f'Finished booting; starting uvicorn...', datetime=datetime.utcnow())
 logger.info(f'--------------------------------------', datetime=datetime.utcnow())
 
+
 def main():
     uvicorn.run(app, host="127.0.0.1", port=8080)
 
 
 if __name__ == "__main__":
     main()
-
diff --git a/weather_provider_api/routers/weather/api_models.py b/weather_provider_api/routers/weather/api_models.py
@@ -21,6 +21,7 @@
     API request models, response models, and examples.
 """
 
+
 # Note: while this can be done automatically, and for e.g. sources as well,
 # I don't want new OpenAPI specs to be created when e.g. a source is added.
 
@@ -94,6 +95,21 @@ class WeatherContentRequestQuery:
     )
 
 
+@dataclass
+class WeatherContentRequestMultiLocationQuery:
+    begin: str = Query(
+        None, description="From date and time", example="2019-01-01 00:00"
+    )
+    end: str = Query(None, description="To date and time", example="2019-01-31 23:59")
+    locations: str = Query(
+        None, description="Locations in either WGS84 (lat,lon) or RD (x,y) format, "
+                          "in parentheses, separated by a comma", example='(52.1, 5.18), (52.2, 5.22)'
+    )
+    factors: List[str] = Query(
+        None, description="Only return these weather factors (default: all factors)"
+    )
+
+
 class WeatherContentRequestBody(BaseModel):
     begin: str = Field(
         None, description="From date and time", example="2019-01-01 00:00"
@@ -137,7 +153,7 @@ def iterencode(self, obj, **kwargs):
             elif obj == -FloatEncoder.INFINITY:
                 yield "-Infinity"
             else:
-                yield format(obj, ".2f")
+                yield format(obj, ".4f")
         elif isinstance(obj, dict):
             last_index = len(obj) - 1
             yield "{"

diff --git a/weather_provider_api/routers/weather/api_view_v2.py b/weather_provider_api/routers/weather/api_view_v2.py
@@ -23,7 +23,7 @@
     WeatherFormattingRequestQuery,
     WeatherModel,
     WeatherSource,
-    result_mime_types,
+    result_mime_types, WeatherContentRequestMultiLocationQuery,
 )
 from weather_provider_api.routers.weather.controller import WeatherController
 from weather_provider_api.routers.weather.sources.weather_alert.weather_alert import WeatherAlert
@@ -184,3 +184,66 @@ async def get_alarm():  # pragma: no cover
     """
     weather_alert = WeatherAlert()
     return weather_alert.get_alarm()
+
+
+# Handler for requests with multiple locations:
+@app.get("/sources/{source_id}/models/{model_id}/multiple-locations/", tags=["sync"])
+async def get_sync_weather_multi_loc(
+    source_id: str,
+    model_id: str,
+    cleanup_tasks: BackgroundTasks,
+    ret_args: WeatherContentRequestMultiLocationQuery = Depends(),
+    fmt_args: WeatherFormattingRequestQuery = Depends(),
+    accept: str = Depends(header_accept_type),
+):  # pragma: no cover
+    source_id = source_id.lower()
+    model_id = model_id.lower()
+
+    coords = controller.str_to_coords(ret_args.locations)
+
+    begin = parse_datetime(ret_args.begin, raise_errors=True, loc=["query", "begin"])
+    end = parse_datetime(
+        ret_args.end,
+        round_missing_time_up=True,
+        raise_errors=True,
+        loc=["query", "end"],
+    )
+
+    try:
+        weather_data = controller.get_weather(
+            source_id,
+            model_id,
+            fetch_async=False,
+            coords=coords,
+            begin=begin,
+            end=end,
+            factors=ret_args.factors,
+        )
+    except FileNotFoundError as e:
+        raise HTTPException(status_code=404, detail=e.args[0])
+
+    if weather_data is None:
+        raise HTTPException(
+            status_code=404, detail="No data was found for the given period"
+        )
+
+    response_format = fmt_args.response_format or accept
+
+    converted_weather_data = controller.convert_names_and_units(
+        source_id, model_id, False, weather_data, fmt_args.units
+    )
+
+    coords = [
+        (lat_val, lon_val)
+        for (lat_val, lon_val) in zip(
+            converted_weather_data.coords["lat"].values,
+            converted_weather_data.coords["lon"].values,
+        )
+    ]
+
+    response, optional_file_path = serializers.file_or_text_response(
+        converted_weather_data, response_format, source_id, model_id, ret_args, coords
+    )
+    cleanup_tasks.add_task(remove_file, optional_file_path)
+
+    return response