Merge pull request #1046 from OasisLMF/release/1.28.8

Release 1.28.8 (30th May 2024)

CHANGELOG.rst

@@ -1,6 +1,13 @@
 OasisPlatform Changelog
 =======================
 
+`1.28.8`_
+ ---------
+* [#1000](https://github.com/OasisLMF/OasisPlatform/pull/1000) - Release 1.28.7
+* [#1059](https://github.com/OasisLMF/OasisPlatform/pull/1059) - Backport 1.28.x -  Fixed missing call to param update method - V1 Workers (#1045)
+* [#1042](https://github.com/OasisLMF/OasisPlatform/pull/1045) - Check older paramter names are updated and working in 2.3.4  
+.. _`1.28.8`:  https://github.com/OasisLMF/OasisPlatform/compare/1.28.7...1.28.8
+
 `1.28.7`_
  ---------
 * [#993, #971, #784, #664, #798](https://github.com/OasisLMF/OasisPlatform/pull/994) - Logging fixes for workers 

VERSION

@@ -1 +1 @@
-1.28.7
+1.28.8

requirements-server.txt

@@ -118,7 +118,7 @@ drf-yasg==1.21.5
     # via -r requirements-server.in
 greenlet==2.0.2
     # via sqlalchemy
-gunicorn==20.1.0
+gunicorn==22.0.0
     # via -r requirements-server.in
 hyperlink==21.0.0
     # via
@@ -168,11 +168,12 @@ numpy==1.24.2
     #   numba
     #   pandas
     #   pyarrow
-ods-tools==3.1.4
+ods-tools==3.1.5
     # via -r requirements-server.in
 packaging==23.0
     # via
     #   drf-yasg
+    #   gunicorn
     #   ods-tools
 pandas==1.5.3
     # via
@@ -196,7 +197,7 @@ pycparser==2.21
     # via cffi
 pyjwt==2.6.0
     # via djangorestframework-simplejwt
-pymysql==1.1.0
+pymysql==1.1.1
     # via -r requirements-server.in
 pyopenssl==24.0.0
     # via twisted
@@ -236,7 +237,7 @@ six==1.16.0
     #   service-identity
 sqlalchemy==2.0.5.post1
     # via -r requirements-server.in
-sqlparse==0.4.4
+sqlparse==0.5.0
     # via
     #   django
     #   django-debug-toolbar

requirements-worker.txt

@@ -128,9 +128,9 @@ numpy==1.22.4
     #   scikit-learn
     #   scipy
     #   shapely
-oasislmf[extra]==1.28.7
+oasislmf[extra]==1.28.8
     # via -r requirements-worker.in
-ods-tools==3.1.4
+ods-tools==3.1.5
     # via oasislmf
 packaging==23.0
     # via
@@ -156,7 +156,7 @@ pyarrow==14.0.1
     # via oasislmf
 pycparser==2.21
     # via cffi
-pymysql==1.1.0
+pymysql==1.1.1
     # via -r requirements-worker.in
 pyproj==3.4.1
     # via geopandas

requirements.txt

@@ -199,7 +199,7 @@ geopandas==0.12.2
     # via oasislmf
 greenlet==2.0.2
     # via sqlalchemy
-gunicorn==20.1.0
+gunicorn==22.0.0
     # via -r ./requirements-server.in
 hyperlink==21.0.0
     # via
@@ -276,9 +276,9 @@ numpy==1.22.4
     #   scikit-learn
     #   scipy
     #   shapely
-oasislmf[extra]==1.28.7
+oasislmf[extra]==1.28.8
     # via -r ./requirements-worker.in
-ods-tools==3.1.4
+ods-tools==3.1.5
     # via
     #   -r ./requirements-server.in
     #   oasislmf
@@ -288,6 +288,7 @@ packaging==23.0
     #   drf-yasg
     #   fastparquet
     #   geopandas
+    #   gunicorn
     #   ods-tools
     #   pyproject-api
     #   pytest
@@ -337,7 +338,7 @@ pyflakes==3.0.1
     # via flake8
 pyjwt==2.6.0
     # via djangorestframework-simplejwt
-pymysql==1.1.0
+pymysql==1.1.1
     # via
     #   -r ./requirements-server.in
     #   -r ./requirements-worker.in
@@ -428,7 +429,7 @@ sqlalchemy==2.0.5.post1
     # via
     #   -r ./requirements-server.in
     #   -r ./requirements-worker.in
-sqlparse==0.4.4
+sqlparse==0.5.0
     # via
     #   django
     #   django-debug-toolbar

src/model_execution_worker/tasks.py

@@ -17,7 +17,6 @@
 from celery.signals import worker_ready
 from celery.exceptions import WorkerLostError, Terminated
 
-from oasislmf.manager import OasisManager
 from oasislmf.utils.data import get_json
 from oasislmf.utils.exceptions import OasisException
 from oasislmf.utils.status import OASIS_TASK_STATUS
@@ -284,6 +283,8 @@ def start_analysis(analysis_settings, input_location, complex_data_files=None, *
 
     """
     # Check that the input archive exists and is valid
+    from oasislmf.manager import OasisManager
+
     logger.info("args: {}".format(str(locals())))
     logger.info(str(get_worker_versions()))
     tmpdir_persist = settings.getboolean('worker', 'KEEP_RUN_DIR', fallback=False)
@@ -330,7 +331,10 @@ def start_analysis(analysis_settings, input_location, complex_data_files=None, *
 
         # Create and log params
         run_params = {**config, **task_params}
-        params = paths_to_absolute_paths(run_params, config_path)
+        gen_losses_params = OasisManager()._params_generate_losses(**run_params)
+        post_hook_params = OasisManager()._params_post_analysis(**run_params)
+        params = paths_to_absolute_paths({**gen_losses_params, **post_hook_params}, config_path)
+
         if debug_worker:
             log_params(params, kwargs)
 
@@ -391,7 +395,9 @@ def generate_input(self,
 
     # Start Oasis file generation
     notify_api_status(analysis_pk, 'INPUTS_GENERATION_STARTED')
-    filestore.media_root = settings.get('worker', 'MEDIA_ROOT')
+    from oasislmf.manager import OasisManager
+
+    # filestore.media_root = settings.get('worker', 'MEDIA_ROOT')
     tmpdir_persist = settings.getboolean('worker', 'KEEP_RUN_DIR', fallback=False)
     tmpdir_base = settings.get('worker', 'BASE_RUN_DIR', fallback=None)
 
@@ -432,7 +438,11 @@ def generate_input(self,
         config_path = get_oasislmf_config_path(settings)
         config = get_json(config_path)
         lookup_params = {**{k: v for k, v in config.items() if not k.startswith('oed_')}, **task_params}
-        params = paths_to_absolute_paths(lookup_params, config_path)
+
+        gen_files_params = OasisManager()._params_generate_files(**lookup_params)
+        pre_hook_params = OasisManager()._params_exposure_pre_analysis(**lookup_params)
+        params = paths_to_absolute_paths({**gen_files_params, **pre_hook_params}, config_path)
+
         if debug_worker:
             log_params(params, kwargs, exclude_keys=[
                 'profile_loc',

tests/test_tasks.py

@@ -109,21 +109,26 @@ def test_custom_model_runner_does_not_exist___generate_losses_is_called_output_f
                 def fake_run_dir(*args, **kwargs):
                     yield run_dir
 
-                with patch('src.model_execution_worker.tasks.OasisManager', Mock(return_value=cmd_instance)) as cmd_mock, \
+                with patch('oasislmf.manager.OasisManager.generate_oasis_losses', Mock(return_value='mocked result')) as cmd_mock, \
                         patch('src.model_execution_worker.tasks.get_worker_versions', Mock(return_value='')), \
                         patch('src.model_execution_worker.tasks.filestore.compress') as tarfile, \
                         patch('src.model_execution_worker.tasks.TASK_LOG_DIR', log_dir), \
                         patch('src.model_execution_worker.tasks.TemporaryDir', fake_run_dir):
 
-                    cmd_instance.generate_oasis_losses.return_value = "mocked result"  # Mock the return value
                     output_location, log_location, error_location, returncode = start_analysis(
                         os.path.join(media_root, 'analysis_settings.json'),
                         os.path.join(media_root, 'location.tar'),
                         log_filename=log_file,
                     )
-                    expected_params = {**params, **{"analysis_settings_json": os.path.join(media_root, 'analysis_settings.json')}}
-                    cmd_instance.generate_oasis_losses.assert_called_once_with(**expected_params)
-                    tarfile.assert_called_once_with(os.path.join(media_root, output_location), os.path.join(run_dir, 'output'), 'output')
+
+                    cmd_mock.assert_called_once()
+                    called_args = cmd_mock.call_args.kwargs
+                    self.assertEqual(called_args.get('oasis_files_dir', None), params.get('oasis_files_dir'))
+                    self.assertEqual(called_args.get('model_run_dir', None), params.get('model_run_dir'))
+                    self.assertEqual(called_args.get('ktools_fifo_relative', None), params.get('ktools_fifo_relative'))
+                    self.assertEqual(called_args.get('verbose', None), params.get('verbose'))
+                    self.assertEqual(called_args.get('analysis_settings.json', None), params.get('analysis_settings.json'))
+                    tarfile.assert_called_once_with(ANY, os.path.join(run_dir, 'output'), 'output')
 
 
 class StartAnalysisTask(TestCase):