Merge pull request #1044 from OasisLMF/release/2.3.5

* Fix missing filestore arg in complex model file prep (#1041) * Fixed missing call to param update method - V1 Workers (#1045) * Add missing calls to method param func, this checks and aligns any deprecated param names * Fix unit test * Azure Postgres Flexible server support (#1040) * Update keycloak to mount in cert from KeyVault * Test chart install - minikube without azure values * Fix running V2 workers with custom OED specification files (#1051) * fix "oed_schema_info" getting removed from task params * Pass oed_custom_spec into OedExposure load * pep * Updated Package Requirements: pymysql==1.1.1 --------- Co-authored-by: awsbuild <[email protected]> * Set Test version 2.3.5rc2 * Fix unittest (#1055) * tAdd stable/2.3.x to compatibility checks and fixed V2 worker monitor issue (#1054) * Add last stable 2.3.x image to compatibility checks * Fix failed jobs * f * Add check for raw_output_locations in worker result * Updated Package Requirements aiohttp==3.9.4 * Set scan external to manual trigger only (#1058) * Set version 2.3.5 (#1060) Co-authored-by: awsbuild <[email protected]> * Set version 2.3.5 * Update changelog --------- Co-authored-by: awsbuild <[email protected]>
OasisLMF · May 30, 2024 · 4d4ef58 · 4d4ef58
2 parents be24e30 + 0637527
commit 4d4ef58
Show file tree

Hide file tree

Showing 14 changed files with 156 additions and 68 deletions.
diff --git a/.github/workflows/scan-external.yml b/.github/workflows/scan-external.yml
@@ -1,13 +1,6 @@
 name: External Vulnerability Scanning
 
 on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - stable**
-
   workflow_dispatch:
     inputs:
       cve_severity:

diff --git a/.github/workflows/test-images.yml b/.github/workflows/test-images.yml
@@ -55,10 +55,11 @@ jobs:
       piwind_branch: ${{ steps.piwind.outputs.branch }}
 
       release_tag: ${{ steps.released_images.outputs.prev_release_tag }}
-      release_stable_15: ${{ steps.released_images.outputs.stable_15 }}
-      release_stable_23: ${{ steps.released_images.outputs.stable_23 }}
-      release_stable_27: ${{ steps.released_images.outputs.stable_27 }}
-      release_stable_28: ${{ steps.released_images.outputs.stable_28 }}
+      release_stable_1-15: ${{ steps.released_images.outputs.stable_1-15 }}
+      release_stable_1-23: ${{ steps.released_images.outputs.stable_1-23 }}
+      release_stable_1-27: ${{ steps.released_images.outputs.stable_1-27 }}
+      release_stable_1-28: ${{ steps.released_images.outputs.stable_1-28 }}
+      release_stable_2-3: ${{ steps.released_images.outputs.stable_2-3 }}
 
       build_server_img: ${{ steps.built_images.outputs.server_img }}
       build_server_tag: ${{ steps.built_images.outputs.server_tag }}
@@ -91,16 +92,19 @@ jobs:
 
           # Find latest LTS from each stable branch
           stable_1_15=$( ./scripts/find_latest.sh -j 1 -i 15 )
-          echo "stable_15=$stable_1_15" >> $GITHUB_OUTPUT
+          echo "stable_1-15=$stable_1_15" >> $GITHUB_OUTPUT
 
           stable_1_23=$( ./scripts/find_latest.sh -j 1 -i 23 )
-          echo "stable_23=$stable_1_23" >> $GITHUB_OUTPUT
+          echo "stable_1-23=$stable_1_23" >> $GITHUB_OUTPUT
 
           stable_1_27=$( ./scripts/find_latest.sh -j 1 -i 27 )
-          echo "stable_27=$stable_1_27" >> $GITHUB_OUTPUT
+          echo "stable_1-27=$stable_1_27" >> $GITHUB_OUTPUT
 
           stable_1_28=$( ./scripts/find_latest.sh -j 1 -i 28 )
-          echo "stable_28=$stable_1_28" >> $GITHUB_OUTPUT
+          echo "stable_1-28=$stable_1_28" >> $GITHUB_OUTPUT
+
+          stable_2_3=$( ./scripts/find_latest.sh -j 2 -i 3 )
+          echo "stable_2-3=$stable_2_3" >> $GITHUB_OUTPUT
 
       - name: Select PiWind branch
         id: piwind
@@ -229,8 +233,8 @@ jobs:
       pytest_opts: "--docker-compose=./docker/plat2-v2.docker-compose.yml ${{ needs.setup.outputs.pytest_opts }}"
       storage_suffix: '-worker-debian'
 
-  stable_compatibility_15:
-    name: Test stable worker (${{ needs.setup.outputs.release_stable_15 }})
+  stable_compatibility_1-15:
+    name: Test stable worker (${{ needs.setup.outputs.release_stable_1-15 }})
     secrets: inherit
     needs: [setup]
     uses: OasisLMF/OasisPiWind/.github/workflows/integration.yml@main
@@ -239,14 +243,14 @@ jobs:
       server_image: ${{ needs.setup.outputs.build_server_img }}
       server_tag: ${{ needs.setup.outputs.build_server_tag }}
       worker_image: 'coreoasis/model_worker'
-      worker_tag: ${{ needs.setup.outputs.release_stable_15 }}
+      worker_tag: ${{ needs.setup.outputs.release_stable_1-15 }}
       worker_api_ver: 'v1'
       debug_mode: 1
       pytest_opts: "--docker-compose=./docker/plat2-v2.docker-compose.yml ${{ needs.setup.outputs.pytest_opts }}"
-      storage_suffix: "_worker-${{ needs.setup.outputs.release_stable_15 }}"
+      storage_suffix: "_worker-${{ needs.setup.outputs.release_stable_1-15 }}"
 
-  stable_compatibility_23:
-    name: Test stable worker (${{ needs.setup.outputs.release_stable_23 }})
+  stable_compatibility_1-23:
+    name: Test stable worker (${{ needs.setup.outputs.release_stable_1-23 }})
     secrets: inherit
     needs: [setup]
     uses: OasisLMF/OasisPiWind/.github/workflows/integration.yml@main
@@ -255,14 +259,14 @@ jobs:
       server_image: ${{ needs.setup.outputs.build_server_img }}
       server_tag: ${{ needs.setup.outputs.build_server_tag }}
       worker_image: 'coreoasis/model_worker'
-      worker_tag: ${{ needs.setup.outputs.release_stable_23 }}
+      worker_tag: ${{ needs.setup.outputs.release_stable_1-23 }}
       worker_api_ver: 'v1'
       debug_mode: 1
       pytest_opts: "--docker-compose=./docker/plat2-v2.docker-compose.yml ${{ needs.setup.outputs.pytest_opts }}"
-      storage_suffix: "_worker-${{ needs.setup.outputs.release_stable_23 }}"
+      storage_suffix: "_worker-${{ needs.setup.outputs.release_stable_1-23 }}"
 
-  stable_compatibility_27:
-    name: Test stable worker (${{ needs.setup.outputs.release_stable_27 }})
+  stable_compatibility_1-27:
+    name: Test stable worker (${{ needs.setup.outputs.release_stable_1-27 }})
     secrets: inherit
     needs: [setup]
     uses: OasisLMF/OasisPiWind/.github/workflows/integration.yml@main
@@ -271,14 +275,14 @@ jobs:
       server_image: ${{ needs.setup.outputs.build_server_img }}
       server_tag: ${{ needs.setup.outputs.build_server_tag }}
       worker_image: 'coreoasis/model_worker'
-      worker_tag: ${{ needs.setup.outputs.release_stable_27 }}
+      worker_tag: ${{ needs.setup.outputs.release_stable_1-27 }}
       worker_api_ver: 'v1'
       debug_mode: 1
       pytest_opts: "--docker-compose=./docker/plat2-v2.docker-compose.yml ${{ needs.setup.outputs.pytest_opts }}"
-      storage_suffix: "_worker-${{ needs.setup.outputs.release_stable_27 }}"
+      storage_suffix: "_worker-${{ needs.setup.outputs.release_stable_1-27 }}"
 
-  stable_compatibility_28:
-    name: Test stable worker (${{ needs.setup.outputs.release_stable_28 }})
+  stable_compatibility_1-28:
+    name: test stable worker (${{ needs.setup.outputs.release_stable_1-28 }})
     secrets: inherit
     needs: [setup]
     uses: OasisLMF/OasisPiWind/.github/workflows/integration.yml@main
@@ -287,9 +291,25 @@ jobs:
       server_image: ${{ needs.setup.outputs.build_server_img }}
       server_tag: ${{ needs.setup.outputs.build_server_tag }}
       worker_image: 'coreoasis/model_worker'
-      worker_tag: ${{ needs.setup.outputs.release_stable_28 }}
+      worker_tag: ${{ needs.setup.outputs.release_stable_1-28 }}
       worker_api_ver: 'v1'
       debug_mode: 1
       pytest_opts: "--docker-compose=./docker/plat2-v2.docker-compose.yml ${{ needs.setup.outputs.pytest_opts }}"
-      storage_suffix: "_worker-${{ needs.setup.outputs.release_stable_28 }}"
+      storage_suffix: "_worker-${{ needs.setup.outputs.release_stable_1-28 }}"
+
+  stable_compatibility_2-3:
+    name: test stable worker (${{ needs.setup.outputs.release_stable_2-3 }})
+    secrets: inherit
+    needs: [setup]
+    uses: OasisLMF/OasisPiWind/.github/workflows/integration.yml@main
+    with:
+      piwind_branch: 'stable/2.3.x'
+      server_image: ${{ needs.setup.outputs.build_server_img }}
+      server_tag: ${{ needs.setup.outputs.build_server_tag }}
+      worker_image: 'coreoasis/model_worker'
+      worker_tag: ${{ needs.setup.outputs.release_stable_2-3 }}
+      worker_api_ver: 'v2'
+      debug_mode: 1
+      pytest_opts: "--docker-compose=./docker/plat2-v2.docker-compose.yml ${{ needs.setup.outputs.pytest_opts }}"
+      storage_suffix: "_worker-${{ needs.setup.outputs.release_stable_2-3 }}"
 
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -1,6 +1,19 @@
 OasisPlatform Changelog
 =======================
 
+`2.3.5`_
+ ---------
+* [#1058](https://github.com/OasisLMF/OasisPlatform/pull/1058) -  CI - Disable external docker images scanning 
+* [#1060](https://github.com/OasisLMF/OasisPlatform/pull/1060) - Set version 2.3.5
+* [#1040](https://github.com/OasisLMF/OasisPlatform/pull/1040) - Azure Postgres Flexible server support   
+* [#1038, #1039](https://github.com/OasisLMF/OasisPlatform/pull/1041) - Minor bug fixes for worker and server
+* [#1042](https://github.com/OasisLMF/OasisPlatform/pull/1045) - Check older paramter names are updated and working in 2.3.4  
+* [#1051](https://github.com/OasisLMF/OasisPlatform/pull/1051) - Fix running V2 workers with custom OED specification files
+* [#1020](https://github.com/OasisLMF/OasisPlatform/pull/1020) - Release 2.3.4 (staging)
+* [#1052](https://github.com/OasisLMF/OasisPlatform/pull/1054) - Lot3 - worker monitor compatibility fix needed 
+* [#1055](https://github.com/OasisLMF/OasisPlatform/pull/1055) - Fix failing unittest in release branch 2.3.5
+.. _`2.3.5`:  https://github.com/OasisLMF/OasisPlatform/compare/2.3.4...2.3.5
+
 `2.3.4`_
  ---------
 * [#1023](https://github.com/OasisLMF/OasisPlatform/pull/1024) - Error reading sub-task logs from S3 when AWS_LOCATION is set

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-2.3.4
+2.3.5
diff --git a/kubernetes/charts/oasis-platform/templates/keycloak.yaml b/kubernetes/charts/oasis-platform/templates/keycloak.yaml
@@ -98,6 +98,12 @@ spec:
           ports:
             - containerPort: {{ .Values.keycloak.port }}
           env:
+           {{- if (.Values.azure).secretProvider }}
+           {{- if hasKey .Values.azure.secretProvider.secrets "keycloak-cert" }}
+            - name: KC_DB_URL_PROPERTIES
+              value: "?sslmode=verify-full&sslcert=root.crt"
+           {{- end }}
+           {{- end }}
             - name: KC_LOGLEVEL
               value: DEBUG
             - name: PROXY_ADDRESS_FORWARDING
@@ -172,16 +178,34 @@ spec:
             - name: realm-config
               mountPath: /opt/keycloak/data/import/oasis-realm.json
               subPath: oasis
+            {{- if (.Values.azure).secretProvider }}
+            {{- if hasKey .Values.azure.secretProvider.secrets "keycloak-cert" }}
+            - name: azure-keycloak-cert
+              mountPath: /opt/keycloak/.postgresql/root.crt
+              subPath: keycloak-cert-file
+              readOnly: true
+            {{- end }}
+            {{- end }}
 
       volumes:
         - name: realm-config
           configMap:
             name: {{ $realmSecretName }}
-{{- if (.Values.azure).secretProvider }}
+    {{- if (.Values.azure).secretProvider }}
+      {{- if hasKey .Values.azure.secretProvider.secrets "keycloak-cert" }}
+        - name: azure-keycloak-cert
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: "azure-secret-provider"
+              objectName: "keycloak-cert-file"
+              objectType: "secret"
+      {{- end }}
         - name: azure-secret-provider
           csi:
             driver: secrets-store.csi.k8s.io
             readOnly: true
             volumeAttributes:
               secretProviderClass: "azure-secret-provider"
-{{- end }}
+    {{- end }}
diff --git a/kubernetes/worker-controller/requirements.txt b/kubernetes/worker-controller/requirements.txt
@@ -4,7 +4,7 @@
 #
 #    pip-compile kubernetes/worker-controller/requirements.in
 #
-aiohttp==3.9.0
+aiohttp==3.9.4
     # via
     #   -r kubernetes/worker-controller/requirements.in
     #   kubernetes-asyncio

diff --git a/requirements-server.txt b/requirements-server.txt
@@ -209,9 +209,9 @@ numpy==1.25.2
     #   numba
     #   pandas
     #   pyarrow
-oasis-data-manager==0.1.1
+oasis-data-manager==0.1.3
     # via ods-tools
-ods-tools==3.2.3
+ods-tools==3.2.4
     # via -r requirements-server.in
 packaging==23.0
     # via
@@ -243,7 +243,7 @@ pycparser==2.21
     # via cffi
 pyjwt==2.6.0
     # via djangorestframework-simplejwt
-pymysql==1.1.0
+pymysql==1.1.1
     # via -r requirements-server.in
 pyopenssl==24.0.0
     # via
@@ -302,8 +302,6 @@ twisted[tls]==22.10.0
     # via daphne
 txaio==23.1.1
     # via autobahn
-typing==3.7.4.3
-    # via oasis-data-manager
 typing-extensions==4.5.0
     # via
     #   azure-core

diff --git a/requirements-worker.txt b/requirements-worker.txt
@@ -149,9 +149,9 @@ oasis-data-manager==0.1.1
     # via
     #   oasislmf
     #   ods-tools
-oasislmf[extra]==2.3.4
+oasislmf[extra]==2.3.5
     # via -r requirements-worker.in
-ods-tools==3.2.3
+ods-tools==3.2.4
     # via oasislmf
 packaging==23.0
     # via
@@ -178,7 +178,7 @@ pyarrow==16.0.0
     # via oasislmf
 pycparser==2.21
     # via cffi
-pymysql==1.1.0
+pymysql==1.1.1
     # via -r requirements-worker.in
 pyproj==3.5.0
     # via geopandas

diff --git a/requirements.txt b/requirements.txt
@@ -340,9 +340,9 @@ oasis-data-manager==0.1.1
     # via
     #   oasislmf
     #   ods-tools
-oasislmf[extra]==2.3.4
+oasislmf[extra]==2.3.5
     # via -r ./requirements-worker.in
-ods-tools==3.2.3
+ods-tools==3.2.4
     # via
     #   -r ./requirements-server.in
     #   oasislmf
@@ -415,9 +415,9 @@ pyflakes==3.0.1
     # via flake8
 pygments==2.14.0
     # via ipython
-pyjwt==2.6.0
+pyjwt==2.8.0
     # via djangorestframework-simplejwt
-pymysql==1.1.0
+pymysql==1.1.1
     # via
     #   -r ./requirements-server.in
     #   -r ./requirements-worker.in

diff --git a/src/model_execution_worker/distributed_tasks.py b/src/model_execution_worker/distributed_tasks.py
@@ -37,6 +37,7 @@
     get_worker_versions,
     merge_dirs,
     prepare_complex_model_file_inputs,
+    config_strip_default_exposure,
 )
 
 
@@ -97,7 +98,7 @@ def notify_subtask_status(analysis_id, initiator_id, task_slug, subtask_status,
     ).delay()
 
 
-def load_location_data(loc_filepath):
+def load_location_data(loc_filepath, oed_schema_info=None):
     """ Returns location file as DataFrame
 
     Returns a DataFrame of Loaction data with 'loc_id' row assgined
@@ -112,7 +113,10 @@ def load_location_data(loc_filepath):
         from oasislmf.utils.data import prepare_location_df
         from ods_tools.oed.exposure import OedExposure
 
-        exposure = OedExposure(location=pathlib.Path(os.path.abspath(loc_filepath)))
+        exposure = OedExposure(
+            location=pathlib.Path(os.path.abspath(loc_filepath)),
+            oed_schema_info=oed_schema_info,
+        )
         exposure.location.dataframe = prepare_location_df(exposure.location.dataframe)
         return exposure.location.dataframe
 
@@ -481,8 +485,8 @@ def prepare_input_generation_params(
 
     model_id = settings.get('worker', 'model_id')
     config_path = get_oasislmf_config_path(settings, model_id)
-    config = get_json(config_path)
-    lookup_params = {**{k: v for k, v in config.items() if not k.startswith('oed_')}, **params}
+    config = config_strip_default_exposure(get_json(config_path))
+    lookup_params = {**config, **params}
 
     from oasislmf.manager import OasisManager
     gen_files_params = OasisManager()._params_generate_files(**lookup_params)
@@ -569,7 +573,10 @@ def prepare_keys_file_chunk(
             output_directory=chunk_target_dir,
         )
 
-        location_df = load_location_data(params['oed_location_csv'])
+        location_df = load_location_data(
+            loc_filepath=params['oed_location_csv'],
+            oed_schema_info=params.get('oed_schema_info', None)
+        )
         location_df = np.array_split(location_df, num_chunks)[chunk_idx]
         location_df.reset_index(drop=True, inplace=True)
 
@@ -872,7 +879,7 @@ def prepare_losses_generation_params(
 
     model_id = settings.get('worker', 'model_id')
     config_path = get_oasislmf_config_path(settings, model_id)
-    config = get_json(config_path)
+    config = config_strip_default_exposure(get_json(config_path))
     run_params = {**config, **params}
 
     from oasislmf.manager import OasisManager