PR pipeline restructuring - one workflow to call multiple workflows .…

….. (#1294) * PR pipeline restructuring - one workflow calls multiple workflows Signed-off-by: noopur <[email protected]> * Bandit scan needs specific permissions Signed-off-by: noopur <[email protected]> * Specific permissions for Trivy scan Signed-off-by: noopur <[email protected]> * Final changes Signed-off-by: noopur <[email protected]> * Adding job names for better readability Signed-off-by: noopur <[email protected]> --------- Signed-off-by: noopur <[email protected]>
securefederatedai · Jan 23, 2025 · ecd3603 · ecd3603
1 parent 9d45112
commit ecd3603
Show file tree

Hide file tree

Showing 19 changed files with 181 additions and 39 deletions.
diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml
@@ -5,8 +5,8 @@ on:
     branches: 
       - develop
       - v1.7.x
-  pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
+  workflow_call:
+  workflow_dispatch:
 
 jobs:
   bandit_scan:

diff --git a/.github/workflows/docker-bench-security.yml b/.github/workflows/docker-bench-security.yml
@@ -1,8 +1,8 @@
 name: Docker Bench for Security
 
 on:
-  pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
+  workflow_call:
+  workflow_dispatch:
 
 permissions:
   contents: read

diff --git a/.github/workflows/double_ws_export.yml b/.github/workflows/double_ws_export.yml
@@ -4,8 +4,8 @@
 name: Double workspace export
 
 on:
-  pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
+  workflow_call:
+  workflow_dispatch:
 
 permissions:
   contents: read

diff --git a/.github/workflows/experimental_workflow_tests.yml b/.github/workflows/experimental_workflow_tests.yml
@@ -5,8 +5,8 @@ on:
     branches: 
       - develop
       - v1.7.x
-  pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
+  workflow_call:
+  workflow_dispatch:
 
 permissions:
   contents: read

diff --git a/.github/workflows/federated_runtime.yml b/.github/workflows/federated_runtime.yml
@@ -4,9 +4,7 @@
 name: Federated Runtime 301 MNIST Watermarking
 
 on:
-  pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
-
+  workflow_call:
   workflow_dispatch:
 
 permissions:

diff --git a/.github/workflows/gandlf.yml b/.github/workflows/gandlf.yml
@@ -4,8 +4,8 @@
 name: GaNDLF TaskRunner
 
 on:
-  pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
+  workflow_call:
+  workflow_dispatch:
 
 permissions:
   contents: read

diff --git a/.github/workflows/hadolint.yml b/.github/workflows/hadolint.yml
@@ -4,8 +4,8 @@
 name: Hadolint Security Scan
 
 on:
-  pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
+  workflow_call:
+  workflow_dispatch:
 
 permissions:
   contents: read

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -4,8 +4,8 @@
 name: Check code format
 
 on:
-  pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
+  workflow_call:
+  workflow_dispatch:
 
 permissions:
   contents: read

diff --git a/.github/workflows/pki.yml b/.github/workflows/pki.yml
@@ -4,8 +4,8 @@
 name: Private Key Infrastructure
 
 on:
-  pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
+  workflow_call:
+  workflow_dispatch:
 
 permissions:
   contents: read

diff --git a/.github/workflows/pr_pipeline.yml b/.github/workflows/pr_pipeline.yml
@@ -0,0 +1,147 @@
+name: OpenFL PR Pipeline
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.base_ref }}-${{ github.head_ref }}-${{ github.actor }}
+  cancel-in-progress: true
+
+jobs:
+  bandit_code_scan:
+    name: Bandit Code Scan
+    # DO NOT remove the permissions block. Without this, these permissions are assumed as none in the called workflow and the workflow fails.
+    permissions:
+      contents: read # for actions/checkout to fetch code
+      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
+      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
+    uses: ./.github/workflows/bandit.yml
+
+  check_code_format:
+    name: Check code format
+    uses: ./.github/workflows/lint.yml
+
+  docker_bench_security:
+    name: Docker Bench for Security
+    uses: ./.github/workflows/docker-bench-security.yml
+
+  double_workspace_export:
+    name: Double workspace export
+    uses: ./.github/workflows/double_ws_export.yml
+
+  fr_301_watermark_nb_run:
+    name: Federated Runtime 301 MNIST Watermarking
+    uses: ./.github/workflows/federated_runtime.yml
+
+  gandlf_taskrunner:
+    name: GaNDLF TaskRunner
+    uses: ./.github/workflows/gandlf.yml
+
+  hadolint_security_scan:
+    name: Hadolint Security Scan
+    uses: ./.github/workflows/hadolint.yml
+
+  private_key_infrastructure:
+    name: Private Key Infrastructure
+    uses: ./.github/workflows/pki.yml
+
+  pytest_coverage:
+    name: Pytest and code coverage
+    uses: ./.github/workflows/pytest_coverage.yml
+
+  straggler_handling:
+    name: Straggler Handling Test
+    uses: ./.github/workflows/straggler-handling.yml
+
+  task_runner:
+    name: TaskRunner
+    uses: ./.github/workflows/taskrunner.yml
+
+  taskrunner_eden_compression:
+    name: TaskRunner (Eden Compression)
+    uses: ./.github/workflows/taskrunner_eden_pipeline.yml
+
+  tr_docker_gramine_direct:
+    name: TaskRunner (docker/gramine-direct)
+    uses: ./.github/workflows/tr_docker_gramine_direct.yml
+
+  tr_docker_native:
+    name: TaskRunner (docker/native)
+    uses: ./.github/workflows/tr_docker_native.yml
+
+  trivy_scan:
+    name: Trivy Scan
+    # DO NOT remove the permissions block. Without this, these permissions are assumed as none in the called workflow and the workflow fails.
+    permissions:
+      contents: read # for actions/checkout to fetch code
+      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
+      actions: read #  only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
+    uses: ./.github/workflows/trivy.yml
+
+  wf_functional_e2e:
+    name: Workflow Functional E2E
+    uses: ./.github/workflows/wf_functional_e2e.yml
+
+  workflow_interface_101_mnist:
+    name: Workflow Interface 101 MNIST Notebook
+    uses: ./.github/workflows/workflow_interface_101_mnist.yml
+
+  workflow_interface_tests:
+    name: Workflow Interface Tests
+    uses: ./.github/workflows/experimental_workflow_tests.yml
+
+  pipeline_status:
+    name: Pipeline Status
+    runs-on: ubuntu-22.04
+    if: always()
+    needs: # Ensure to add any job added above to this list as well as the steps below
+      [
+        bandit_code_scan,
+        check_code_format,
+        docker_bench_security,
+        double_workspace_export,
+        gandlf_taskrunner,
+        fr_301_watermark_nb_run,
+        hadolint_security_scan,
+        private_key_infrastructure,
+        pytest_coverage,
+        straggler_handling,
+        task_runner,
+        taskrunner_eden_compression,
+        trivy_scan,
+        tr_docker_gramine_direct,
+        tr_docker_native,
+        wf_functional_e2e,
+        workflow_interface_101_mnist,
+        workflow_interface_tests,
+      ]
+    steps:
+      - name: Fail job if any of the above jobs have failed
+        if: |
+          (needs.bandit_code_scan.result != 'success' && needs.bandit_code_scan.result != 'skipped') ||
+          (needs.check_code_format.result != 'success' && needs.check_code_format.result != 'skipped') ||
+          (needs.docker_bench_security.result != 'success' && needs.docker_bench_security.result != 'skipped') ||
+          (needs.double_workspace_export.result != 'success' && needs.double_workspace_export.result != 'skipped') ||
+          (needs.fr_301_watermark_nb_run.result != 'success' && needs.fr_301_watermark_nb_run.result != 'skipped') ||
+          (needs.gandlf_taskrunner.result != 'success' && needs.gandlf_taskrunner.result != 'skipped') ||
+          (needs.hadolint_security_scan.result != 'success' && needs.hadolint_security_scan.result != 'skipped') ||
+          (needs.private_key_infrastructure.result != 'success' && needs.private_key_infrastructure.result != 'skipped') ||
+          (needs.pytest_coverage.result != 'success' && needs.pytest_coverage.result != 'skipped') ||
+          (needs.straggler_handling.result != 'success' && needs.straggler_handling.result != 'skipped') ||
+          (needs.task_runner.result != 'success' && needs.task_runner.result != 'skipped') ||
+          (needs.taskrunner_eden_compression.result != 'success' && needs.taskrunner_eden_compression.result != 'skipped') ||
+          (needs.trivy_scan.result != 'success' && needs.trivy_scan.result != 'skipped') ||
+          (needs.tr_docker_gramine_direct.result != 'success' && needs.tr_docker_gramine_direct.result != 'skipped') ||
+          (needs.tr_docker_native.result != 'success' && needs.tr_docker_native.result != 'skipped') ||
+          (needs.wf_functional_e2e.result != 'success' && needs.wf_functional_e2e.result != 'skipped') ||
+          (needs.workflow_interface_101_mnist.result != 'success' && needs.workflow_interface_101_mnist.result != 'skipped') ||
+          (needs.workflow_interface_tests.result != 'success' && needs.workflow_interface_tests.result != 'skipped')
+        run: |
+          echo "One or more jobs failed. Exiting pipeline."
+          exit 1
+      
+      - name: Print pipeline status
+        run: |
+          echo "All jobs passed. Pipeline completed successfully."
diff --git a/.github/workflows/pytest_coverage.yml b/.github/workflows/pytest_coverage.yml
@@ -4,8 +4,7 @@
 name: Pytest and code coverage 
 
 on:
-  pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
+  workflow_call:
   workflow_dispatch:
 
 permissions:

diff --git a/.github/workflows/straggler-handling.yml b/.github/workflows/straggler-handling.yml
@@ -4,8 +4,8 @@
 name: Straggler Handling Test
 
 on:
-  pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
+  workflow_call:
+  workflow_dispatch:
 
 permissions:
   contents: read

diff --git a/.github/workflows/taskrunner.yml b/.github/workflows/taskrunner.yml
@@ -4,8 +4,8 @@
 name: TaskRunner
 
 on:
-  pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
+  workflow_call:
+  workflow_dispatch:
 
 permissions:
   contents: read

diff --git a/.github/workflows/taskrunner_eden_pipeline.yml b/.github/workflows/taskrunner_eden_pipeline.yml
@@ -4,8 +4,8 @@
 name: TaskRunner (Eden Compression)
 
 on:
-  pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
+  workflow_call:
+  workflow_dispatch:
 
 permissions:
   contents: read

diff --git a/.github/workflows/tr_docker_gramine_direct.yml b/.github/workflows/tr_docker_gramine_direct.yml
@@ -2,8 +2,8 @@
 name: TaskRunner (docker/gramine-direct)
 
 on:
-  pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
+  workflow_call:
+  workflow_dispatch:
 
 permissions:
   contents: read

diff --git a/.github/workflows/tr_docker_native.yml b/.github/workflows/tr_docker_native.yml
@@ -2,8 +2,8 @@
 name: TaskRunner (docker/native)
 
 on:
-  pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
+  workflow_call:
+  workflow_dispatch:
 
 permissions:
   contents: read

diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml
@@ -4,8 +4,9 @@ on:
     branches: 
       - develop
       - v1.7.x
-  pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
+  workflow_call:
+  workflow_dispatch:
+
 jobs:
   build:
     if: github.event.pull_request.draft == false

diff --git a/.github/workflows/wf_functional_e2e.yml b/.github/workflows/wf_functional_e2e.yml
@@ -4,9 +4,8 @@
 name: Workflow_Functional_E2E
 
 on:
-  pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
-
+  workflow_call:
+
   workflow_dispatch:
     inputs:
       num_rounds:

diff --git a/.github/workflows/workflow_interface_101_mnist.yml b/.github/workflows/workflow_interface_101_mnist.yml
@@ -5,9 +5,7 @@
 name: Workflow Interface 101 MNIST Notebook
 
 on:
-  pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
-
+  workflow_call:
   workflow_dispatch:
 
 permissions: