diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml index 646bebbc61..d5d3da0c7a 100644 --- a/.github/workflows/bandit.yml +++ b/.github/workflows/bandit.yml @@ -5,8 +5,8 @@ on: branches: - develop - v1.7.x - pull_request: - types: [opened, synchronize, reopened, ready_for_review] + workflow_call: + workflow_dispatch: jobs: bandit_scan: diff --git a/.github/workflows/docker-bench-security.yml b/.github/workflows/docker-bench-security.yml index 64a98d474b..4faa807151 100644 --- a/.github/workflows/docker-bench-security.yml +++ b/.github/workflows/docker-bench-security.yml @@ -1,8 +1,8 @@ name: Docker Bench for Security on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] + workflow_call: + workflow_dispatch: permissions: contents: read diff --git a/.github/workflows/double_ws_export.yml b/.github/workflows/double_ws_export.yml index da2304ffa8..165d9239ac 100644 --- a/.github/workflows/double_ws_export.yml +++ b/.github/workflows/double_ws_export.yml @@ -4,8 +4,8 @@ name: Double workspace export on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] + workflow_call: + workflow_dispatch: permissions: contents: read diff --git a/.github/workflows/experimental_workflow_tests.yml b/.github/workflows/experimental_workflow_tests.yml index 5f8391317d..ce30aaa7db 100644 --- a/.github/workflows/experimental_workflow_tests.yml +++ b/.github/workflows/experimental_workflow_tests.yml @@ -5,8 +5,8 @@ on: branches: - develop - v1.7.x - pull_request: - types: [opened, synchronize, reopened, ready_for_review] + workflow_call: + workflow_dispatch: permissions: contents: read diff --git a/.github/workflows/federated_runtime.yml b/.github/workflows/federated_runtime.yml index ead1d2791c..a6210eda33 100644 --- a/.github/workflows/federated_runtime.yml +++ b/.github/workflows/federated_runtime.yml @@ -4,9 +4,7 @@ name: Federated Runtime 301 MNIST Watermarking on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - + workflow_call: workflow_dispatch: permissions: diff --git a/.github/workflows/gandlf.yml b/.github/workflows/gandlf.yml index d80d8f443c..24949c510c 100644 --- a/.github/workflows/gandlf.yml +++ b/.github/workflows/gandlf.yml @@ -4,8 +4,8 @@ name: GaNDLF TaskRunner on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] + workflow_call: + workflow_dispatch: permissions: contents: read diff --git a/.github/workflows/hadolint.yml b/.github/workflows/hadolint.yml index a02c63a6d2..7e9dcd4963 100644 --- a/.github/workflows/hadolint.yml +++ b/.github/workflows/hadolint.yml @@ -4,8 +4,8 @@ name: Hadolint Security Scan on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] + workflow_call: + workflow_dispatch: permissions: contents: read diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index afa5ea6a2f..916eb5816a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -4,8 +4,8 @@ name: Check code format on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] + workflow_call: + workflow_dispatch: permissions: contents: read diff --git a/.github/workflows/pki.yml b/.github/workflows/pki.yml index f5e176fd6d..e8b650e31a 100644 --- a/.github/workflows/pki.yml +++ b/.github/workflows/pki.yml @@ -4,8 +4,8 @@ name: Private Key Infrastructure on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] + workflow_call: + workflow_dispatch: permissions: contents: read diff --git a/.github/workflows/pr_pipeline.yml b/.github/workflows/pr_pipeline.yml new file mode 100644 index 0000000000..4b6cdde289 --- /dev/null +++ b/.github/workflows/pr_pipeline.yml @@ -0,0 +1,147 @@ +name: OpenFL PR Pipeline + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.base_ref }}-${{ github.head_ref }}-${{ github.actor }} + cancel-in-progress: true + +jobs: + bandit_code_scan: + name: Bandit Code Scan + # DO NOT remove the permissions block. Without this, these permissions are assumed as none in the called workflow and the workflow fails. + permissions: + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status + uses: ./.github/workflows/bandit.yml + + check_code_format: + name: Check code format + uses: ./.github/workflows/lint.yml + + docker_bench_security: + name: Docker Bench for Security + uses: ./.github/workflows/docker-bench-security.yml + + double_workspace_export: + name: Double workspace export + uses: ./.github/workflows/double_ws_export.yml + + fr_301_watermark_nb_run: + name: Federated Runtime 301 MNIST Watermarking + uses: ./.github/workflows/federated_runtime.yml + + gandlf_taskrunner: + name: GaNDLF TaskRunner + uses: ./.github/workflows/gandlf.yml + + hadolint_security_scan: + name: Hadolint Security Scan + uses: ./.github/workflows/hadolint.yml + + private_key_infrastructure: + name: Private Key Infrastructure + uses: ./.github/workflows/pki.yml + + pytest_coverage: + name: Pytest and code coverage + uses: ./.github/workflows/pytest_coverage.yml + + straggler_handling: + name: Straggler Handling Test + uses: ./.github/workflows/straggler-handling.yml + + task_runner: + name: TaskRunner + uses: ./.github/workflows/taskrunner.yml + + taskrunner_eden_compression: + name: TaskRunner (Eden Compression) + uses: ./.github/workflows/taskrunner_eden_pipeline.yml + + tr_docker_gramine_direct: + name: TaskRunner (docker/gramine-direct) + uses: ./.github/workflows/tr_docker_gramine_direct.yml + + tr_docker_native: + name: TaskRunner (docker/native) + uses: ./.github/workflows/tr_docker_native.yml + + trivy_scan: + name: Trivy Scan + # DO NOT remove the permissions block. Without this, these permissions are assumed as none in the called workflow and the workflow fails. + permissions: + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status + uses: ./.github/workflows/trivy.yml + + wf_functional_e2e: + name: Workflow Functional E2E + uses: ./.github/workflows/wf_functional_e2e.yml + + workflow_interface_101_mnist: + name: Workflow Interface 101 MNIST Notebook + uses: ./.github/workflows/workflow_interface_101_mnist.yml + + workflow_interface_tests: + name: Workflow Interface Tests + uses: ./.github/workflows/experimental_workflow_tests.yml + + pipeline_status: + name: Pipeline Status + runs-on: ubuntu-22.04 + if: always() + needs: # Ensure to add any job added above to this list as well as the steps below + [ + bandit_code_scan, + check_code_format, + docker_bench_security, + double_workspace_export, + gandlf_taskrunner, + fr_301_watermark_nb_run, + hadolint_security_scan, + private_key_infrastructure, + pytest_coverage, + straggler_handling, + task_runner, + taskrunner_eden_compression, + trivy_scan, + tr_docker_gramine_direct, + tr_docker_native, + wf_functional_e2e, + workflow_interface_101_mnist, + workflow_interface_tests, + ] + steps: + - name: Fail job if any of the above jobs have failed + if: | + (needs.bandit_code_scan.result != 'success' && needs.bandit_code_scan.result != 'skipped') || + (needs.check_code_format.result != 'success' && needs.check_code_format.result != 'skipped') || + (needs.docker_bench_security.result != 'success' && needs.docker_bench_security.result != 'skipped') || + (needs.double_workspace_export.result != 'success' && needs.double_workspace_export.result != 'skipped') || + (needs.fr_301_watermark_nb_run.result != 'success' && needs.fr_301_watermark_nb_run.result != 'skipped') || + (needs.gandlf_taskrunner.result != 'success' && needs.gandlf_taskrunner.result != 'skipped') || + (needs.hadolint_security_scan.result != 'success' && needs.hadolint_security_scan.result != 'skipped') || + (needs.private_key_infrastructure.result != 'success' && needs.private_key_infrastructure.result != 'skipped') || + (needs.pytest_coverage.result != 'success' && needs.pytest_coverage.result != 'skipped') || + (needs.straggler_handling.result != 'success' && needs.straggler_handling.result != 'skipped') || + (needs.task_runner.result != 'success' && needs.task_runner.result != 'skipped') || + (needs.taskrunner_eden_compression.result != 'success' && needs.taskrunner_eden_compression.result != 'skipped') || + (needs.trivy_scan.result != 'success' && needs.trivy_scan.result != 'skipped') || + (needs.tr_docker_gramine_direct.result != 'success' && needs.tr_docker_gramine_direct.result != 'skipped') || + (needs.tr_docker_native.result != 'success' && needs.tr_docker_native.result != 'skipped') || + (needs.wf_functional_e2e.result != 'success' && needs.wf_functional_e2e.result != 'skipped') || + (needs.workflow_interface_101_mnist.result != 'success' && needs.workflow_interface_101_mnist.result != 'skipped') || + (needs.workflow_interface_tests.result != 'success' && needs.workflow_interface_tests.result != 'skipped') + run: | + echo "One or more jobs failed. Exiting pipeline." + exit 1 + + - name: Print pipeline status + run: | + echo "All jobs passed. Pipeline completed successfully." diff --git a/.github/workflows/pytest_coverage.yml b/.github/workflows/pytest_coverage.yml index a5ea0c14d9..12fa084b2f 100644 --- a/.github/workflows/pytest_coverage.yml +++ b/.github/workflows/pytest_coverage.yml @@ -4,8 +4,7 @@ name: Pytest and code coverage on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] + workflow_call: workflow_dispatch: permissions: diff --git a/.github/workflows/straggler-handling.yml b/.github/workflows/straggler-handling.yml index 64a2f07153..b5bb6996ce 100644 --- a/.github/workflows/straggler-handling.yml +++ b/.github/workflows/straggler-handling.yml @@ -4,8 +4,8 @@ name: Straggler Handling Test on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] + workflow_call: + workflow_dispatch: permissions: contents: read diff --git a/.github/workflows/taskrunner.yml b/.github/workflows/taskrunner.yml index 088ee60c64..11692a4b5c 100644 --- a/.github/workflows/taskrunner.yml +++ b/.github/workflows/taskrunner.yml @@ -4,8 +4,8 @@ name: TaskRunner on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] + workflow_call: + workflow_dispatch: permissions: contents: read diff --git a/.github/workflows/taskrunner_eden_pipeline.yml b/.github/workflows/taskrunner_eden_pipeline.yml index 3430f89463..02eea0e2dc 100644 --- a/.github/workflows/taskrunner_eden_pipeline.yml +++ b/.github/workflows/taskrunner_eden_pipeline.yml @@ -4,8 +4,8 @@ name: TaskRunner (Eden Compression) on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] + workflow_call: + workflow_dispatch: permissions: contents: read diff --git a/.github/workflows/tr_docker_gramine_direct.yml b/.github/workflows/tr_docker_gramine_direct.yml index 309351f385..169189a42d 100644 --- a/.github/workflows/tr_docker_gramine_direct.yml +++ b/.github/workflows/tr_docker_gramine_direct.yml @@ -2,8 +2,8 @@ name: TaskRunner (docker/gramine-direct) on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] + workflow_call: + workflow_dispatch: permissions: contents: read diff --git a/.github/workflows/tr_docker_native.yml b/.github/workflows/tr_docker_native.yml index f5af424a18..9fcb9b8759 100644 --- a/.github/workflows/tr_docker_native.yml +++ b/.github/workflows/tr_docker_native.yml @@ -2,8 +2,8 @@ name: TaskRunner (docker/native) on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] + workflow_call: + workflow_dispatch: permissions: contents: read diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index 0a77cf5fe7..5322b6faf3 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -4,8 +4,9 @@ on: branches: - develop - v1.7.x - pull_request: - types: [opened, synchronize, reopened, ready_for_review] + workflow_call: + workflow_dispatch: + jobs: build: if: github.event.pull_request.draft == false diff --git a/.github/workflows/wf_functional_e2e.yml b/.github/workflows/wf_functional_e2e.yml index 1831949299..090c98d28c 100644 --- a/.github/workflows/wf_functional_e2e.yml +++ b/.github/workflows/wf_functional_e2e.yml @@ -4,9 +4,8 @@ name: Workflow_Functional_E2E on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - + workflow_call: + workflow_dispatch: inputs: num_rounds: diff --git a/.github/workflows/workflow_interface_101_mnist.yml b/.github/workflows/workflow_interface_101_mnist.yml index e5146fc32e..78c82559be 100644 --- a/.github/workflows/workflow_interface_101_mnist.yml +++ b/.github/workflows/workflow_interface_101_mnist.yml @@ -5,9 +5,7 @@ name: Workflow Interface 101 MNIST Notebook on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - + workflow_call: workflow_dispatch: permissions: