Add tests for Fed EVal #2
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | ||
| # Task Runner E2E tests for dockerized approach | ||
| name: Task_Runner_Eval_DWS_E2E # Please do not modify the name as it is used in the composite action | ||
| on: | ||
| workflow_dispatch: | ||
| inputs: | ||
| num_rounds: | ||
| description: "Number of rounds to train" | ||
| required: false | ||
| default: "5" | ||
| type: string | ||
| num_collaborators: | ||
| description: "Number of collaborators" | ||
| required: false | ||
| default: "2" | ||
| type: string | ||
| job_to_run: | ||
| description: "Job to run" | ||
| required: true | ||
| default: "all" | ||
| type: choice | ||
| options: | ||
| - tls | ||
| - non_tls | ||
| - no_client_auth | ||
| - all | ||
| model_name: | ||
| description: "Model name" | ||
| required: true | ||
| default: "torch_cnn_mnist" | ||
| type: choice | ||
| options: | ||
| - torch_cnn_mnist | ||
| - keras_cnn_mnist | ||
| - both | ||
| python_version: | ||
| description: "Python version" | ||
| required: true | ||
| default: "3.10" | ||
| type: choice | ||
| options: | ||
| - "3.10" | ||
| - "3.11" | ||
| - "3.12" | ||
| permissions: | ||
| contents: read | ||
| # Environment variables common for all the jobs | ||
| env: | ||
| env: | ||
| NUM_ROUNDS: ${{ inputs.num_rounds || 5 }} | ||
| NUM_COLLABORATORS: ${{ inputs.num_collaborators || 2 }} | ||
| MODEL_NAME: ${{ inputs.model_name || 'both' }} | ||
| PYTHON_VERSION: ${{ inputs.python_version || '3.10' }} | ||
| JOBS_TO_RUN: ${{ inputs.jobs_to_run || 'all' }} | ||
| jobs: | ||
| test_with_tls_dws_eval: | ||
| if: ${{ env.job_to_run == 'tls' || env.job_to_run == 'all' }} | ||
| name: tr_tls_dws_eval | ||
| runs-on: ubuntu-22.04 | ||
| timeout-minutes: 15 | ||
| strategy: | ||
| matrix: | ||
| model_name: ${{ fromJSON(env.model_name == 'both' && '["torch_cnn_mnist","keras_cnn_mnist"]' || '["' + env.model_name + '"]') }} | ||
| python_version: [${{ env.python_version }}] | ||
| fail-fast: false # do not immediately fail if one of the combinations fail | ||
| env: | ||
| MODEL_NAME: ${{ matrix.model_name }} | ||
| PYTHON_VERSION: ${{ matrix.python_version }} | ||
| steps: | ||
| - name: Checkout OpenFL repository | ||
| id: checkout_openfl | ||
| uses: actions/[email protected] | ||
| with: | ||
| fetch-depth: 2 # needed for detecting changes | ||
| submodules: "true" | ||
| token: ${{ secrets.GITHUB_TOKEN }} | ||
| - name: Pre test run | ||
| uses: ./.github/actions/tr_pre_test_run | ||
| if: ${{ always() }} | ||
| - name: Run Task Runner E2E tests with TLS | ||
| id: run_tests | ||
| run: | | ||
| python -m pytest -s tests/end_to_end/test_suites/tr_with_eval_tests.py \ | ||
| -m task_runner_dockerized_ws --model_name ${{ env.MODEL_NAME }} \ | ||
| --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} | ||
| echo "Task runner end to end test run completed" | ||
| - name: Post test run | ||
| uses: ./.github/actions/tr_post_test_run | ||
| if: ${{ always() }} | ||
| with: | ||
| test_type: "tr_tls_dockerized_ws" | ||
| test_with_non_tls_dws_eval: | ||
| if: ${{ env.job_to_run == 'non_tls' || env.job_to_run == 'all' }} | ||
| name: tr_non_tls_dws_eval | ||
| runs-on: ubuntu-22.04 | ||
| timeout-minutes: 15 | ||
| strategy: | ||
| matrix: | ||
| model_name: ${{ fromJSON(env.model_name == 'both' && '["torch_cnn_mnist","keras_cnn_mnist"]' || '["' + env.model_name + '"]') }} | ||
| python_version: [${{ env.python_version }}] | ||
| fail-fast: false # do not immediately fail if one of the combinations fail | ||
| env: | ||
| MODEL_NAME: ${{ matrix.model_name }} | ||
| PYTHON_VERSION: ${{ matrix.python_version }} | ||
| steps: | ||
| - name: Checkout OpenFL repository | ||
| id: checkout_openfl | ||
| uses: actions/[email protected] | ||
| with: | ||
| fetch-depth: 2 # needed for detecting changes | ||
| submodules: "true" | ||
| token: ${{ secrets.GITHUB_TOKEN }} | ||
| - name: Pre test run | ||
| uses: ./.github/actions/tr_pre_test_run | ||
| if: ${{ always() }} | ||
| - name: Run Task Runner E2E tests without TLS | ||
| id: run_tests | ||
| run: | | ||
| python -m pytest -s tests/end_to_end/test_suites/tr_with_eval_tests.py \ | ||
| -m task_runner_dockerized_ws --model_name ${{ env.MODEL_NAME }} \ | ||
| --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_tls | ||
| echo "Task runner end to end test run completed" | ||
| - name: Post test run | ||
| uses: ./.github/actions/tr_post_test_run | ||
| if: ${{ always() }} | ||
| with: | ||
| test_type: "tr_non_tls_dockerized_ws" | ||
| test_with_no_client_auth_dws_eval: | ||
| if: ${{ env.job_to_run == 'no_client_auth' || env.job_to_run == 'all' }} | ||
| name: tr_no_client_auth_dws_eval | ||
| runs-on: ubuntu-22.04 | ||
| timeout-minutes: 15 | ||
| strategy: | ||
| matrix: | ||
| model_name: ${{ fromJSON(env.model_name == 'both' && '["torch_cnn_mnist","keras_cnn_mnist"]' || '["' + env.model_name + '"]') }} | ||
| python_version: [${{ env.python_version }}] | ||
| fail-fast: false # do not immediately fail if one of the combinations fail | ||
| env: | ||
| MODEL_NAME: ${{ matrix.model_name }} | ||
| PYTHON_VERSION: ${{ matrix.python_version }} | ||
| steps: | ||
| - name: Checkout OpenFL repository | ||
| id: checkout_openfl | ||
| uses: actions/[email protected] | ||
| with: | ||
| fetch-depth: 2 # needed for detecting changes | ||
| submodules: "true" | ||
| token: ${{ secrets.GITHUB_TOKEN }} | ||
| - name: Pre test run | ||
| uses: ./.github/actions/tr_pre_test_run | ||
| if: ${{ always() }} | ||
| - name: Run Task Runner E2E tests without TLS | ||
| id: run_tests | ||
| run: | | ||
| python -m pytest -s tests/end_to_end/test_suites/tr_with_eval_tests.py \ | ||
| -m task_runner_dockerized_ws --model_name ${{ env.MODEL_NAME }} \ | ||
| --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_client_auth | ||
| echo "Task runner end to end test run completed" | ||
| - name: Post test run | ||
| uses: ./.github/actions/tr_post_test_run | ||
| if: ${{ always() }} | ||
| with: | ||
| test_type: "tr_no_client_auth_dockerized_ws" | ||
