From 048c309d5448dcef6e99858449818b08bc809d43 Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 10:13:00 +0000 Subject: [PATCH 01/39] Testing GITHUB_STEP_SUMMARY Signed-off-by: noopur --- tests/end_to_end/utils/summary_helper.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/end_to_end/utils/summary_helper.py b/tests/end_to_end/utils/summary_helper.py index 0ed8aa0a3a..aceb13184c 100644 --- a/tests/end_to_end/utils/summary_helper.py +++ b/tests/end_to_end/utils/summary_helper.py @@ -104,7 +104,7 @@ def get_testcase_result(): def print_task_runner_score(): """ - Main function to get the test case results and aggregator logs + Function to get the test case results and aggregator logs And write the results to GitHub step summary IMP: Do not fail the test in any scenario """ @@ -131,6 +131,8 @@ def print_task_runner_score(): model_name = os.getenv("MODEL_NAME") summary_file = os.getenv("GITHUB_STEP_SUMMARY") + print(f"Summary file: {summary_file}") + # Validate the model name and create the workspace name if not model_name.upper() in constants.ModelName._member_names_: print( @@ -169,8 +171,13 @@ def print_task_runner_score(): def print_federated_runtime_score(): + """ + Function to get the federated runtime score from the director log file + And write the results to GitHub step summary + IMP: Do not fail the test in any scenario + """ summary_file = os.getenv("GITHUB_STEP_SUMMARY") - + print(f"Summary file: {summary_file}") search_string = "Aggregated model validation score" last_occurrence = aggregated_model_score = None From 439c979cb7b3089ec632da12901e5af938cca046 Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 10:55:01 +0000 Subject: [PATCH 02/39] Modified Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 16 +++++++++++ tests/end_to_end/utils/summary_helper.py | 32 +++++++++++++++------ 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index 3b336e22af..3088252090 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -18,6 +18,17 @@ on: required: false default: "2" type: string + jobs_to_run: + description: "Jobs to run" + type: choice + default: "all" + options: + - all + - test_with_tls + - test_with_non_tls + - test_with_no_client_auth + - test_memory_logs + required: false permissions: contents: read @@ -26,10 +37,12 @@ permissions: env: NUM_ROUNDS: ${{ inputs.num_rounds || '5' }} NUM_COLLABORATORS: ${{ inputs.num_collaborators || '2' }} + JOBS_TO_RUN: ${{ inputs.jobs_to_run || 'all' }} jobs: test_with_tls: name: tr_tls + if: ${{ contains(env.JOBS_TO_RUN, 'all') || contains(env.JOBS_TO_RUN, 'test_with_tls') }} runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: @@ -73,6 +86,7 @@ jobs: test_with_non_tls: name: tr_non_tls + if: ${{ contains(env.JOBS_TO_RUN, 'all') || contains(env.JOBS_TO_RUN, 'test_with_non_tls') }} runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: @@ -116,6 +130,7 @@ jobs: test_with_no_client_auth: name: tr_no_client_auth + if: ${{ contains(env.JOBS_TO_RUN, 'all') || contains(env.JOBS_TO_RUN, 'test_with_no_client_auth') }} runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: @@ -159,6 +174,7 @@ jobs: test_memory_logs: name: tr_tls_memory_logs + if: ${{ contains(env.JOBS_TO_RUN, 'all') || contains(env.JOBS_TO_RUN, 'test_memory_logs') }} runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: diff --git a/tests/end_to_end/utils/summary_helper.py b/tests/end_to_end/utils/summary_helper.py index aceb13184c..31788662ef 100644 --- a/tests/end_to_end/utils/summary_helper.py +++ b/tests/end_to_end/utils/summary_helper.py @@ -42,9 +42,14 @@ def get_aggregated_accuracy(agg_log_file): return agg_accuracy agg_accuracy_dict = convert_to_json(agg_log_file) - agg_accuracy = agg_accuracy_dict[-1].get( - "aggregator/aggregated_model_validation/accuracy", "Not Found" - ) + print(f"agg_accuracy_dict is: {agg_accuracy_dict}") + + if not agg_accuracy_dict: + print(f"Aggregator log file {agg_log_file} is empty. Cannot get aggregated accuracy, returning 'Not Found'") + else: + agg_accuracy = agg_accuracy_dict[-1].get( + "aggregator/aggregated_model_validation/accuracy", "Not Found" + ) return agg_accuracy @@ -129,9 +134,7 @@ def print_task_runner_score(): num_cols = os.getenv("NUM_COLLABORATORS") num_rounds = os.getenv("NUM_ROUNDS") model_name = os.getenv("MODEL_NAME") - summary_file = os.getenv("GITHUB_STEP_SUMMARY") - - print(f"Summary file: {summary_file}") + summary_file = _get_summary_file() # Validate the model name and create the workspace name if not model_name.upper() in constants.ModelName._member_names_: @@ -176,8 +179,7 @@ def print_federated_runtime_score(): And write the results to GitHub step summary IMP: Do not fail the test in any scenario """ - summary_file = os.getenv("GITHUB_STEP_SUMMARY") - print(f"Summary file: {summary_file}") + summary_file = _get_summary_file() search_string = "Aggregated model validation score" last_occurrence = aggregated_model_score = None @@ -217,6 +219,20 @@ def print_federated_runtime_score(): print(f"| {aggregated_model_score} |", file=fh) +def _get_summary_file(): + """ + Function to get the summary file path + Returns: + summary_file: Path to the summary file + """ + summary_file = Path(os.getenv("GITHUB_STEP_SUMMARY")) + print(f"Summary file: {summary_file}") + if "step_summary" not in summary_file.name or ".env" not in summary_file.name: + print("Invalid summary file. Exiting...") + exit(1) + return summary_file + + def fetch_args(): """ Function to fetch the commandline arguments. From 3e0573232d28c0243fae0b135f827a3bf2c9d064 Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 10:58:29 +0000 Subject: [PATCH 03/39] Modified Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index 3088252090..f41b85f341 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -42,7 +42,7 @@ env: jobs: test_with_tls: name: tr_tls - if: ${{ contains(env.JOBS_TO_RUN, 'all') || contains(env.JOBS_TO_RUN, 'test_with_tls') }} + if: ${{ env.JOBS_TO_RUN == 'all' }} || ${{ env.JOBS_TO_RUN == 'test_with_tls' }} runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: @@ -86,7 +86,7 @@ jobs: test_with_non_tls: name: tr_non_tls - if: ${{ contains(env.JOBS_TO_RUN, 'all') || contains(env.JOBS_TO_RUN, 'test_with_non_tls') }} + if: ${{ env.JOBS_TO_RUN == 'all' }} || ${{ env.JOBS_TO_RUN == 'test_with_non_tls' }} runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: @@ -130,7 +130,7 @@ jobs: test_with_no_client_auth: name: tr_no_client_auth - if: ${{ contains(env.JOBS_TO_RUN, 'all') || contains(env.JOBS_TO_RUN, 'test_with_no_client_auth') }} + if: ${{ env.JOBS_TO_RUN == 'all' }} || ${{ env.JOBS_TO_RUN == 'test_with_no_client_auth' }} runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: @@ -174,7 +174,7 @@ jobs: test_memory_logs: name: tr_tls_memory_logs - if: ${{ contains(env.JOBS_TO_RUN, 'all') || contains(env.JOBS_TO_RUN, 'test_memory_logs') }} + if: ${{ env.JOBS_TO_RUN == 'all' }} || ${{ env.JOBS_TO_RUN == 'test_memory_logs' }} runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: From d538750805d28d545f9d89f9a7a59bec6680ece9 Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 16:07:28 +0000 Subject: [PATCH 04/39] Initial job added Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 27 ++++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index f41b85f341..1eeda1a96f 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -40,9 +40,25 @@ env: JOBS_TO_RUN: ${{ inputs.jobs_to_run || 'all' }} jobs: + initial_job: + if: | + (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') || + (github.event_name == 'workflow_dispatch') + name: Initial Job + runs-on: ubuntu-22.04 + outputs: + jobs_to_run: ${{ steps.jobs_to_run.outputs.jobs_to_run }} + steps: + - name: Jobs to run + id: jobs_to_run + run: | + echo Jobs to run: ${{ env.JOBS_TO_RUN }} + echo "jobs_to_run=${{ env.JOBS_TO_RUN }}" >> "$GITHUB_OUTPUT" + test_with_tls: name: tr_tls - if: ${{ env.JOBS_TO_RUN == 'all' }} || ${{ env.JOBS_TO_RUN == 'test_with_tls' }} + needs: initial_job + if: ${{ needs.initial_job.outputs.jobs_to_run == 'all' }} || ${{ needs.initial_job.outputs.jobs_to_run == 'test_with_tls' }} runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: @@ -86,7 +102,8 @@ jobs: test_with_non_tls: name: tr_non_tls - if: ${{ env.JOBS_TO_RUN == 'all' }} || ${{ env.JOBS_TO_RUN == 'test_with_non_tls' }} + needs: initial_job + if: ${{ needs.initial_job.outputs.jobs_to_run == 'all' }} || ${{ needs.initial_job.outputs.jobs_to_run == 'test_with_non_tls' }} runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: @@ -130,7 +147,8 @@ jobs: test_with_no_client_auth: name: tr_no_client_auth - if: ${{ env.JOBS_TO_RUN == 'all' }} || ${{ env.JOBS_TO_RUN == 'test_with_no_client_auth' }} + needs: initial_job + if: ${{ needs.initial_job.outputs.jobs_to_run == 'all' }} || ${{ needs.initial_job.outputs.jobs_to_run == 'test_with_no_client_auth' }} runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: @@ -174,7 +192,8 @@ jobs: test_memory_logs: name: tr_tls_memory_logs - if: ${{ env.JOBS_TO_RUN == 'all' }} || ${{ env.JOBS_TO_RUN == 'test_memory_logs' }} + needs: initial_job + if: ${{ needs.initial_job.outputs.jobs_to_run == 'all' }} || ${{ needs.initial_job.outputs.jobs_to_run == 'test_memory_logs' }} runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: From da804db2f0fa975a82db67bb6c42ac800f39477f Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 16:12:14 +0000 Subject: [PATCH 05/39] Initial job added Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index 1eeda1a96f..831a1ced0e 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -89,6 +89,7 @@ jobs: - name: Run Task Runner E2E tests with TLS id: run_tests run: | + echo ${{ needs.initial_job.outputs.jobs_to_run }} python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \ --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} @@ -134,6 +135,7 @@ jobs: - name: Run Task Runner E2E tests without TLS id: run_tests run: | + echo ${{ needs.initial_job.outputs.jobs_to_run }} python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \ --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_tls @@ -179,6 +181,7 @@ jobs: - name: Run Task Runner E2E tests without TLS id: run_tests run: | + echo ${{ needs.initial_job.outputs.jobs_to_run }} python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \ --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_client_auth @@ -224,6 +227,7 @@ jobs: - name: Run Task Runner E2E tests with TLS and memory logs id: run_tests run: | + echo ${{ needs.initial_job.outputs.jobs_to_run }} python -m pytest -s tests/end_to_end/test_suites/memory_logs_tests.py \ -k test_log_memory_usage_basic --model_name ${{ env.MODEL_NAME }} \ --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} \ From ffa5e02d5778b373de087b86cef6fce50968f0b2 Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 16:17:02 +0000 Subject: [PATCH 06/39] Initial job added Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index 831a1ced0e..ce811914af 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -58,7 +58,7 @@ jobs: test_with_tls: name: tr_tls needs: initial_job - if: ${{ needs.initial_job.outputs.jobs_to_run == 'all' }} || ${{ needs.initial_job.outputs.jobs_to_run == 'test_with_tls' }} + if: needs.initial_job.outputs.jobs_to_run == 'all' || needs.initial_job.outputs.jobs_to_run == 'test_with_tls' runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: @@ -104,7 +104,7 @@ jobs: test_with_non_tls: name: tr_non_tls needs: initial_job - if: ${{ needs.initial_job.outputs.jobs_to_run == 'all' }} || ${{ needs.initial_job.outputs.jobs_to_run == 'test_with_non_tls' }} + if: needs.initial_job.outputs.jobs_to_run == 'all' || needs.initial_job.outputs.jobs_to_run == 'test_with_non_tls' runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: @@ -150,7 +150,7 @@ jobs: test_with_no_client_auth: name: tr_no_client_auth needs: initial_job - if: ${{ needs.initial_job.outputs.jobs_to_run == 'all' }} || ${{ needs.initial_job.outputs.jobs_to_run == 'test_with_no_client_auth' }} + if: needs.initial_job.outputs.jobs_to_run == 'all' || needs.initial_job.outputs.jobs_to_run == 'test_with_no_client_auth' runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: @@ -196,7 +196,7 @@ jobs: test_memory_logs: name: tr_tls_memory_logs needs: initial_job - if: ${{ needs.initial_job.outputs.jobs_to_run == 'all' }} || ${{ needs.initial_job.outputs.jobs_to_run == 'test_memory_logs' }} + if: needs.initial_job.outputs.jobs_to_run == 'all' || needs.initial_job.outputs.jobs_to_run == 'test_memory_logs' runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: From 11bfa61471a9491ca0d261a545db9c25a9dc4c2e Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 16:17:57 +0000 Subject: [PATCH 07/39] Check GITHUB_STEP_SUMMARY values Signed-off-by: noopur --- tests/end_to_end/utils/summary_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/end_to_end/utils/summary_helper.py b/tests/end_to_end/utils/summary_helper.py index 31788662ef..fdd84347c7 100644 --- a/tests/end_to_end/utils/summary_helper.py +++ b/tests/end_to_end/utils/summary_helper.py @@ -225,7 +225,7 @@ def _get_summary_file(): Returns: summary_file: Path to the summary file """ - summary_file = Path(os.getenv("GITHUB_STEP_SUMMARY")) + summary_file = os.getenv("GITHUB_STEP_SUMMARY") print(f"Summary file: {summary_file}") if "step_summary" not in summary_file.name or ".env" not in summary_file.name: print("Invalid summary file. Exiting...") From 61a38893c21ea1b9fc570d54c22cd39e37a413d8 Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 16:34:17 +0000 Subject: [PATCH 08/39] More changes Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 82 ++++++++++++++------- 1 file changed, 57 insertions(+), 25 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index ce811914af..be25918e3f 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -18,6 +18,25 @@ on: required: false default: "2" type: string + model_name: + description: "Model name" + required: false + default: "all" + type: choice + options: + - all + - torch_cnn_mnist + - keras_cnn_mnist + python_version: + description: "Python version" + required: false + default: "all" + type: choice + options: + - all + - "3.10" + - "3.11" + - "3.12" jobs_to_run: description: "Jobs to run" type: choice @@ -37,36 +56,53 @@ permissions: env: NUM_ROUNDS: ${{ inputs.num_rounds || '5' }} NUM_COLLABORATORS: ${{ inputs.num_collaborators || '2' }} + MODEL_NAME: ${{ inputs.model_name || 'all' }} + PYTHON_VERSION: ${{ inputs.python_version || 'all' }} JOBS_TO_RUN: ${{ inputs.jobs_to_run || 'all' }} jobs: - initial_job: + input_value_selection: if: | (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') || (github.event_name == 'workflow_dispatch') name: Initial Job runs-on: ubuntu-22.04 outputs: - jobs_to_run: ${{ steps.jobs_to_run.outputs.jobs_to_run }} + selected_jobs: ${{ steps.input_value_selection.outputs.jobs_to_run }} + selected_models: ${{ steps.input_value_selection.outputs.models_to_run }} + selected_python_versions: ${{ steps.input_value_selection.outputs.python_versions_to_run }} steps: - - name: Jobs to run - id: jobs_to_run + - name: Job to select input values + id: input_value_selection run: | - echo Jobs to run: ${{ env.JOBS_TO_RUN }} + if [ "${{ env.MODEL_NAME }}" == "all" ]; then + echo "models_to_run=[\"torch_cnn_mnist\", \"keras_cnn_mnist\"]" >> "$GITHUB_OUTPUT" + else + echo "models_to_run=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" + fi + if [ "${{ env.PYTHON_VERSION }}" == "all" ]; then + echo "python_versions_to_run=[\"3.10\", \"3.11\", \"3.12\"]" >> "$GITHUB_OUTPUT" + else + echo "python_versions_to_run=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" + fi echo "jobs_to_run=${{ env.JOBS_TO_RUN }}" >> "$GITHUB_OUTPUT" + echo "Selected model(s): $models_to_run" >> $GITHUB_STEP_SUMMARY + echo "Selected python version(s): $python_versions_to_run" >> $GITHUB_STEP_SUMMARY + echo "Selected job(s): $jobs_to_run" >> $GITHUB_STEP_SUMMARY + test_with_tls: name: tr_tls - needs: initial_job - if: needs.initial_job.outputs.jobs_to_run == 'all' || needs.initial_job.outputs.jobs_to_run == 'test_with_tls' + needs: input_value_selection + if: needs.input_value_selection.outputs.selected_jobs == 'all' || needs.input_value_selection.outputs.selected_jobs == 'test_with_tls' runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: matrix: # Models like XGBoost (xgb_higgs) and torch_cnn_histology require runners with higher memory and CPU to run. # Thus these models are excluded from the matrix for now. - model_name: ["torch_cnn_mnist", "keras_cnn_mnist"] - python_version: ["3.10", "3.11", "3.12"] + model_name: ${{ needs.input_value_selection.outputs.selected_models }} + python_version: ${{ needs.input_value_selection.outputs.selected_python_versions }} fail-fast: false # do not immediately fail if one of the combinations fail env: @@ -89,7 +125,6 @@ jobs: - name: Run Task Runner E2E tests with TLS id: run_tests run: | - echo ${{ needs.initial_job.outputs.jobs_to_run }} python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \ --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} @@ -103,16 +138,16 @@ jobs: test_with_non_tls: name: tr_non_tls - needs: initial_job - if: needs.initial_job.outputs.jobs_to_run == 'all' || needs.initial_job.outputs.jobs_to_run == 'test_with_non_tls' + needs: input_value_selection + if: needs.input_value_selection.outputs.selected_jobs == 'all' || needs.input_value_selection.outputs.selected_jobs == 'test_with_non_tls' runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: matrix: # Testing this scenario only for torch_cnn_mnist model and python 3.10 # If required, this can be extended to other models and python versions - model_name: ["torch_cnn_mnist"] - python_version: ["3.10"] + model_name: ${{ needs.input_value_selection.outputs.selected_models == "all" && ["torch_cnn_mnist"] || needs.input_value_selection.outputs.selected_models }} + python_version: ${{ needs.input_value_selection.outputs.selected_python_versions == "all" && ["3.10"] || needs.input_value_selection.outputs.selected_python_versions }} fail-fast: false # do not immediately fail if one of the combinations fail env: @@ -135,7 +170,6 @@ jobs: - name: Run Task Runner E2E tests without TLS id: run_tests run: | - echo ${{ needs.initial_job.outputs.jobs_to_run }} python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \ --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_tls @@ -149,16 +183,16 @@ jobs: test_with_no_client_auth: name: tr_no_client_auth - needs: initial_job - if: needs.initial_job.outputs.jobs_to_run == 'all' || needs.initial_job.outputs.jobs_to_run == 'test_with_no_client_auth' + needs: input_value_selection + if: needs.input_value_selection.outputs.selected_jobs == 'all' || needs.input_value_selection.outputs.selected_jobs == 'test_with_no_client_auth' runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: matrix: # Testing this scenario for keras_cnn_mnist model and python 3.10 # If required, this can be extended to other models and python versions - model_name: ["keras_cnn_mnist"] - python_version: ["3.10"] + model_name: ${{ needs.input_value_selection.outputs.selected_models == "all" && ["keras_cnn_mnist"] || needs.input_value_selection.outputs.selected_models }} + python_version: ${{ needs.input_value_selection.outputs.selected_python_versions == "all" && ["3.10"] || needs.input_value_selection.outputs.selected_python_versions }} fail-fast: false # do not immediately fail if one of the combinations fail env: @@ -181,7 +215,6 @@ jobs: - name: Run Task Runner E2E tests without TLS id: run_tests run: | - echo ${{ needs.initial_job.outputs.jobs_to_run }} python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \ --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_client_auth @@ -195,16 +228,16 @@ jobs: test_memory_logs: name: tr_tls_memory_logs - needs: initial_job - if: needs.initial_job.outputs.jobs_to_run == 'all' || needs.initial_job.outputs.jobs_to_run == 'test_memory_logs' + needs: input_value_selection + if: needs.input_value_selection.outputs.selected_jobs == 'all' || needs.input_value_selection.outputs.selected_jobs == 'test_memory_logs' runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: matrix: # Testing this scenario only for torch_cnn_mnist model and python 3.10 # If required, this can be extended to other models and python versions - model_name: ["torch_cnn_mnist"] - python_version: ["3.10"] + model_name: ${{ needs.input_value_selection.outputs.selected_models == "all" && ["torch_cnn_mnist"] || needs.input_value_selection.outputs.selected_models }} + python_version: ${{ needs.input_value_selection.outputs.selected_python_versions == "all" && ["3.10"] || needs.input_value_selection.outputs.selected_python_versions }} fail-fast: false # do not immediately fail if one of the combinations fail env: @@ -227,7 +260,6 @@ jobs: - name: Run Task Runner E2E tests with TLS and memory logs id: run_tests run: | - echo ${{ needs.initial_job.outputs.jobs_to_run }} python -m pytest -s tests/end_to_end/test_suites/memory_logs_tests.py \ -k test_log_memory_usage_basic --model_name ${{ env.MODEL_NAME }} \ --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} \ From b2044894ba40507d2f3271f527630d4bc1abdee6 Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 16:36:44 +0000 Subject: [PATCH 09/39] More changes Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 24 ++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index be25918e3f..f7c115cf35 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -21,7 +21,7 @@ on: model_name: description: "Model name" required: false - default: "all" + default: 'all' type: choice options: - all @@ -30,17 +30,17 @@ on: python_version: description: "Python version" required: false - default: "all" + default: 'all' type: choice options: - all - - "3.10" + - '3.10' - "3.11" - "3.12" jobs_to_run: description: "Jobs to run" type: choice - default: "all" + default: 'all' options: - all - test_with_tls @@ -75,12 +75,12 @@ jobs: - name: Job to select input values id: input_value_selection run: | - if [ "${{ env.MODEL_NAME }}" == "all" ]; then + if [ "${{ env.MODEL_NAME }}" == 'all' ]; then echo "models_to_run=[\"torch_cnn_mnist\", \"keras_cnn_mnist\"]" >> "$GITHUB_OUTPUT" else echo "models_to_run=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" fi - if [ "${{ env.PYTHON_VERSION }}" == "all" ]; then + if [ "${{ env.PYTHON_VERSION }}" == 'all' ]; then echo "python_versions_to_run=[\"3.10\", \"3.11\", \"3.12\"]" >> "$GITHUB_OUTPUT" else echo "python_versions_to_run=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" @@ -146,8 +146,8 @@ jobs: matrix: # Testing this scenario only for torch_cnn_mnist model and python 3.10 # If required, this can be extended to other models and python versions - model_name: ${{ needs.input_value_selection.outputs.selected_models == "all" && ["torch_cnn_mnist"] || needs.input_value_selection.outputs.selected_models }} - python_version: ${{ needs.input_value_selection.outputs.selected_python_versions == "all" && ["3.10"] || needs.input_value_selection.outputs.selected_python_versions }} + model_name: ${{ needs.input_value_selection.outputs.selected_models == 'all' && ['torch_cnn_mnist'] || needs.input_value_selection.outputs.selected_models }} + python_version: ${{ needs.input_value_selection.outputs.selected_python_versions == 'all' && ['3.10'] || needs.input_value_selection.outputs.selected_python_versions }} fail-fast: false # do not immediately fail if one of the combinations fail env: @@ -191,8 +191,8 @@ jobs: matrix: # Testing this scenario for keras_cnn_mnist model and python 3.10 # If required, this can be extended to other models and python versions - model_name: ${{ needs.input_value_selection.outputs.selected_models == "all" && ["keras_cnn_mnist"] || needs.input_value_selection.outputs.selected_models }} - python_version: ${{ needs.input_value_selection.outputs.selected_python_versions == "all" && ["3.10"] || needs.input_value_selection.outputs.selected_python_versions }} + model_name: ${{ needs.input_value_selection.outputs.selected_models == 'all' && ['keras_cnn_mnist'] || needs.input_value_selection.outputs.selected_models }} + python_version: ${{ needs.input_value_selection.outputs.selected_python_versions == 'all' && ['3.10'] || needs.input_value_selection.outputs.selected_python_versions }} fail-fast: false # do not immediately fail if one of the combinations fail env: @@ -236,8 +236,8 @@ jobs: matrix: # Testing this scenario only for torch_cnn_mnist model and python 3.10 # If required, this can be extended to other models and python versions - model_name: ${{ needs.input_value_selection.outputs.selected_models == "all" && ["torch_cnn_mnist"] || needs.input_value_selection.outputs.selected_models }} - python_version: ${{ needs.input_value_selection.outputs.selected_python_versions == "all" && ["3.10"] || needs.input_value_selection.outputs.selected_python_versions }} + model_name: ${{ needs.input_value_selection.outputs.selected_models == 'all' && ['torch_cnn_mnist'] || needs.input_value_selection.outputs.selected_models }} + python_version: ${{ needs.input_value_selection.outputs.selected_python_versions == 'all' && ['3.10'] || needs.input_value_selection.outputs.selected_python_versions }} fail-fast: false # do not immediately fail if one of the combinations fail env: From ee2fcc447badcd6fa7c4e36930fcf6b26074fcc7 Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 16:38:20 +0000 Subject: [PATCH 10/39] More changes Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index f7c115cf35..1e69dff436 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -146,8 +146,8 @@ jobs: matrix: # Testing this scenario only for torch_cnn_mnist model and python 3.10 # If required, this can be extended to other models and python versions - model_name: ${{ needs.input_value_selection.outputs.selected_models == 'all' && ['torch_cnn_mnist'] || needs.input_value_selection.outputs.selected_models }} - python_version: ${{ needs.input_value_selection.outputs.selected_python_versions == 'all' && ['3.10'] || needs.input_value_selection.outputs.selected_python_versions }} + model_name: ${{ needs.input_value_selection.outputs.selected_models == 'all' && [\'torch_cnn_mnist\'] || needs.input_value_selection.outputs.selected_models }} + python_version: ${{ needs.input_value_selection.outputs.selected_python_versions == 'all' && [\'3.10\'] || needs.input_value_selection.outputs.selected_python_versions }} fail-fast: false # do not immediately fail if one of the combinations fail env: @@ -191,8 +191,8 @@ jobs: matrix: # Testing this scenario for keras_cnn_mnist model and python 3.10 # If required, this can be extended to other models and python versions - model_name: ${{ needs.input_value_selection.outputs.selected_models == 'all' && ['keras_cnn_mnist'] || needs.input_value_selection.outputs.selected_models }} - python_version: ${{ needs.input_value_selection.outputs.selected_python_versions == 'all' && ['3.10'] || needs.input_value_selection.outputs.selected_python_versions }} + model_name: ${{ needs.input_value_selection.outputs.selected_models == 'all' && [\'keras_cnn_mnist\'] || needs.input_value_selection.outputs.selected_models }} + python_version: ${{ needs.input_value_selection.outputs.selected_python_versions == 'all' && [\'3.10\'] || needs.input_value_selection.outputs.selected_python_versions }} fail-fast: false # do not immediately fail if one of the combinations fail env: @@ -236,8 +236,8 @@ jobs: matrix: # Testing this scenario only for torch_cnn_mnist model and python 3.10 # If required, this can be extended to other models and python versions - model_name: ${{ needs.input_value_selection.outputs.selected_models == 'all' && ['torch_cnn_mnist'] || needs.input_value_selection.outputs.selected_models }} - python_version: ${{ needs.input_value_selection.outputs.selected_python_versions == 'all' && ['3.10'] || needs.input_value_selection.outputs.selected_python_versions }} + model_name: ${{ needs.input_value_selection.outputs.selected_models == 'all' && [\'torch_cnn_mnist\'] || needs.input_value_selection.outputs.selected_models }} + python_version: ${{ needs.input_value_selection.outputs.selected_python_versions == 'all' && [\'3.10\'] || needs.input_value_selection.outputs.selected_python_versions }} fail-fast: false # do not immediately fail if one of the combinations fail env: From b84ebd8e7a06d67ef0898ac2a022ddef80ac6716 Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 16:54:44 +0000 Subject: [PATCH 11/39] More changes Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 50 ++++++++++++++------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index 1e69dff436..524aefbf7b 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -69,27 +69,43 @@ jobs: runs-on: ubuntu-22.04 outputs: selected_jobs: ${{ steps.input_value_selection.outputs.jobs_to_run }} - selected_models: ${{ steps.input_value_selection.outputs.models_to_run }} - selected_python_versions: ${{ steps.input_value_selection.outputs.python_versions_to_run }} + selected_models_for_tls: ${{ steps.input_value_selection.outputs.models_for_tls }} + selected_python_for_tls: ${{ steps.input_value_selection.outputs.python_for_tls }} + selected_models_for_non_tls: ${{ steps.input_value_selection.outputs.models_for_non_tls }} + selected_models_for_no_client_auth: ${{ steps.input_value_selection.outputs.models_for_no_client_auth }} + selected_models_for_memory_logs: ${{ steps.input_value_selection.outputs.models_for_memory_logs }} + selected_python_for_non_tls: ${{ steps.input_value_selection.outputs.python_for_non_tls }} + selected_python_for_no_client_auth: ${{ steps.input_value_selection.outputs.python_for_no_client_auth }} + selected_python_for_memory_logs: ${{ steps.input_value_selection.outputs.python_for_memory_logs }} steps: - name: Job to select input values id: input_value_selection run: | if [ "${{ env.MODEL_NAME }}" == 'all' ]; then - echo "models_to_run=[\"torch_cnn_mnist\", \"keras_cnn_mnist\"]" >> "$GITHUB_OUTPUT" + echo "models_for_tls=[\"torch_cnn_mnist\", \"keras_cnn_mnist\"]" >> "$GITHUB_OUTPUT" + echo "models_for_non_tls=[\"torch_cnn_mnist\"]" >> "$GITHUB_OUTPUT" + echo "models_for_no_client_auth=[\"keras_cnn_mnist\"]" >> "$GITHUB_OUTPUT" + echo "models_for_memory_logs=[\"torch_cnn_mnist\"]" >> "$GITHUB_OUTPUT" else - echo "models_to_run=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" + echo "models_for_tls=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" + echo "models_for_non_tls=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" + echo "models_for_no_client_auth=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" + echo "models_for_memory_logs=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" fi if [ "${{ env.PYTHON_VERSION }}" == 'all' ]; then - echo "python_versions_to_run=[\"3.10\", \"3.11\", \"3.12\"]" >> "$GITHUB_OUTPUT" + echo "python_for_tls=[\"3.10\", \"3.11\", \"3.12\"]" >> "$GITHUB_OUTPUT" + echo "python_for_non_tls=[\"3.10\"]" >> "$GITHUB_OUTPUT" + echo "python_for_no_client_auth=[\"3.10\"]" >> "$GITHUB_OUTPUT" + echo "python_for_memory_logs=[\"3.10\"]" >> "$GITHUB_OUTPUT" else - echo "python_versions_to_run=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" + echo "python_for_tls=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" + echo "python_for_non_tls=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" + echo "python_for_no_client_auth=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" + echo "python_for_memory_logs=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" fi echo "jobs_to_run=${{ env.JOBS_TO_RUN }}" >> "$GITHUB_OUTPUT" - echo "Selected model(s): $models_to_run" >> $GITHUB_STEP_SUMMARY - echo "Selected python version(s): $python_versions_to_run" >> $GITHUB_STEP_SUMMARY - echo "Selected job(s): $jobs_to_run" >> $GITHUB_STEP_SUMMARY + echo $GITHUB_OUTPUT test_with_tls: name: tr_tls @@ -101,8 +117,8 @@ jobs: matrix: # Models like XGBoost (xgb_higgs) and torch_cnn_histology require runners with higher memory and CPU to run. # Thus these models are excluded from the matrix for now. - model_name: ${{ needs.input_value_selection.outputs.selected_models }} - python_version: ${{ needs.input_value_selection.outputs.selected_python_versions }} + model_name: ${{ needs.input_value_selection.outputs.selected_models_for_tls }} + python_version: ${{ needs.input_value_selection.outputs.selected_python_for_tls }} fail-fast: false # do not immediately fail if one of the combinations fail env: @@ -146,8 +162,8 @@ jobs: matrix: # Testing this scenario only for torch_cnn_mnist model and python 3.10 # If required, this can be extended to other models and python versions - model_name: ${{ needs.input_value_selection.outputs.selected_models == 'all' && [\'torch_cnn_mnist\'] || needs.input_value_selection.outputs.selected_models }} - python_version: ${{ needs.input_value_selection.outputs.selected_python_versions == 'all' && [\'3.10\'] || needs.input_value_selection.outputs.selected_python_versions }} + model_name: ${{ needs.input_value_selection.outputs.selected_models_for_non_tls }} + python_version: ${{ needs.input_value_selection.outputs.selected_python_for_non_tls }} fail-fast: false # do not immediately fail if one of the combinations fail env: @@ -191,8 +207,8 @@ jobs: matrix: # Testing this scenario for keras_cnn_mnist model and python 3.10 # If required, this can be extended to other models and python versions - model_name: ${{ needs.input_value_selection.outputs.selected_models == 'all' && [\'keras_cnn_mnist\'] || needs.input_value_selection.outputs.selected_models }} - python_version: ${{ needs.input_value_selection.outputs.selected_python_versions == 'all' && [\'3.10\'] || needs.input_value_selection.outputs.selected_python_versions }} + model_name: ${{ needs.input_value_selection.outputs.selected_models_for_no_client_auth }} + python_version: ${{ needs.input_value_selection.outputs.selected_python_for_no_client_auth }} fail-fast: false # do not immediately fail if one of the combinations fail env: @@ -236,8 +252,8 @@ jobs: matrix: # Testing this scenario only for torch_cnn_mnist model and python 3.10 # If required, this can be extended to other models and python versions - model_name: ${{ needs.input_value_selection.outputs.selected_models == 'all' && [\'torch_cnn_mnist\'] || needs.input_value_selection.outputs.selected_models }} - python_version: ${{ needs.input_value_selection.outputs.selected_python_versions == 'all' && [\'3.10\'] || needs.input_value_selection.outputs.selected_python_versions }} + model_name: ${{ needs.input_value_selection.outputs.selected_models_for_memory_logs }} + python_version: ${{ needs.input_value_selection.outputs.selected_python_for_memory_logs }} fail-fast: false # do not immediately fail if one of the combinations fail env: From 1bc9799697e6765856a2f7cccae49f3462c2d85d Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 17:05:12 +0000 Subject: [PATCH 12/39] More changes Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index 524aefbf7b..1b9d6356dc 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -117,8 +117,8 @@ jobs: matrix: # Models like XGBoost (xgb_higgs) and torch_cnn_histology require runners with higher memory and CPU to run. # Thus these models are excluded from the matrix for now. - model_name: ${{ needs.input_value_selection.outputs.selected_models_for_tls }} - python_version: ${{ needs.input_value_selection.outputs.selected_python_for_tls }} + model_name: ${{ fromJson(needs.input_value_selection.outputs.selected_models_for_tls) }} + python_version: ${{ fromJson(needs.input_value_selection.outputs.selected_python_for_tls) }} fail-fast: false # do not immediately fail if one of the combinations fail env: @@ -162,8 +162,8 @@ jobs: matrix: # Testing this scenario only for torch_cnn_mnist model and python 3.10 # If required, this can be extended to other models and python versions - model_name: ${{ needs.input_value_selection.outputs.selected_models_for_non_tls }} - python_version: ${{ needs.input_value_selection.outputs.selected_python_for_non_tls }} + model_name: ${{ fromJson(needs.input_value_selection.outputs.selected_models_for_non_tls) }} + python_version: ${{ fromJson(needs.input_value_selection.outputs.selected_python_for_non_tls) }} fail-fast: false # do not immediately fail if one of the combinations fail env: @@ -207,8 +207,8 @@ jobs: matrix: # Testing this scenario for keras_cnn_mnist model and python 3.10 # If required, this can be extended to other models and python versions - model_name: ${{ needs.input_value_selection.outputs.selected_models_for_no_client_auth }} - python_version: ${{ needs.input_value_selection.outputs.selected_python_for_no_client_auth }} + model_name: ${{ fromJson(needs.input_value_selection.outputs.selected_models_for_no_client_auth) }} + python_version: ${{ fromJson(needs.input_value_selection.outputs.selected_python_for_no_client_auth) }} fail-fast: false # do not immediately fail if one of the combinations fail env: @@ -252,8 +252,8 @@ jobs: matrix: # Testing this scenario only for torch_cnn_mnist model and python 3.10 # If required, this can be extended to other models and python versions - model_name: ${{ needs.input_value_selection.outputs.selected_models_for_memory_logs }} - python_version: ${{ needs.input_value_selection.outputs.selected_python_for_memory_logs }} + model_name: ${{ fromJson(needs.input_value_selection.outputs.selected_models_for_memory_logs) }} + python_version: ${{ fromJson(needs.input_value_selection.outputs.selected_python_for_memory_logs) }} fail-fast: false # do not immediately fail if one of the combinations fail env: From 1d58df5bb25533355a22af1910376be910c666a2 Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 17:17:12 +0000 Subject: [PATCH 13/39] Summary file check Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 40 +++++++++++++-------- tests/end_to_end/utils/summary_helper.py | 5 ++- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index 1b9d6356dc..56d1ea3731 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -65,7 +65,7 @@ jobs: if: | (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') || (github.event_name == 'workflow_dispatch') - name: Initial Job + name: Select input values runs-on: ubuntu-22.04 outputs: selected_jobs: ${{ steps.input_value_selection.outputs.jobs_to_run }} @@ -81,6 +81,15 @@ jobs: - name: Job to select input values id: input_value_selection run: | + # --------------------------------------------------------------- + # Models like XGBoost (xgb_higgs) and torch_cnn_histology require runners with higher memory and CPU to run. + # Thus these models are excluded from the matrix for now. + # Below combinations will be used by default if no input is provided. + # TLS - models [torch_cnn_mnist, keras_cnn_mnist] and python versions [3.10, 3.11, 3.12] + # Non-TLS - models [torch_cnn_mnist] and python version [3.10] + # No client auth - models [keras_cnn_mnist] and python version [3.10] + # Memory logs - models [torch_cnn_mnist] and python version [3.10] + # --------------------------------------------------------------- if [ "${{ env.MODEL_NAME }}" == 'all' ]; then echo "models_for_tls=[\"torch_cnn_mnist\", \"keras_cnn_mnist\"]" >> "$GITHUB_OUTPUT" echo "models_for_non_tls=[\"torch_cnn_mnist\"]" >> "$GITHUB_OUTPUT" @@ -105,18 +114,27 @@ jobs: fi echo "jobs_to_run=${{ env.JOBS_TO_RUN }}" >> "$GITHUB_OUTPUT" - echo $GITHUB_OUTPUT + # --------------------------------------------------------------- + # Print the selected values as a summary for better visibility + # --------------------------------------------------------------- + echo "Selected jobs: $jobs_to_run" >> $GITHUB_STEP_SUMMARY + echo "Selected models for TLS: $models_for_tls" >> $GITHUB_STEP_SUMMARY + echo "Selected python versions for TLS: $python_for_tls" >> $GITHUB_STEP_SUMMARY + echo "Selected models for non-TLS: $models_for_non_tls" >> $GITHUB_STEP_SUMMARY + echo "Selected python versions for non-TLS: $python_for_non_tls" >> $GITHUB_STEP_SUMMARY + echo "Selected models for no client auth: $models_for_no_client_auth" >> $GITHUB_STEP_SUMMARY + echo "Selected python versions for no client auth: $python_for_no_client_auth" >> $GITHUB_STEP_SUMMARY + echo "Selected models for memory logs: $models_for_memory_logs" >> $GITHUB_STEP_SUMMARY + echo "Selected python versions for memory logs: $python_for_memory_logs" >> $GITHUB_STEP_SUMMARY test_with_tls: - name: tr_tls + name: Test with TLS needs: input_value_selection if: needs.input_value_selection.outputs.selected_jobs == 'all' || needs.input_value_selection.outputs.selected_jobs == 'test_with_tls' runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: matrix: - # Models like XGBoost (xgb_higgs) and torch_cnn_histology require runners with higher memory and CPU to run. - # Thus these models are excluded from the matrix for now. model_name: ${{ fromJson(needs.input_value_selection.outputs.selected_models_for_tls) }} python_version: ${{ fromJson(needs.input_value_selection.outputs.selected_python_for_tls) }} fail-fast: false # do not immediately fail if one of the combinations fail @@ -153,15 +171,13 @@ jobs: test_type: "tr_tls" test_with_non_tls: - name: tr_non_tls + name: Test without TLS needs: input_value_selection if: needs.input_value_selection.outputs.selected_jobs == 'all' || needs.input_value_selection.outputs.selected_jobs == 'test_with_non_tls' runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: matrix: - # Testing this scenario only for torch_cnn_mnist model and python 3.10 - # If required, this can be extended to other models and python versions model_name: ${{ fromJson(needs.input_value_selection.outputs.selected_models_for_non_tls) }} python_version: ${{ fromJson(needs.input_value_selection.outputs.selected_python_for_non_tls) }} fail-fast: false # do not immediately fail if one of the combinations fail @@ -198,15 +214,13 @@ jobs: test_type: "tr_non_tls" test_with_no_client_auth: - name: tr_no_client_auth + name: Test without client authentication needs: input_value_selection if: needs.input_value_selection.outputs.selected_jobs == 'all' || needs.input_value_selection.outputs.selected_jobs == 'test_with_no_client_auth' runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: matrix: - # Testing this scenario for keras_cnn_mnist model and python 3.10 - # If required, this can be extended to other models and python versions model_name: ${{ fromJson(needs.input_value_selection.outputs.selected_models_for_no_client_auth) }} python_version: ${{ fromJson(needs.input_value_selection.outputs.selected_python_for_no_client_auth) }} fail-fast: false # do not immediately fail if one of the combinations fail @@ -243,15 +257,13 @@ jobs: test_type: "tr_no_client_auth" test_memory_logs: - name: tr_tls_memory_logs + name: Test memory usage needs: input_value_selection if: needs.input_value_selection.outputs.selected_jobs == 'all' || needs.input_value_selection.outputs.selected_jobs == 'test_memory_logs' runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: matrix: - # Testing this scenario only for torch_cnn_mnist model and python 3.10 - # If required, this can be extended to other models and python versions model_name: ${{ fromJson(needs.input_value_selection.outputs.selected_models_for_memory_logs) }} python_version: ${{ fromJson(needs.input_value_selection.outputs.selected_python_for_memory_logs) }} fail-fast: false # do not immediately fail if one of the combinations fail diff --git a/tests/end_to_end/utils/summary_helper.py b/tests/end_to_end/utils/summary_helper.py index fdd84347c7..f3f2a2fc07 100644 --- a/tests/end_to_end/utils/summary_helper.py +++ b/tests/end_to_end/utils/summary_helper.py @@ -227,7 +227,10 @@ def _get_summary_file(): """ summary_file = os.getenv("GITHUB_STEP_SUMMARY") print(f"Summary file: {summary_file}") - if "step_summary" not in summary_file.name or ".env" not in summary_file.name: + # For local runs, the summary file can be .env file. + # For GitHub actions, the summary file name should start with prefix "step_summary". + # For e.g. /home/runner/work/_temp/_runner_file_commands/step_summary_ea653f69-bf1c-4a83-a115-c801b1d312a7 + if "step_summary" not in summary_file or ".env" not in summary_file: print("Invalid summary file. Exiting...") exit(1) return summary_file From bc0581ed0c10dfb535049ea4dd59f943ee212402 Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 17:27:26 +0000 Subject: [PATCH 14/39] Correction Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 19 ++++++++++--------- tests/end_to_end/utils/summary_helper.py | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index 56d1ea3731..cf52856b69 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -117,15 +117,16 @@ jobs: # --------------------------------------------------------------- # Print the selected values as a summary for better visibility # --------------------------------------------------------------- - echo "Selected jobs: $jobs_to_run" >> $GITHUB_STEP_SUMMARY - echo "Selected models for TLS: $models_for_tls" >> $GITHUB_STEP_SUMMARY - echo "Selected python versions for TLS: $python_for_tls" >> $GITHUB_STEP_SUMMARY - echo "Selected models for non-TLS: $models_for_non_tls" >> $GITHUB_STEP_SUMMARY - echo "Selected python versions for non-TLS: $python_for_non_tls" >> $GITHUB_STEP_SUMMARY - echo "Selected models for no client auth: $models_for_no_client_auth" >> $GITHUB_STEP_SUMMARY - echo "Selected python versions for no client auth: $python_for_no_client_auth" >> $GITHUB_STEP_SUMMARY - echo "Selected models for memory logs: $models_for_memory_logs" >> $GITHUB_STEP_SUMMARY - echo "Selected python versions for memory logs: $python_for_memory_logs" >> $GITHUB_STEP_SUMMARY + echo "${{ toJSON(steps.input_value_selection.outputs) }}" >> $GITHUB_STEP_SUMMARY + # echo "Selected jobs: $jobs_to_run" >> $GITHUB_STEP_SUMMARY + # echo "Selected models for TLS: $models_for_tls" >> $GITHUB_STEP_SUMMARY + # echo "Selected python versions for TLS: $python_for_tls" >> $GITHUB_STEP_SUMMARY + # echo "Selected models for non-TLS: $models_for_non_tls" >> $GITHUB_STEP_SUMMARY + # echo "Selected python versions for non-TLS: $python_for_non_tls" >> $GITHUB_STEP_SUMMARY + # echo "Selected models for no client auth: $models_for_no_client_auth" >> $GITHUB_STEP_SUMMARY + # echo "Selected python versions for no client auth: $python_for_no_client_auth" >> $GITHUB_STEP_SUMMARY + # echo "Selected models for memory logs: $models_for_memory_logs" >> $GITHUB_STEP_SUMMARY + # echo "Selected python versions for memory logs: $python_for_memory_logs" >> $GITHUB_STEP_SUMMARY test_with_tls: name: Test with TLS diff --git a/tests/end_to_end/utils/summary_helper.py b/tests/end_to_end/utils/summary_helper.py index f3f2a2fc07..7fa1812860 100644 --- a/tests/end_to_end/utils/summary_helper.py +++ b/tests/end_to_end/utils/summary_helper.py @@ -230,7 +230,7 @@ def _get_summary_file(): # For local runs, the summary file can be .env file. # For GitHub actions, the summary file name should start with prefix "step_summary". # For e.g. /home/runner/work/_temp/_runner_file_commands/step_summary_ea653f69-bf1c-4a83-a115-c801b1d312a7 - if "step_summary" not in summary_file or ".env" not in summary_file: + if "step_summary" not in summary_file and ".env" not in summary_file: print("Invalid summary file. Exiting...") exit(1) return summary_file From 731154eac445fa2cf19b37b1d0b1eb3b8243083d Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 17:28:35 +0000 Subject: [PATCH 15/39] Correction Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index cf52856b69..1cfa95fc09 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -215,7 +215,7 @@ jobs: test_type: "tr_non_tls" test_with_no_client_auth: - name: Test without client authentication + name: Test without client auth needs: input_value_selection if: needs.input_value_selection.outputs.selected_jobs == 'all' || needs.input_value_selection.outputs.selected_jobs == 'test_with_no_client_auth' runs-on: ubuntu-22.04 From 2e569a795ea38f85de721f6f8cc2c4d20f901398 Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 17:30:49 +0000 Subject: [PATCH 16/39] Correction Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index 1cfa95fc09..b8fdcad370 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -114,6 +114,8 @@ jobs: fi echo "jobs_to_run=${{ env.JOBS_TO_RUN }}" >> "$GITHUB_OUTPUT" + - name: Print output values + run: | # --------------------------------------------------------------- # Print the selected values as a summary for better visibility # --------------------------------------------------------------- From 93e454f9fd4611d01c0b7ff82d2e10c0c11b86cb Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 17:45:18 +0000 Subject: [PATCH 17/39] Correction Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 34 +++++++++++++-------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index b8fdcad370..df51b66315 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -65,7 +65,7 @@ jobs: if: | (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') || (github.event_name == 'workflow_dispatch') - name: Select input values + name: Input value selection runs-on: ubuntu-22.04 outputs: selected_jobs: ${{ steps.input_value_selection.outputs.jobs_to_run }} @@ -89,7 +89,9 @@ jobs: # Non-TLS - models [torch_cnn_mnist] and python version [3.10] # No client auth - models [keras_cnn_mnist] and python version [3.10] # Memory logs - models [torch_cnn_mnist] and python version [3.10] - # --------------------------------------------------------------- + # --------------------------------------------------------------- + echo "jobs_to_run=${{ env.JOBS_TO_RUN }}" >> "$GITHUB_OUTPUT" + if [ "${{ env.MODEL_NAME }}" == 'all' ]; then echo "models_for_tls=[\"torch_cnn_mnist\", \"keras_cnn_mnist\"]" >> "$GITHUB_OUTPUT" echo "models_for_non_tls=[\"torch_cnn_mnist\"]" >> "$GITHUB_OUTPUT" @@ -112,23 +114,29 @@ jobs: echo "python_for_no_client_auth=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" echo "python_for_memory_logs=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" fi - echo "jobs_to_run=${{ env.JOBS_TO_RUN }}" >> "$GITHUB_OUTPUT" - name: Print output values + id: print_output_values run: | # --------------------------------------------------------------- # Print the selected values as a summary for better visibility # --------------------------------------------------------------- - echo "${{ toJSON(steps.input_value_selection.outputs) }}" >> $GITHUB_STEP_SUMMARY - # echo "Selected jobs: $jobs_to_run" >> $GITHUB_STEP_SUMMARY - # echo "Selected models for TLS: $models_for_tls" >> $GITHUB_STEP_SUMMARY - # echo "Selected python versions for TLS: $python_for_tls" >> $GITHUB_STEP_SUMMARY - # echo "Selected models for non-TLS: $models_for_non_tls" >> $GITHUB_STEP_SUMMARY - # echo "Selected python versions for non-TLS: $python_for_non_tls" >> $GITHUB_STEP_SUMMARY - # echo "Selected models for no client auth: $models_for_no_client_auth" >> $GITHUB_STEP_SUMMARY - # echo "Selected python versions for no client auth: $python_for_no_client_auth" >> $GITHUB_STEP_SUMMARY - # echo "Selected models for memory logs: $models_for_memory_logs" >> $GITHUB_STEP_SUMMARY - # echo "Selected python versions for memory logs: $python_for_memory_logs" >> $GITHUB_STEP_SUMMARY + echo "Selected jobs: ${{ steps.input_value_selection.outputs.jobs_to_run }}" >> $GITHUB_STEP_SUMMARY + echo "| Job | Model | Python Version |" >> $GITHUB_STEP_SUMMARY + echo "| ---- | ---- | ---- |" >> $GITHUB_STEP_SUMMARY + if [ "${{ env.JOBS_TO_RUN }}" == 'all' || "${{ env.JOBS_TO_RUN }}" == "test_with_tls" ]; then + echo "| Test with TLS | ${{ steps.input_value_selection.outputs.models_for_tls }} | ${{ steps.input_value_selection.outputs.python_for_tls }} |" >> $GITHUB_STEP_SUMMARY + fi + if [ "${{ env.JOBS_TO_RUN }}" == 'all' || "${{ env.JOBS_TO_RUN }}" == "test_with_non_tls" ]; then + echo "| Test without TLS | ${{ steps.input_value_selection.outputs.models_for_non_tls }} | ${{ steps.input_value_selection.outputs.python_for_non_tls }} |" >> $GITHUB_STEP_SUMMARY + fi + if [ "${{ env.JOBS_TO_RUN }}" == 'all' || "${{ env.JOBS_TO_RUN }}" == "test_with_no_client_auth" ]; then + echo "| Test without client auth | ${{ steps.input_value_selection.outputs.models_for_no_client_auth }} | ${{ steps.input_value_selection.outputs.python_for_no_client_auth }} |" >> $GITHUB_STEP_SUMMARY + fi + if [ "${{ env.JOBS_TO_RUN }}" == 'all' || "${{ env.JOBS_TO_RUN }}" == "test_memory_logs" ]; then + echo "| Test memory logs | ${{ steps.input_value_selection.outputs.models_for_memory_logs }} | ${{ steps.input_value_selection.outputs.python_for_memory_logs }} |" >> $GITHUB_STEP_SUMMARY + fi + echo "" >> $GITHUB_STEP_SUMMARY test_with_tls: name: Test with TLS From ddb5e60adff77873523c8646812063c7e48ccf78 Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 17:47:22 +0000 Subject: [PATCH 18/39] Correction Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index df51b66315..c9da939a3c 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -124,16 +124,16 @@ jobs: echo "Selected jobs: ${{ steps.input_value_selection.outputs.jobs_to_run }}" >> $GITHUB_STEP_SUMMARY echo "| Job | Model | Python Version |" >> $GITHUB_STEP_SUMMARY echo "| ---- | ---- | ---- |" >> $GITHUB_STEP_SUMMARY - if [ "${{ env.JOBS_TO_RUN }}" == 'all' || "${{ env.JOBS_TO_RUN }}" == "test_with_tls" ]; then + if [ "${{ steps.input_value_selection.outputs.jobs_to_run }}" == 'all' || "${{ steps.input_value_selection.outputs.jobs_to_run }}" == "test_with_tls" ]; then echo "| Test with TLS | ${{ steps.input_value_selection.outputs.models_for_tls }} | ${{ steps.input_value_selection.outputs.python_for_tls }} |" >> $GITHUB_STEP_SUMMARY fi - if [ "${{ env.JOBS_TO_RUN }}" == 'all' || "${{ env.JOBS_TO_RUN }}" == "test_with_non_tls" ]; then + if [ "${{ steps.input_value_selection.outputs.jobs_to_run }}" == 'all' || "${{ steps.input_value_selection.outputs.jobs_to_run }}" == "test_with_non_tls" ]; then echo "| Test without TLS | ${{ steps.input_value_selection.outputs.models_for_non_tls }} | ${{ steps.input_value_selection.outputs.python_for_non_tls }} |" >> $GITHUB_STEP_SUMMARY fi - if [ "${{ env.JOBS_TO_RUN }}" == 'all' || "${{ env.JOBS_TO_RUN }}" == "test_with_no_client_auth" ]; then + if [ "${{ steps.input_value_selection.outputs.jobs_to_run }}" == 'all' || "${{ steps.input_value_selection.outputs.jobs_to_run }}" == "test_with_no_client_auth" ]; then echo "| Test without client auth | ${{ steps.input_value_selection.outputs.models_for_no_client_auth }} | ${{ steps.input_value_selection.outputs.python_for_no_client_auth }} |" >> $GITHUB_STEP_SUMMARY fi - if [ "${{ env.JOBS_TO_RUN }}" == 'all' || "${{ env.JOBS_TO_RUN }}" == "test_memory_logs" ]; then + if [ "${{ steps.input_value_selection.outputs.jobs_to_run }}" == 'all' || "${{ steps.input_value_selection.outputs.jobs_to_run }}" == "test_memory_logs" ]; then echo "| Test memory logs | ${{ steps.input_value_selection.outputs.models_for_memory_logs }} | ${{ steps.input_value_selection.outputs.python_for_memory_logs }} |" >> $GITHUB_STEP_SUMMARY fi echo "" >> $GITHUB_STEP_SUMMARY From f121898402ac41ba2888187662f31c44c9291c44 Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 17:53:22 +0000 Subject: [PATCH 19/39] Correction Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index c9da939a3c..41ec3fc5e5 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -124,16 +124,16 @@ jobs: echo "Selected jobs: ${{ steps.input_value_selection.outputs.jobs_to_run }}" >> $GITHUB_STEP_SUMMARY echo "| Job | Model | Python Version |" >> $GITHUB_STEP_SUMMARY echo "| ---- | ---- | ---- |" >> $GITHUB_STEP_SUMMARY - if [ "${{ steps.input_value_selection.outputs.jobs_to_run }}" == 'all' || "${{ steps.input_value_selection.outputs.jobs_to_run }}" == "test_with_tls" ]; then + if [ steps.input_value_selection.outputs.jobs_to_run == 'all' || steps.input_value_selection.outputs.jobs_to_run == 'test_with_tls' ]; then echo "| Test with TLS | ${{ steps.input_value_selection.outputs.models_for_tls }} | ${{ steps.input_value_selection.outputs.python_for_tls }} |" >> $GITHUB_STEP_SUMMARY fi - if [ "${{ steps.input_value_selection.outputs.jobs_to_run }}" == 'all' || "${{ steps.input_value_selection.outputs.jobs_to_run }}" == "test_with_non_tls" ]; then + if [ steps.input_value_selection.outputs.jobs_to_run == 'all' || steps.input_value_selection.outputs.jobs_to_run == 'test_with_non_tls' ]; then echo "| Test without TLS | ${{ steps.input_value_selection.outputs.models_for_non_tls }} | ${{ steps.input_value_selection.outputs.python_for_non_tls }} |" >> $GITHUB_STEP_SUMMARY fi - if [ "${{ steps.input_value_selection.outputs.jobs_to_run }}" == 'all' || "${{ steps.input_value_selection.outputs.jobs_to_run }}" == "test_with_no_client_auth" ]; then + if [ steps.input_value_selection.outputs.jobs_to_run == 'all' || steps.input_value_selection.outputs.jobs_to_run == 'test_with_no_client_auth' ]; then echo "| Test without client auth | ${{ steps.input_value_selection.outputs.models_for_no_client_auth }} | ${{ steps.input_value_selection.outputs.python_for_no_client_auth }} |" >> $GITHUB_STEP_SUMMARY fi - if [ "${{ steps.input_value_selection.outputs.jobs_to_run }}" == 'all' || "${{ steps.input_value_selection.outputs.jobs_to_run }}" == "test_memory_logs" ]; then + if [ steps.input_value_selection.outputs.jobs_to_run == 'all' || steps.input_value_selection.outputs.jobs_to_run == 'test_memory_logs' ]; then echo "| Test memory logs | ${{ steps.input_value_selection.outputs.models_for_memory_logs }} | ${{ steps.input_value_selection.outputs.python_for_memory_logs }} |" >> $GITHUB_STEP_SUMMARY fi echo "" >> $GITHUB_STEP_SUMMARY From 32b760869ba0c78c1492b1268d67742e06e96501 Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 17:56:42 +0000 Subject: [PATCH 20/39] Correction Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 72 ++++++++++----------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index 41ec3fc5e5..b6f13cbcd0 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -61,25 +61,25 @@ env: JOBS_TO_RUN: ${{ inputs.jobs_to_run || 'all' }} jobs: - input_value_selection: + input_selection: if: | (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') || (github.event_name == 'workflow_dispatch') name: Input value selection runs-on: ubuntu-22.04 outputs: - selected_jobs: ${{ steps.input_value_selection.outputs.jobs_to_run }} - selected_models_for_tls: ${{ steps.input_value_selection.outputs.models_for_tls }} - selected_python_for_tls: ${{ steps.input_value_selection.outputs.python_for_tls }} - selected_models_for_non_tls: ${{ steps.input_value_selection.outputs.models_for_non_tls }} - selected_models_for_no_client_auth: ${{ steps.input_value_selection.outputs.models_for_no_client_auth }} - selected_models_for_memory_logs: ${{ steps.input_value_selection.outputs.models_for_memory_logs }} - selected_python_for_non_tls: ${{ steps.input_value_selection.outputs.python_for_non_tls }} - selected_python_for_no_client_auth: ${{ steps.input_value_selection.outputs.python_for_no_client_auth }} - selected_python_for_memory_logs: ${{ steps.input_value_selection.outputs.python_for_memory_logs }} + selected_jobs: ${{ steps.input_selection.outputs.jobs_to_run }} + selected_models_for_tls: ${{ steps.input_selection.outputs.models_for_tls }} + selected_python_for_tls: ${{ steps.input_selection.outputs.python_for_tls }} + selected_models_for_non_tls: ${{ steps.input_selection.outputs.models_for_non_tls }} + selected_models_for_no_client_auth: ${{ steps.input_selection.outputs.models_for_no_client_auth }} + selected_models_for_memory_logs: ${{ steps.input_selection.outputs.models_for_memory_logs }} + selected_python_for_non_tls: ${{ steps.input_selection.outputs.python_for_non_tls }} + selected_python_for_no_client_auth: ${{ steps.input_selection.outputs.python_for_no_client_auth }} + selected_python_for_memory_logs: ${{ steps.input_selection.outputs.python_for_memory_logs }} steps: - name: Job to select input values - id: input_value_selection + id: input_selection run: | # --------------------------------------------------------------- # Models like XGBoost (xgb_higgs) and torch_cnn_histology require runners with higher memory and CPU to run. @@ -121,33 +121,33 @@ jobs: # --------------------------------------------------------------- # Print the selected values as a summary for better visibility # --------------------------------------------------------------- - echo "Selected jobs: ${{ steps.input_value_selection.outputs.jobs_to_run }}" >> $GITHUB_STEP_SUMMARY + echo "Selected jobs: ${{ steps.input_selection.outputs.jobs_to_run }}" >> $GITHUB_STEP_SUMMARY echo "| Job | Model | Python Version |" >> $GITHUB_STEP_SUMMARY echo "| ---- | ---- | ---- |" >> $GITHUB_STEP_SUMMARY - if [ steps.input_value_selection.outputs.jobs_to_run == 'all' || steps.input_value_selection.outputs.jobs_to_run == 'test_with_tls' ]; then - echo "| Test with TLS | ${{ steps.input_value_selection.outputs.models_for_tls }} | ${{ steps.input_value_selection.outputs.python_for_tls }} |" >> $GITHUB_STEP_SUMMARY + if [ ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'all' || ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'test_with_tls' ]; then + echo "| Test with TLS | ${{ steps.input_selection.outputs.models_for_tls }} | ${{ steps.input_selection.outputs.python_for_tls }} |" >> $GITHUB_STEP_SUMMARY fi - if [ steps.input_value_selection.outputs.jobs_to_run == 'all' || steps.input_value_selection.outputs.jobs_to_run == 'test_with_non_tls' ]; then - echo "| Test without TLS | ${{ steps.input_value_selection.outputs.models_for_non_tls }} | ${{ steps.input_value_selection.outputs.python_for_non_tls }} |" >> $GITHUB_STEP_SUMMARY + if [ ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'all' || ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'test_with_non_tls' ]; then + echo "| Test without TLS | ${{ steps.input_selection.outputs.models_for_non_tls }} | ${{ steps.input_selection.outputs.python_for_non_tls }} |" >> $GITHUB_STEP_SUMMARY fi - if [ steps.input_value_selection.outputs.jobs_to_run == 'all' || steps.input_value_selection.outputs.jobs_to_run == 'test_with_no_client_auth' ]; then - echo "| Test without client auth | ${{ steps.input_value_selection.outputs.models_for_no_client_auth }} | ${{ steps.input_value_selection.outputs.python_for_no_client_auth }} |" >> $GITHUB_STEP_SUMMARY + if [ ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'all' || ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'test_with_no_client_auth' ]; then + echo "| Test without client auth | ${{ steps.input_selection.outputs.models_for_no_client_auth }} | ${{ steps.input_selection.outputs.python_for_no_client_auth }} |" >> $GITHUB_STEP_SUMMARY fi - if [ steps.input_value_selection.outputs.jobs_to_run == 'all' || steps.input_value_selection.outputs.jobs_to_run == 'test_memory_logs' ]; then - echo "| Test memory logs | ${{ steps.input_value_selection.outputs.models_for_memory_logs }} | ${{ steps.input_value_selection.outputs.python_for_memory_logs }} |" >> $GITHUB_STEP_SUMMARY + if [ ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'all' || ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'test_memory_logs' ]; then + echo "| Test memory logs | ${{ steps.input_selection.outputs.models_for_memory_logs }} | ${{ steps.input_selection.outputs.python_for_memory_logs }} |" >> $GITHUB_STEP_SUMMARY fi echo "" >> $GITHUB_STEP_SUMMARY test_with_tls: name: Test with TLS - needs: input_value_selection - if: needs.input_value_selection.outputs.selected_jobs == 'all' || needs.input_value_selection.outputs.selected_jobs == 'test_with_tls' + needs: input_selection + if: needs.input_selection.outputs.selected_jobs == 'all' || needs.input_selection.outputs.selected_jobs == 'test_with_tls' runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: matrix: - model_name: ${{ fromJson(needs.input_value_selection.outputs.selected_models_for_tls) }} - python_version: ${{ fromJson(needs.input_value_selection.outputs.selected_python_for_tls) }} + model_name: ${{ fromJson(needs.input_selection.outputs.selected_models_for_tls) }} + python_version: ${{ fromJson(needs.input_selection.outputs.selected_python_for_tls) }} fail-fast: false # do not immediately fail if one of the combinations fail env: @@ -183,14 +183,14 @@ jobs: test_with_non_tls: name: Test without TLS - needs: input_value_selection - if: needs.input_value_selection.outputs.selected_jobs == 'all' || needs.input_value_selection.outputs.selected_jobs == 'test_with_non_tls' + needs: input_selection + if: needs.input_selection.outputs.selected_jobs == 'all' || needs.input_selection.outputs.selected_jobs == 'test_with_non_tls' runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: matrix: - model_name: ${{ fromJson(needs.input_value_selection.outputs.selected_models_for_non_tls) }} - python_version: ${{ fromJson(needs.input_value_selection.outputs.selected_python_for_non_tls) }} + model_name: ${{ fromJson(needs.input_selection.outputs.selected_models_for_non_tls) }} + python_version: ${{ fromJson(needs.input_selection.outputs.selected_python_for_non_tls) }} fail-fast: false # do not immediately fail if one of the combinations fail env: @@ -226,14 +226,14 @@ jobs: test_with_no_client_auth: name: Test without client auth - needs: input_value_selection - if: needs.input_value_selection.outputs.selected_jobs == 'all' || needs.input_value_selection.outputs.selected_jobs == 'test_with_no_client_auth' + needs: input_selection + if: needs.input_selection.outputs.selected_jobs == 'all' || needs.input_selection.outputs.selected_jobs == 'test_with_no_client_auth' runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: matrix: - model_name: ${{ fromJson(needs.input_value_selection.outputs.selected_models_for_no_client_auth) }} - python_version: ${{ fromJson(needs.input_value_selection.outputs.selected_python_for_no_client_auth) }} + model_name: ${{ fromJson(needs.input_selection.outputs.selected_models_for_no_client_auth) }} + python_version: ${{ fromJson(needs.input_selection.outputs.selected_python_for_no_client_auth) }} fail-fast: false # do not immediately fail if one of the combinations fail env: @@ -269,14 +269,14 @@ jobs: test_memory_logs: name: Test memory usage - needs: input_value_selection - if: needs.input_value_selection.outputs.selected_jobs == 'all' || needs.input_value_selection.outputs.selected_jobs == 'test_memory_logs' + needs: input_selection + if: needs.input_selection.outputs.selected_jobs == 'all' || needs.input_selection.outputs.selected_jobs == 'test_memory_logs' runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: matrix: - model_name: ${{ fromJson(needs.input_value_selection.outputs.selected_models_for_memory_logs) }} - python_version: ${{ fromJson(needs.input_value_selection.outputs.selected_python_for_memory_logs) }} + model_name: ${{ fromJson(needs.input_selection.outputs.selected_models_for_memory_logs) }} + python_version: ${{ fromJson(needs.input_selection.outputs.selected_python_for_memory_logs) }} fail-fast: false # do not immediately fail if one of the combinations fail env: From f7ec4bbb9b60bc2c246ab7e53bcc7fec90903a5a Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 18:05:23 +0000 Subject: [PATCH 21/39] Correction Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index b6f13cbcd0..39762b6a4c 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -122,6 +122,7 @@ jobs: # Print the selected values as a summary for better visibility # --------------------------------------------------------------- echo "Selected jobs: ${{ steps.input_selection.outputs.jobs_to_run }}" >> $GITHUB_STEP_SUMMARY + echo ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} >> $GITHUB_STEP_SUMMARY echo "| Job | Model | Python Version |" >> $GITHUB_STEP_SUMMARY echo "| ---- | ---- | ---- |" >> $GITHUB_STEP_SUMMARY if [ ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'all' || ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'test_with_tls' ]; then From 1e15d307b8cfb2b690069383d2398bcdbeee6a16 Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 18:07:06 +0000 Subject: [PATCH 22/39] Correction Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 24 ++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index 39762b6a4c..a76e443d9c 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -125,18 +125,18 @@ jobs: echo ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} >> $GITHUB_STEP_SUMMARY echo "| Job | Model | Python Version |" >> $GITHUB_STEP_SUMMARY echo "| ---- | ---- | ---- |" >> $GITHUB_STEP_SUMMARY - if [ ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'all' || ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'test_with_tls' ]; then - echo "| Test with TLS | ${{ steps.input_selection.outputs.models_for_tls }} | ${{ steps.input_selection.outputs.python_for_tls }} |" >> $GITHUB_STEP_SUMMARY - fi - if [ ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'all' || ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'test_with_non_tls' ]; then - echo "| Test without TLS | ${{ steps.input_selection.outputs.models_for_non_tls }} | ${{ steps.input_selection.outputs.python_for_non_tls }} |" >> $GITHUB_STEP_SUMMARY - fi - if [ ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'all' || ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'test_with_no_client_auth' ]; then - echo "| Test without client auth | ${{ steps.input_selection.outputs.models_for_no_client_auth }} | ${{ steps.input_selection.outputs.python_for_no_client_auth }} |" >> $GITHUB_STEP_SUMMARY - fi - if [ ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'all' || ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'test_memory_logs' ]; then - echo "| Test memory logs | ${{ steps.input_selection.outputs.models_for_memory_logs }} | ${{ steps.input_selection.outputs.python_for_memory_logs }} |" >> $GITHUB_STEP_SUMMARY - fi + # if [ ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'all' || ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'test_with_tls' ]; then + # echo "| Test with TLS | ${{ steps.input_selection.outputs.models_for_tls }} | ${{ steps.input_selection.outputs.python_for_tls }} |" >> $GITHUB_STEP_SUMMARY + # fi + # if [ ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'all' || ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'test_with_non_tls' ]; then + # echo "| Test without TLS | ${{ steps.input_selection.outputs.models_for_non_tls }} | ${{ steps.input_selection.outputs.python_for_non_tls }} |" >> $GITHUB_STEP_SUMMARY + # fi + # if [ ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'all' || ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'test_with_no_client_auth' ]; then + # echo "| Test without client auth | ${{ steps.input_selection.outputs.models_for_no_client_auth }} | ${{ steps.input_selection.outputs.python_for_no_client_auth }} |" >> $GITHUB_STEP_SUMMARY + # fi + # if [ ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'all' || ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'test_memory_logs' ]; then + # echo "| Test memory logs | ${{ steps.input_selection.outputs.models_for_memory_logs }} | ${{ steps.input_selection.outputs.python_for_memory_logs }} |" >> $GITHUB_STEP_SUMMARY + # fi echo "" >> $GITHUB_STEP_SUMMARY test_with_tls: From 92b141775e61810de6935b4389b745497adfe646 Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 18:08:40 +0000 Subject: [PATCH 23/39] Correction Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index a76e443d9c..4f310983a3 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -122,7 +122,7 @@ jobs: # Print the selected values as a summary for better visibility # --------------------------------------------------------------- echo "Selected jobs: ${{ steps.input_selection.outputs.jobs_to_run }}" >> $GITHUB_STEP_SUMMARY - echo ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} >> $GITHUB_STEP_SUMMARY + # echo ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} >> $GITHUB_STEP_SUMMARY echo "| Job | Model | Python Version |" >> $GITHUB_STEP_SUMMARY echo "| ---- | ---- | ---- |" >> $GITHUB_STEP_SUMMARY # if [ ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'all' || ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'test_with_tls' ]; then From 69ad9b88ef3889ef998c317a3a0fc829286bb3e6 Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 18:10:41 +0000 Subject: [PATCH 24/39] Correction Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index 4f310983a3..37a6a24b62 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -118,25 +118,9 @@ jobs: - name: Print output values id: print_output_values run: | - # --------------------------------------------------------------- - # Print the selected values as a summary for better visibility - # --------------------------------------------------------------- echo "Selected jobs: ${{ steps.input_selection.outputs.jobs_to_run }}" >> $GITHUB_STEP_SUMMARY - # echo ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} >> $GITHUB_STEP_SUMMARY echo "| Job | Model | Python Version |" >> $GITHUB_STEP_SUMMARY echo "| ---- | ---- | ---- |" >> $GITHUB_STEP_SUMMARY - # if [ ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'all' || ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'test_with_tls' ]; then - # echo "| Test with TLS | ${{ steps.input_selection.outputs.models_for_tls }} | ${{ steps.input_selection.outputs.python_for_tls }} |" >> $GITHUB_STEP_SUMMARY - # fi - # if [ ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'all' || ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'test_with_non_tls' ]; then - # echo "| Test without TLS | ${{ steps.input_selection.outputs.models_for_non_tls }} | ${{ steps.input_selection.outputs.python_for_non_tls }} |" >> $GITHUB_STEP_SUMMARY - # fi - # if [ ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'all' || ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'test_with_no_client_auth' ]; then - # echo "| Test without client auth | ${{ steps.input_selection.outputs.models_for_no_client_auth }} | ${{ steps.input_selection.outputs.python_for_no_client_auth }} |" >> $GITHUB_STEP_SUMMARY - # fi - # if [ ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'all' || ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'test_memory_logs' ]; then - # echo "| Test memory logs | ${{ steps.input_selection.outputs.models_for_memory_logs }} | ${{ steps.input_selection.outputs.python_for_memory_logs }} |" >> $GITHUB_STEP_SUMMARY - # fi echo "" >> $GITHUB_STEP_SUMMARY test_with_tls: From d4215957d90a803129fe412e16b89385fe73ff95 Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 18:19:30 +0000 Subject: [PATCH 25/39] Correction Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index 37a6a24b62..6732e30f2e 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -84,11 +84,11 @@ jobs: # --------------------------------------------------------------- # Models like XGBoost (xgb_higgs) and torch_cnn_histology require runners with higher memory and CPU to run. # Thus these models are excluded from the matrix for now. - # Below combinations will be used by default if no input is provided. - # TLS - models [torch_cnn_mnist, keras_cnn_mnist] and python versions [3.10, 3.11, 3.12] - # Non-TLS - models [torch_cnn_mnist] and python version [3.10] - # No client auth - models [keras_cnn_mnist] and python version [3.10] - # Memory logs - models [torch_cnn_mnist] and python version [3.10] + # Default combination if no input is provided. + # * TLS - models [torch_cnn_mnist, keras_cnn_mnist] and python versions [3.10, 3.11, 3.12] + # * Non-TLS - models [torch_cnn_mnist] and python version [3.10] + # * No client auth - models [keras_cnn_mnist] and python version [3.10] + # * Memory logs - models [torch_cnn_mnist] and python version [3.10] # --------------------------------------------------------------- echo "jobs_to_run=${{ env.JOBS_TO_RUN }}" >> "$GITHUB_OUTPUT" @@ -121,6 +121,9 @@ jobs: echo "Selected jobs: ${{ steps.input_selection.outputs.jobs_to_run }}" >> $GITHUB_STEP_SUMMARY echo "| Job | Model | Python Version |" >> $GITHUB_STEP_SUMMARY echo "| ---- | ---- | ---- |" >> $GITHUB_STEP_SUMMARY + if [ ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'all' || ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'test_with_tls' ]; then + echo "| Test with TLS | ${{ steps.input_selection.outputs.models_for_tls }} | ${{ steps.input_selection.outputs.python_for_tls }} |" >> $GITHUB_STEP_SUMMARY + fi echo "" >> $GITHUB_STEP_SUMMARY test_with_tls: From 23c919ce890c6f1731f923953e2c450140fc678f Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 18:21:22 +0000 Subject: [PATCH 26/39] Correction Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index 6732e30f2e..ebddf3c33f 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -121,7 +121,7 @@ jobs: echo "Selected jobs: ${{ steps.input_selection.outputs.jobs_to_run }}" >> $GITHUB_STEP_SUMMARY echo "| Job | Model | Python Version |" >> $GITHUB_STEP_SUMMARY echo "| ---- | ---- | ---- |" >> $GITHUB_STEP_SUMMARY - if [ ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'all' || ${{ fromJson(steps.input_selection.outputs.jobs_to_run) }} == 'test_with_tls' ]; then + if [ "${{ steps.input_selection.outputs.jobs_to_run }}" == "all" || "${{ steps.input_selection.outputs.jobs_to_run }}" == "test_with_tls" ]; then echo "| Test with TLS | ${{ steps.input_selection.outputs.models_for_tls }} | ${{ steps.input_selection.outputs.python_for_tls }} |" >> $GITHUB_STEP_SUMMARY fi echo "" >> $GITHUB_STEP_SUMMARY From c67da55df2417eeac477720821b9948d27e348bb Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 18:23:50 +0000 Subject: [PATCH 27/39] Correction Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index ebddf3c33f..7346b373ab 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -121,7 +121,7 @@ jobs: echo "Selected jobs: ${{ steps.input_selection.outputs.jobs_to_run }}" >> $GITHUB_STEP_SUMMARY echo "| Job | Model | Python Version |" >> $GITHUB_STEP_SUMMARY echo "| ---- | ---- | ---- |" >> $GITHUB_STEP_SUMMARY - if [ "${{ steps.input_selection.outputs.jobs_to_run }}" == "all" || "${{ steps.input_selection.outputs.jobs_to_run }}" == "test_with_tls" ]; then + if [ "${{ steps.input_selection.outputs.jobs_to_run }}" == "all" ] || [ "${{ steps.input_selection.outputs.jobs_to_run }}" == "test_with_tls" ]; then echo "| Test with TLS | ${{ steps.input_selection.outputs.models_for_tls }} | ${{ steps.input_selection.outputs.python_for_tls }} |" >> $GITHUB_STEP_SUMMARY fi echo "" >> $GITHUB_STEP_SUMMARY From 76745bb53677b42aca7425e27ab4edd7f024211e Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 18:26:44 +0000 Subject: [PATCH 28/39] Correction Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index 7346b373ab..6053fd2daa 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -124,6 +124,15 @@ jobs: if [ "${{ steps.input_selection.outputs.jobs_to_run }}" == "all" ] || [ "${{ steps.input_selection.outputs.jobs_to_run }}" == "test_with_tls" ]; then echo "| Test with TLS | ${{ steps.input_selection.outputs.models_for_tls }} | ${{ steps.input_selection.outputs.python_for_tls }} |" >> $GITHUB_STEP_SUMMARY fi + if [ "${{ steps.input_selection.outputs.jobs_to_run }}" == "all" ] || [ "${{ steps.input_selection.outputs.jobs_to_run }}" == "test_with_non_tls" ]; then + echo "| Test without TLS | ${{ steps.input_selection.outputs.models_for_non_tls }} | ${{ steps.input_selection.outputs.python_for_non_tls }} |" >> $GITHUB_STEP_SUMMARY + fi + if [ "${{ steps.input_selection.outputs.jobs_to_run }}" == "all" ] || [ "${{ steps.input_selection.outputs.jobs_to_run }}" == "test_with_no_client_auth" ]; then + echo "| Test without client auth | ${{ steps.input_selection.outputs.models_for_no_client_auth }} | ${{ steps.input_selection.outputs.python_for_no_client_auth }} |" >> $GITHUB_STEP_SUMMARY + fi + if [ "${{ steps.input_selection.outputs.jobs_to_run }}" == "all" ] || [ "${{ steps.input_selection.outputs.jobs_to_run }}" == "test_memory_logs" ]; then + echo "| Test memory logs | ${{ steps.input_selection.outputs.models_for_memory_logs }} | ${{ steps.input_selection.outputs.python_for_memory_logs }} |" >> $GITHUB_STEP_SUMMARY + fi echo "" >> $GITHUB_STEP_SUMMARY test_with_tls: From b230c9458e784e4eb6926d93d46064a33934a9d4 Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 18:28:57 +0000 Subject: [PATCH 29/39] Correction Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index 6053fd2daa..6d96a7f8c2 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -118,8 +118,7 @@ jobs: - name: Print output values id: print_output_values run: | - echo "Selected jobs: ${{ steps.input_selection.outputs.jobs_to_run }}" >> $GITHUB_STEP_SUMMARY - echo "| Job | Model | Python Version |" >> $GITHUB_STEP_SUMMARY + echo "| Selected job(s) | Model(s) | Python version(s) |" >> $GITHUB_STEP_SUMMARY echo "| ---- | ---- | ---- |" >> $GITHUB_STEP_SUMMARY if [ "${{ steps.input_selection.outputs.jobs_to_run }}" == "all" ] || [ "${{ steps.input_selection.outputs.jobs_to_run }}" == "test_with_tls" ]; then echo "| Test with TLS | ${{ steps.input_selection.outputs.models_for_tls }} | ${{ steps.input_selection.outputs.python_for_tls }} |" >> $GITHUB_STEP_SUMMARY From 3249a86088f6d1a60342b46d2b7c372905582a2f Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 19:03:49 +0000 Subject: [PATCH 30/39] Correction Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index 6d96a7f8c2..f0b798eb2d 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -115,25 +115,6 @@ jobs: echo "python_for_memory_logs=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" fi - - name: Print output values - id: print_output_values - run: | - echo "| Selected job(s) | Model(s) | Python version(s) |" >> $GITHUB_STEP_SUMMARY - echo "| ---- | ---- | ---- |" >> $GITHUB_STEP_SUMMARY - if [ "${{ steps.input_selection.outputs.jobs_to_run }}" == "all" ] || [ "${{ steps.input_selection.outputs.jobs_to_run }}" == "test_with_tls" ]; then - echo "| Test with TLS | ${{ steps.input_selection.outputs.models_for_tls }} | ${{ steps.input_selection.outputs.python_for_tls }} |" >> $GITHUB_STEP_SUMMARY - fi - if [ "${{ steps.input_selection.outputs.jobs_to_run }}" == "all" ] || [ "${{ steps.input_selection.outputs.jobs_to_run }}" == "test_with_non_tls" ]; then - echo "| Test without TLS | ${{ steps.input_selection.outputs.models_for_non_tls }} | ${{ steps.input_selection.outputs.python_for_non_tls }} |" >> $GITHUB_STEP_SUMMARY - fi - if [ "${{ steps.input_selection.outputs.jobs_to_run }}" == "all" ] || [ "${{ steps.input_selection.outputs.jobs_to_run }}" == "test_with_no_client_auth" ]; then - echo "| Test without client auth | ${{ steps.input_selection.outputs.models_for_no_client_auth }} | ${{ steps.input_selection.outputs.python_for_no_client_auth }} |" >> $GITHUB_STEP_SUMMARY - fi - if [ "${{ steps.input_selection.outputs.jobs_to_run }}" == "all" ] || [ "${{ steps.input_selection.outputs.jobs_to_run }}" == "test_memory_logs" ]; then - echo "| Test memory logs | ${{ steps.input_selection.outputs.models_for_memory_logs }} | ${{ steps.input_selection.outputs.python_for_memory_logs }} |" >> $GITHUB_STEP_SUMMARY - fi - echo "" >> $GITHUB_STEP_SUMMARY - test_with_tls: name: Test with TLS needs: input_selection From 94da0df29da3ec633204ef04eaf8d336a4ef6b77 Mon Sep 17 00:00:00 2001 From: noopur Date: Wed, 22 Jan 2025 19:04:49 +0000 Subject: [PATCH 31/39] Correction Signed-off-by: noopur --- tests/end_to_end/utils/summary_helper.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/end_to_end/utils/summary_helper.py b/tests/end_to_end/utils/summary_helper.py index 7fa1812860..0d03fb12f0 100644 --- a/tests/end_to_end/utils/summary_helper.py +++ b/tests/end_to_end/utils/summary_helper.py @@ -42,7 +42,6 @@ def get_aggregated_accuracy(agg_log_file): return agg_accuracy agg_accuracy_dict = convert_to_json(agg_log_file) - print(f"agg_accuracy_dict is: {agg_accuracy_dict}") if not agg_accuracy_dict: print(f"Aggregator log file {agg_log_file} is empty. Cannot get aggregated accuracy, returning 'Not Found'") From 04ac6c7487048883dd7a3ab919c2dfe11840a27a Mon Sep 17 00:00:00 2001 From: noopur Date: Thu, 23 Jan 2025 04:02:22 +0000 Subject: [PATCH 32/39] Final changes in E2E yaml Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 38 +++++++++++---------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index f0b798eb2d..e7d3f40467 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -21,7 +21,7 @@ on: model_name: description: "Model name" required: false - default: 'all' + default: "all" type: choice options: - all @@ -30,17 +30,17 @@ on: python_version: description: "Python version" required: false - default: 'all' + default: "all" type: choice options: - all - - '3.10' + - "3.10" - "3.11" - "3.12" jobs_to_run: description: "Jobs to run" type: choice - default: 'all' + default: "all" options: - all - test_with_tls @@ -54,20 +54,22 @@ permissions: # Environment variables common for all the jobs env: - NUM_ROUNDS: ${{ inputs.num_rounds || '5' }} - NUM_COLLABORATORS: ${{ inputs.num_collaborators || '2' }} - MODEL_NAME: ${{ inputs.model_name || 'all' }} - PYTHON_VERSION: ${{ inputs.python_version || 'all' }} - JOBS_TO_RUN: ${{ inputs.jobs_to_run || 'all' }} + NUM_ROUNDS: ${{ inputs.num_rounds || "5" }} + NUM_COLLABORATORS: ${{ inputs.num_collaborators || "2" }} + MODEL_NAME: ${{ inputs.model_name || "all" }} + PYTHON_VERSION: ${{ inputs.python_version || "all" }} + JOBS_TO_RUN: ${{ inputs.jobs_to_run || "all" }} jobs: input_selection: if: | - (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') || - (github.event_name == 'workflow_dispatch') + (github.event_name == "schedule" && github.repository_owner == "securefederatedai") || + (github.event_name == "workflow_dispatch") name: Input value selection runs-on: ubuntu-22.04 outputs: + # Output all the variables related to models and python versions to be used in the matrix strategy + # for different jobs, however their usage depends on the selected job. selected_jobs: ${{ steps.input_selection.outputs.jobs_to_run }} selected_models_for_tls: ${{ steps.input_selection.outputs.models_for_tls }} selected_python_for_tls: ${{ steps.input_selection.outputs.python_for_tls }} @@ -84,7 +86,7 @@ jobs: # --------------------------------------------------------------- # Models like XGBoost (xgb_higgs) and torch_cnn_histology require runners with higher memory and CPU to run. # Thus these models are excluded from the matrix for now. - # Default combination if no input is provided. + # Default combination if no input is provided (i.e. "all" is selected). # * TLS - models [torch_cnn_mnist, keras_cnn_mnist] and python versions [3.10, 3.11, 3.12] # * Non-TLS - models [torch_cnn_mnist] and python version [3.10] # * No client auth - models [keras_cnn_mnist] and python version [3.10] @@ -92,7 +94,7 @@ jobs: # --------------------------------------------------------------- echo "jobs_to_run=${{ env.JOBS_TO_RUN }}" >> "$GITHUB_OUTPUT" - if [ "${{ env.MODEL_NAME }}" == 'all' ]; then + if [ "${{ env.MODEL_NAME }}" == "all" ]; then echo "models_for_tls=[\"torch_cnn_mnist\", \"keras_cnn_mnist\"]" >> "$GITHUB_OUTPUT" echo "models_for_non_tls=[\"torch_cnn_mnist\"]" >> "$GITHUB_OUTPUT" echo "models_for_no_client_auth=[\"keras_cnn_mnist\"]" >> "$GITHUB_OUTPUT" @@ -103,7 +105,7 @@ jobs: echo "models_for_no_client_auth=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" echo "models_for_memory_logs=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" fi - if [ "${{ env.PYTHON_VERSION }}" == 'all' ]; then + if [ "${{ env.PYTHON_VERSION }}" == "all" ]; then echo "python_for_tls=[\"3.10\", \"3.11\", \"3.12\"]" >> "$GITHUB_OUTPUT" echo "python_for_non_tls=[\"3.10\"]" >> "$GITHUB_OUTPUT" echo "python_for_no_client_auth=[\"3.10\"]" >> "$GITHUB_OUTPUT" @@ -118,7 +120,7 @@ jobs: test_with_tls: name: Test with TLS needs: input_selection - if: needs.input_selection.outputs.selected_jobs == 'all' || needs.input_selection.outputs.selected_jobs == 'test_with_tls' + if: needs.input_selection.outputs.selected_jobs == "all" || needs.input_selection.outputs.selected_jobs == "test_with_tls" runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: @@ -161,7 +163,7 @@ jobs: test_with_non_tls: name: Test without TLS needs: input_selection - if: needs.input_selection.outputs.selected_jobs == 'all' || needs.input_selection.outputs.selected_jobs == 'test_with_non_tls' + if: needs.input_selection.outputs.selected_jobs == "all" || needs.input_selection.outputs.selected_jobs == "test_with_non_tls" runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: @@ -204,7 +206,7 @@ jobs: test_with_no_client_auth: name: Test without client auth needs: input_selection - if: needs.input_selection.outputs.selected_jobs == 'all' || needs.input_selection.outputs.selected_jobs == 'test_with_no_client_auth' + if: needs.input_selection.outputs.selected_jobs == "all" || needs.input_selection.outputs.selected_jobs == "test_with_no_client_auth" runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: @@ -247,7 +249,7 @@ jobs: test_memory_logs: name: Test memory usage needs: input_selection - if: needs.input_selection.outputs.selected_jobs == 'all' || needs.input_selection.outputs.selected_jobs == 'test_memory_logs' + if: needs.input_selection.outputs.selected_jobs == "all" || needs.input_selection.outputs.selected_jobs == "test_memory_logs" runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: From 5740e83f978456c3f92ffd7c11a2e9a0cdfead63 Mon Sep 17 00:00:00 2001 From: noopur Date: Thu, 23 Jan 2025 04:05:40 +0000 Subject: [PATCH 33/39] Final changes in E2E yaml Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index e7d3f40467..abf925ad26 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -54,8 +54,8 @@ permissions: # Environment variables common for all the jobs env: - NUM_ROUNDS: ${{ inputs.num_rounds || "5" }} - NUM_COLLABORATORS: ${{ inputs.num_collaborators || "2" }} + NUM_ROUNDS: ${{ inputs.num_rounds || 5 }} + NUM_COLLABORATORS: ${{ inputs.num_collaborators || 2 }} MODEL_NAME: ${{ inputs.model_name || "all" }} PYTHON_VERSION: ${{ inputs.python_version || "all" }} JOBS_TO_RUN: ${{ inputs.jobs_to_run || "all" }} From e803bf73b1e3ec04e4b9412b5a3b61639ac9bdd0 Mon Sep 17 00:00:00 2001 From: noopur Date: Thu, 23 Jan 2025 04:07:06 +0000 Subject: [PATCH 34/39] Final changes in E2E yaml Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index abf925ad26..77bf547ed6 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -53,12 +53,13 @@ permissions: contents: read # Environment variables common for all the jobs +# DO NOT use double quotes for the values of the environment variables env: NUM_ROUNDS: ${{ inputs.num_rounds || 5 }} NUM_COLLABORATORS: ${{ inputs.num_collaborators || 2 }} - MODEL_NAME: ${{ inputs.model_name || "all" }} - PYTHON_VERSION: ${{ inputs.python_version || "all" }} - JOBS_TO_RUN: ${{ inputs.jobs_to_run || "all" }} + MODEL_NAME: ${{ inputs.model_name || 'all' }} + PYTHON_VERSION: ${{ inputs.python_version || 'all' }} + JOBS_TO_RUN: ${{ inputs.jobs_to_run || 'all' }} jobs: input_selection: From ea58cd463f9b6d7b5fb7282126172ce4cf045fb4 Mon Sep 17 00:00:00 2001 From: noopur Date: Thu, 23 Jan 2025 04:12:23 +0000 Subject: [PATCH 35/39] All single quotes Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 104 ++++++++++---------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index 77bf547ed6..a25a0fe4fd 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -5,42 +5,42 @@ name: Task_Runner_E2E # Please do not modify the name as it is used in the comp on: schedule: - - cron: "0 0 * * *" # Run every day at midnight + - cron: '0 0 * * *' # Run every day at midnight workflow_dispatch: inputs: num_rounds: - description: "Number of rounds to train" + description: 'Number of rounds to train' required: false - default: "5" + default: '5' type: string num_collaborators: - description: "Number of collaborators" + description: 'Number of collaborators' required: false - default: "2" + default: '2' type: string model_name: - description: "Model name" + description: 'Model name' required: false - default: "all" + default: 'all' type: choice options: - all - torch_cnn_mnist - keras_cnn_mnist python_version: - description: "Python version" + description: 'Python version' required: false - default: "all" + default: 'all' type: choice options: - all - - "3.10" - - "3.11" - - "3.12" + - '3.10' + - '3.11' + - '3.12' jobs_to_run: - description: "Jobs to run" + description: 'Jobs to run' type: choice - default: "all" + default: 'all' options: - all - test_with_tls @@ -64,8 +64,8 @@ env: jobs: input_selection: if: | - (github.event_name == "schedule" && github.repository_owner == "securefederatedai") || - (github.event_name == "workflow_dispatch") + (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') || + (github.event_name == 'workflow_dispatch') name: Input value selection runs-on: ubuntu-22.04 outputs: @@ -87,41 +87,41 @@ jobs: # --------------------------------------------------------------- # Models like XGBoost (xgb_higgs) and torch_cnn_histology require runners with higher memory and CPU to run. # Thus these models are excluded from the matrix for now. - # Default combination if no input is provided (i.e. "all" is selected). + # Default combination if no input is provided (i.e. 'all' is selected). # * TLS - models [torch_cnn_mnist, keras_cnn_mnist] and python versions [3.10, 3.11, 3.12] # * Non-TLS - models [torch_cnn_mnist] and python version [3.10] # * No client auth - models [keras_cnn_mnist] and python version [3.10] # * Memory logs - models [torch_cnn_mnist] and python version [3.10] # --------------------------------------------------------------- - echo "jobs_to_run=${{ env.JOBS_TO_RUN }}" >> "$GITHUB_OUTPUT" + echo 'jobs_to_run=${{ env.JOBS_TO_RUN }}' >> '$GITHUB_OUTPUT' - if [ "${{ env.MODEL_NAME }}" == "all" ]; then - echo "models_for_tls=[\"torch_cnn_mnist\", \"keras_cnn_mnist\"]" >> "$GITHUB_OUTPUT" - echo "models_for_non_tls=[\"torch_cnn_mnist\"]" >> "$GITHUB_OUTPUT" - echo "models_for_no_client_auth=[\"keras_cnn_mnist\"]" >> "$GITHUB_OUTPUT" - echo "models_for_memory_logs=[\"torch_cnn_mnist\"]" >> "$GITHUB_OUTPUT" + if [ '${{ env.MODEL_NAME }}' == 'all' ]; then + echo 'models_for_tls=[\'torch_cnn_mnist\', \'keras_cnn_mnist\']' >> '$GITHUB_OUTPUT' + echo 'models_for_non_tls=[\'torch_cnn_mnist\']' >> '$GITHUB_OUTPUT' + echo 'models_for_no_client_auth=[\'keras_cnn_mnist\']' >> '$GITHUB_OUTPUT' + echo 'models_for_memory_logs=[\'torch_cnn_mnist\']' >> '$GITHUB_OUTPUT' else - echo "models_for_tls=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" - echo "models_for_non_tls=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" - echo "models_for_no_client_auth=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" - echo "models_for_memory_logs=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" + echo 'models_for_tls=[\'${{env.MODEL_NAME}}\']' >> '$GITHUB_OUTPUT' + echo 'models_for_non_tls=[\'${{env.MODEL_NAME}}\']' >> '$GITHUB_OUTPUT' + echo 'models_for_no_client_auth=[\'${{env.MODEL_NAME}}\']' >> '$GITHUB_OUTPUT' + echo 'models_for_memory_logs=[\'${{env.MODEL_NAME}}\']' >> '$GITHUB_OUTPUT' fi - if [ "${{ env.PYTHON_VERSION }}" == "all" ]; then - echo "python_for_tls=[\"3.10\", \"3.11\", \"3.12\"]" >> "$GITHUB_OUTPUT" - echo "python_for_non_tls=[\"3.10\"]" >> "$GITHUB_OUTPUT" - echo "python_for_no_client_auth=[\"3.10\"]" >> "$GITHUB_OUTPUT" - echo "python_for_memory_logs=[\"3.10\"]" >> "$GITHUB_OUTPUT" + if [ '${{ env.PYTHON_VERSION }}' == 'all' ]; then + echo 'python_for_tls=[\'3.10\', \'3.11\', \'3.12\']' >> '$GITHUB_OUTPUT' + echo 'python_for_non_tls=[\'3.10\']' >> '$GITHUB_OUTPUT' + echo 'python_for_no_client_auth=[\'3.10\']' >> '$GITHUB_OUTPUT' + echo 'python_for_memory_logs=[\'3.10\']' >> '$GITHUB_OUTPUT' else - echo "python_for_tls=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" - echo "python_for_non_tls=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" - echo "python_for_no_client_auth=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" - echo "python_for_memory_logs=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" + echo 'python_for_tls=[\'${{env.PYTHON_VERSION}}\']' >> '$GITHUB_OUTPUT' + echo 'python_for_non_tls=[\'${{env.PYTHON_VERSION}}\']' >> '$GITHUB_OUTPUT' + echo 'python_for_no_client_auth=[\'${{env.PYTHON_VERSION}}\']' >> '$GITHUB_OUTPUT' + echo 'python_for_memory_logs=[\'${{env.PYTHON_VERSION}}\']' >> '$GITHUB_OUTPUT' fi test_with_tls: name: Test with TLS needs: input_selection - if: needs.input_selection.outputs.selected_jobs == "all" || needs.input_selection.outputs.selected_jobs == "test_with_tls" + if: needs.input_selection.outputs.selected_jobs == 'all' || needs.input_selection.outputs.selected_jobs == 'test_with_tls' runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: @@ -140,7 +140,7 @@ jobs: uses: actions/checkout@v4.1.1 with: fetch-depth: 2 # needed for detecting changes - submodules: "true" + submodules: 'true' token: ${{ secrets.GITHUB_TOKEN }} - name: Pre test run @@ -153,18 +153,18 @@ jobs: python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \ --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} - echo "Task runner end to end test run completed" + echo 'Task runner end to end test run completed' - name: Post test run uses: ./.github/actions/tr_post_test_run if: ${{ always() }} with: - test_type: "tr_tls" + test_type: 'tr_tls' test_with_non_tls: name: Test without TLS needs: input_selection - if: needs.input_selection.outputs.selected_jobs == "all" || needs.input_selection.outputs.selected_jobs == "test_with_non_tls" + if: needs.input_selection.outputs.selected_jobs == 'all' || needs.input_selection.outputs.selected_jobs == 'test_with_non_tls' runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: @@ -183,7 +183,7 @@ jobs: uses: actions/checkout@v4.1.1 with: fetch-depth: 2 # needed for detecting changes - submodules: "true" + submodules: 'true' token: ${{ secrets.GITHUB_TOKEN }} - name: Pre test run @@ -196,18 +196,18 @@ jobs: python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \ --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_tls - echo "Task runner end to end test run completed" + echo 'Task runner end to end test run completed' - name: Post test run uses: ./.github/actions/tr_post_test_run if: ${{ always() }} with: - test_type: "tr_non_tls" + test_type: 'tr_non_tls' test_with_no_client_auth: name: Test without client auth needs: input_selection - if: needs.input_selection.outputs.selected_jobs == "all" || needs.input_selection.outputs.selected_jobs == "test_with_no_client_auth" + if: needs.input_selection.outputs.selected_jobs == 'all' || needs.input_selection.outputs.selected_jobs == 'test_with_no_client_auth' runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: @@ -226,7 +226,7 @@ jobs: uses: actions/checkout@v4.1.1 with: fetch-depth: 2 # needed for detecting changes - submodules: "true" + submodules: 'true' token: ${{ secrets.GITHUB_TOKEN }} - name: Pre test run @@ -239,18 +239,18 @@ jobs: python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \ --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_client_auth - echo "Task runner end to end test run completed" + echo 'Task runner end to end test run completed' - name: Post test run uses: ./.github/actions/tr_post_test_run if: ${{ always() }} with: - test_type: "tr_no_client_auth" + test_type: 'tr_no_client_auth' test_memory_logs: name: Test memory usage needs: input_selection - if: needs.input_selection.outputs.selected_jobs == "all" || needs.input_selection.outputs.selected_jobs == "test_memory_logs" + if: needs.input_selection.outputs.selected_jobs == 'all' || needs.input_selection.outputs.selected_jobs == 'test_memory_logs' runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: @@ -269,7 +269,7 @@ jobs: uses: actions/checkout@v4.1.1 with: fetch-depth: 2 # needed for detecting changes - submodules: "true" + submodules: 'true' token: ${{ secrets.GITHUB_TOKEN }} - name: Pre test run @@ -283,10 +283,10 @@ jobs: -k test_log_memory_usage_basic --model_name ${{ env.MODEL_NAME }} \ --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} \ --log_memory_usage - echo "Task runner memory logs test run completed" + echo 'Task runner memory logs test run completed' - name: Post test run uses: ./.github/actions/tr_post_test_run if: ${{ always() }} with: - test_type: "tr_tls_memory_logs" + test_type: 'tr_tls_memory_logs' From 2193391b5ef22137f62a5a32d424bee188ba2ec1 Mon Sep 17 00:00:00 2001 From: noopur Date: Thu, 23 Jan 2025 04:14:07 +0000 Subject: [PATCH 36/39] Double quotes during output assignments Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 39 ++++++++++----------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index a25a0fe4fd..be366f48ae 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -93,30 +93,29 @@ jobs: # * No client auth - models [keras_cnn_mnist] and python version [3.10] # * Memory logs - models [torch_cnn_mnist] and python version [3.10] # --------------------------------------------------------------- - echo 'jobs_to_run=${{ env.JOBS_TO_RUN }}' >> '$GITHUB_OUTPUT' + echo "jobs_to_run=${{ env.JOBS_TO_RUN }}" >> "$GITHUB_OUTPUT" - if [ '${{ env.MODEL_NAME }}' == 'all' ]; then - echo 'models_for_tls=[\'torch_cnn_mnist\', \'keras_cnn_mnist\']' >> '$GITHUB_OUTPUT' - echo 'models_for_non_tls=[\'torch_cnn_mnist\']' >> '$GITHUB_OUTPUT' - echo 'models_for_no_client_auth=[\'keras_cnn_mnist\']' >> '$GITHUB_OUTPUT' - echo 'models_for_memory_logs=[\'torch_cnn_mnist\']' >> '$GITHUB_OUTPUT' + if [ "${{ env.MODEL_NAME }}" == "all" ]; then + echo "models_for_tls=[\"torch_cnn_mnist\", \"keras_cnn_mnist\"]" >> "$GITHUB_OUTPUT" + echo "models_for_non_tls=[\"torch_cnn_mnist\"]" >> "$GITHUB_OUTPUT" + echo "models_for_no_client_auth=[\"keras_cnn_mnist\"]" >> "$GITHUB_OUTPUT" + echo "models_for_memory_logs=[\"torch_cnn_mnist\"]" >> "$GITHUB_OUTPUT" else - echo 'models_for_tls=[\'${{env.MODEL_NAME}}\']' >> '$GITHUB_OUTPUT' - echo 'models_for_non_tls=[\'${{env.MODEL_NAME}}\']' >> '$GITHUB_OUTPUT' - echo 'models_for_no_client_auth=[\'${{env.MODEL_NAME}}\']' >> '$GITHUB_OUTPUT' - echo 'models_for_memory_logs=[\'${{env.MODEL_NAME}}\']' >> '$GITHUB_OUTPUT' + echo "models_for_tls=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" + echo "models_for_non_tls=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" + echo "models_for_no_client_auth=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" + echo "models_for_memory_logs=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" fi - if [ '${{ env.PYTHON_VERSION }}' == 'all' ]; then - echo 'python_for_tls=[\'3.10\', \'3.11\', \'3.12\']' >> '$GITHUB_OUTPUT' - echo 'python_for_non_tls=[\'3.10\']' >> '$GITHUB_OUTPUT' - echo 'python_for_no_client_auth=[\'3.10\']' >> '$GITHUB_OUTPUT' - echo 'python_for_memory_logs=[\'3.10\']' >> '$GITHUB_OUTPUT' + if [ "${{ env.PYTHON_VERSION }}" == "all" ]; then + echo "python_for_tls=[\"3.10\", \"3.11\", \"3.12\"]" >> "$GITHUB_OUTPUT" + echo "python_for_non_tls=[\"3.10\"]" >> "$GITHUB_OUTPUT" + echo "python_for_no_client_auth=[\"3.10\"]" >> "$GITHUB_OUTPUT" + echo "python_for_memory_logs=[\"3.10\"]" >> "$GITHUB_OUTPUT" else - echo 'python_for_tls=[\'${{env.PYTHON_VERSION}}\']' >> '$GITHUB_OUTPUT' - echo 'python_for_non_tls=[\'${{env.PYTHON_VERSION}}\']' >> '$GITHUB_OUTPUT' - echo 'python_for_no_client_auth=[\'${{env.PYTHON_VERSION}}\']' >> '$GITHUB_OUTPUT' - echo 'python_for_memory_logs=[\'${{env.PYTHON_VERSION}}\']' >> '$GITHUB_OUTPUT' - fi + echo "python_for_tls=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" + echo "python_for_non_tls=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" + echo "python_for_no_client_auth=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" + echo "python_for_memory_logs=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" test_with_tls: name: Test with TLS From a9b94d48444275077b2afb0757e40fb9e5cde44d Mon Sep 17 00:00:00 2001 From: noopur Date: Thu, 23 Jan 2025 04:17:22 +0000 Subject: [PATCH 37/39] Modified Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 50 ++++++++++----------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index be366f48ae..508af46105 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -5,42 +5,42 @@ name: Task_Runner_E2E # Please do not modify the name as it is used in the comp on: schedule: - - cron: '0 0 * * *' # Run every day at midnight + - cron: "0 0 * * *" # Run every day at midnight workflow_dispatch: inputs: num_rounds: - description: 'Number of rounds to train' + description: "Number of rounds to train" required: false - default: '5' + default: "5" type: string num_collaborators: - description: 'Number of collaborators' + description: "Number of collaborators" required: false - default: '2' + default: "2" type: string model_name: - description: 'Model name' + description: "Model name" required: false - default: 'all' + default: "all" type: choice options: - all - torch_cnn_mnist - keras_cnn_mnist python_version: - description: 'Python version' + description: "Python version" required: false - default: 'all' + default: "all" type: choice options: - all - - '3.10' - - '3.11' - - '3.12' + - "3.10" + - "3.11" + - "3.12" jobs_to_run: - description: 'Jobs to run' + description: "Jobs to run" type: choice - default: 'all' + default: "all" options: - all - test_with_tls @@ -139,7 +139,7 @@ jobs: uses: actions/checkout@v4.1.1 with: fetch-depth: 2 # needed for detecting changes - submodules: 'true' + submodules: "true" token: ${{ secrets.GITHUB_TOKEN }} - name: Pre test run @@ -152,13 +152,13 @@ jobs: python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \ --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} - echo 'Task runner end to end test run completed' + echo "Task runner end to end test run completed" - name: Post test run uses: ./.github/actions/tr_post_test_run if: ${{ always() }} with: - test_type: 'tr_tls' + test_type: "tr_tls" test_with_non_tls: name: Test without TLS @@ -182,7 +182,7 @@ jobs: uses: actions/checkout@v4.1.1 with: fetch-depth: 2 # needed for detecting changes - submodules: 'true' + submodules: "true" token: ${{ secrets.GITHUB_TOKEN }} - name: Pre test run @@ -195,13 +195,13 @@ jobs: python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \ --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_tls - echo 'Task runner end to end test run completed' + echo "Task runner end to end test run completed" - name: Post test run uses: ./.github/actions/tr_post_test_run if: ${{ always() }} with: - test_type: 'tr_non_tls' + test_type: "tr_non_tls" test_with_no_client_auth: name: Test without client auth @@ -225,7 +225,7 @@ jobs: uses: actions/checkout@v4.1.1 with: fetch-depth: 2 # needed for detecting changes - submodules: 'true' + submodules: "true" token: ${{ secrets.GITHUB_TOKEN }} - name: Pre test run @@ -238,7 +238,7 @@ jobs: python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \ --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_client_auth - echo 'Task runner end to end test run completed' + echo "Task runner end to end test run completed" - name: Post test run uses: ./.github/actions/tr_post_test_run @@ -268,7 +268,7 @@ jobs: uses: actions/checkout@v4.1.1 with: fetch-depth: 2 # needed for detecting changes - submodules: 'true' + submodules: "true" token: ${{ secrets.GITHUB_TOKEN }} - name: Pre test run @@ -282,10 +282,10 @@ jobs: -k test_log_memory_usage_basic --model_name ${{ env.MODEL_NAME }} \ --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} \ --log_memory_usage - echo 'Task runner memory logs test run completed' + echo "Task runner memory logs test run completed" - name: Post test run uses: ./.github/actions/tr_post_test_run if: ${{ always() }} with: - test_type: 'tr_tls_memory_logs' + test_type: "tr_tls_memory_logs" From 36810f89d6884f97deb4da1710c6fe51ed48274f Mon Sep 17 00:00:00 2001 From: noopur Date: Thu, 23 Jan 2025 04:18:46 +0000 Subject: [PATCH 38/39] Modified Signed-off-by: noopur --- .github/workflows/task_runner_basic_e2e.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index 508af46105..5b64de09df 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -116,6 +116,7 @@ jobs: echo "python_for_non_tls=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" echo "python_for_no_client_auth=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" echo "python_for_memory_logs=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" + fi test_with_tls: name: Test with TLS From a435d408dea05595e2bb320d75fb0b8826ea6fd6 Mon Sep 17 00:00:00 2001 From: noopur Date: Thu, 23 Jan 2025 05:23:46 +0000 Subject: [PATCH 39/39] Check if summary file is valid Signed-off-by: noopur --- tests/end_to_end/utils/summary_helper.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/end_to_end/utils/summary_helper.py b/tests/end_to_end/utils/summary_helper.py index 0d03fb12f0..ed94d424c2 100644 --- a/tests/end_to_end/utils/summary_helper.py +++ b/tests/end_to_end/utils/summary_helper.py @@ -226,13 +226,13 @@ def _get_summary_file(): """ summary_file = os.getenv("GITHUB_STEP_SUMMARY") print(f"Summary file: {summary_file}") - # For local runs, the summary file can be .env file. - # For GitHub actions, the summary file name should start with prefix "step_summary". - # For e.g. /home/runner/work/_temp/_runner_file_commands/step_summary_ea653f69-bf1c-4a83-a115-c801b1d312a7 - if "step_summary" not in summary_file and ".env" not in summary_file: + + # Check if the fetched summary file is valid + if summary_file and os.path.isfile(summary_file): + return summary_file + else: print("Invalid summary file. Exiting...") exit(1) - return summary_file def fetch_args():