From 1766e3b5104cd3217f341f57348a43eb3b47d4fe Mon Sep 17 00:00:00 2001 From: "Chaurasiya, Payal" Date: Wed, 22 Jan 2025 23:51:31 -0800 Subject: [PATCH] Docker test mount the file Signed-off-by: Chaurasiya, Payal --- .../test_suites/memory_logs_tests.py | 5 ---- tests/end_to_end/utils/docker_helper.py | 2 +- tests/end_to_end/utils/federation_helper.py | 4 ++-- tests/end_to_end/utils/tr_common_fixtures.py | 11 +-------- tests/end_to_end/utils/tr_workspace.py | 23 ++++++++++++++++++- 5 files changed, 26 insertions(+), 19 deletions(-) diff --git a/tests/end_to_end/test_suites/memory_logs_tests.py b/tests/end_to_end/test_suites/memory_logs_tests.py index e4b8c3f128..09083c9512 100644 --- a/tests/end_to_end/test_suites/memory_logs_tests.py +++ b/tests/end_to_end/test_suites/memory_logs_tests.py @@ -68,11 +68,6 @@ def _log_memory_usage(request, fed_obj): # Verify the aggregator memory logs aggregator_memory_usage_file = constants.AGG_MEM_USAGE_JSON.format(fed_obj.workspace_path) - if request.config.test_env == "task_runner_dockerized_ws": - ssh.copy_file_from_docker( - "aggregator", f"/workspace/logs/aggregator_memory_usage.json", aggregator_memory_usage_file - ) - assert os.path.exists( aggregator_memory_usage_file ), "Aggregator memory usage file is not available" diff --git a/tests/end_to_end/utils/docker_helper.py b/tests/end_to_end/utils/docker_helper.py index d8178652c0..f69e494cbb 100644 --- a/tests/end_to_end/utils/docker_helper.py +++ b/tests/end_to_end/utils/docker_helper.py @@ -94,7 +94,7 @@ def start_docker_container( docker_participant_path = mount_mapping[0].split(":")[1] else: local_participant_path = os.path.join(local_bind_path, container_name, "workspace") - docker_participant_path = f"{workspace_path}/{container_name}/workspace" + docker_participant_path = "/workspace" volumes = { local_participant_path: {"bind": docker_participant_path, "mode": "rw"}, diff --git a/tests/end_to_end/utils/federation_helper.py b/tests/end_to_end/utils/federation_helper.py index 8f968a3120..b61c53b3bc 100644 --- a/tests/end_to_end/utils/federation_helper.py +++ b/tests/end_to_end/utils/federation_helper.py @@ -134,7 +134,7 @@ def _create_tarball(collaborator_name, data_file_path, local_bind_path, add_data local_bind_path, collaborator_name ) client_cert_entries = "" - tarfiles = f"cert_col_{collaborator_name}.tar plan/data.yaml" + tarfiles = f"cert_{collaborator_name}.tar plan/data.yaml" # If TLS is enabled, client certificates and signed certificates are also included if use_tls: client_cert_entries = [ @@ -740,7 +740,7 @@ def download_data(collaborators, model_name, local_bind_path): log.info("Downloading the data for the model. This will take some time to complete based on the data size ..") try: command = ["python", constants.DATA_SETUP_FILE, str(len(collaborators))] - subprocess.run(command, cwd=local_bind_path, check=True) + subprocess.run(command, cwd=local_bind_path, check=True) # nosec B603 except Exception: raise ex.DataSetupException(f"Failed to download data for {model_name}") diff --git a/tests/end_to_end/utils/tr_common_fixtures.py b/tests/end_to_end/utils/tr_common_fixtures.py index 54bd6e3339..f0e2a47257 100644 --- a/tests/end_to_end/utils/tr_common_fixtures.py +++ b/tests/end_to_end/utils/tr_common_fixtures.py @@ -2,17 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 import pytest -pass -pass -import logging - -import tests.end_to_end.utils.constants as constants -import tests.end_to_end.utils.federation_helper as fh -import tests.end_to_end.utils.ssh_helper as ssh -from tests.end_to_end.models import aggregator as agg_model, model_owner as mo_model -from tests.end_to_end.utils.tr_workspace import create_tr_workspace, create_tr_workspace_dws -log = logging.getLogger(__name__) +from tests.end_to_end.utils.tr_workspace import create_tr_workspace, create_tr_workspace_dws @pytest.fixture(scope="function") diff --git a/tests/end_to_end/utils/tr_workspace.py b/tests/end_to_end/utils/tr_workspace.py index f5ba90f9ab..5a0b27b75d 100644 --- a/tests/end_to_end/utils/tr_workspace.py +++ b/tests/end_to_end/utils/tr_workspace.py @@ -115,6 +115,27 @@ def create_tr_workspace(request, eval_scope=False): def create_tr_workspace_dws(request, eval_scope=False): + """ + Create a task runner workspace for distributed workload simulation. + + This function sets up the necessary environment and configurations for a + task runner workspace, including model owner and collaborator workspaces, + plan modifications, and Docker container setups. + + Args: + request: A request object containing configuration details. + eval_scope (bool, optional): If True, sets up the evaluation scope for + a single round. Defaults to False. + + Returns: + federation_details: An object containing details about the federation + setup, including model owner, aggregator, collaborators, workspace path, + and local bind path. + + Raises: + Exception: If there is an error during the creation of the tar for the + aggregator. + """ collaborators = [] executor = concurrent.futures.ThreadPoolExecutor() @@ -210,7 +231,7 @@ def create_tr_workspace_dws(request, eval_scope=False): # Note: In case of multiple machines setup, scp this tar to the other machine(s) return_code, output, error = ssh.run_command( - f"tar -cf cert_agg.tar plan cert save", work_dir=local_agg_ws_path + f"tar -cf cert_aggregator.tar plan cert save", work_dir=local_agg_ws_path ) if return_code != 0: raise Exception(f"Failed to create tar for aggregator: {error}")