From 95fd6386dacddf169b7ce78cf334d0aff7245517 Mon Sep 17 00:00:00 2001 From: Margaret Sy Date: Thu, 7 Dec 2017 12:43:28 -0800 Subject: [PATCH 01/17] try_to_output_unbuffered takes optional arg for stdout --- dcos_launch/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dcos_launch/util.py b/dcos_launch/util.py index 9b50737a..7fc1ec6a 100644 --- a/dcos_launch/util.py +++ b/dcos_launch/util.py @@ -126,7 +126,7 @@ def test(self, args: list, env_dict: dict, test_host=None, test_port=22) -> int: return try_to_output_unbuffered(self.config, test_host, pytest_cmd, test_port) -def try_to_output_unbuffered(info, test_host: str, bash_cmd: str, port: int) -> int: +def try_to_output_unbuffered(info, test_host: str, bash_cmd: str, port: int, stdout=sys.stdout.buffer) -> int: """ Tries to run a command and directly output to STDOUT Args: @@ -140,7 +140,7 @@ def try_to_output_unbuffered(info, test_host: str, bash_cmd: str, port: int) -> ssh_client = dcos_test_utils.ssh_client.SshClient(info['ssh_user'], info['ssh_private_key']) ssh_client.wait_for_ssh_connection(test_host, port=port) try: - ssh_client.command(test_host, ['bash', '-c', bash_cmd], port=port, stdout=sys.stdout.buffer) + ssh_client.command(test_host, ['bash', '-c', bash_cmd], port=port, stdout=stdout) except subprocess.CalledProcessError as e: log.exception('Test run failed!') return e.returncode From ae71e6b57a6069fab0c915694c7c571db6670de1 Mon Sep 17 00:00:00 2001 From: Margaret Sy Date: Thu, 7 Dec 2017 12:51:26 -0800 Subject: [PATCH 02/17] optionally take output handle for test --- dcos_launch/util.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dcos_launch/util.py b/dcos_launch/util.py index 7fc1ec6a..fc333232 100644 --- a/dcos_launch/util.py +++ b/dcos_launch/util.py @@ -80,12 +80,13 @@ def wait(self): def delete(self): raise NotImplementedError() - def test(self, args: list, env_dict: dict, test_host=None, test_port=22) -> int: + def test(self, args: list, env_dict: dict, test_host=None, test_port=22, output=None) -> int: """ Connects to master host with SSH and then run the internal integration test Args: args: a list of args that will follow the py.test command env_dict: the env to use during the test + output: optional handle to write test output to. If None, defaults to stdout. """ if args is None: args = list() @@ -123,10 +124,10 @@ def test(self, args: list, env_dict: dict, test_host=None, test_port=22) -> int: test_host = details['masters'][0]['public_ip'] if ':' in test_host: test_host, test_port = test_host.split(':') - return try_to_output_unbuffered(self.config, test_host, pytest_cmd, test_port) + return try_to_output_unbuffered(self.config, test_host, pytest_cmd, test_port, stdout=output) -def try_to_output_unbuffered(info, test_host: str, bash_cmd: str, port: int, stdout=sys.stdout.buffer) -> int: +def try_to_output_unbuffered(info, test_host: str, bash_cmd: str, port: int, stdout=None) -> int: """ Tries to run a command and directly output to STDOUT Args: @@ -140,6 +141,8 @@ def try_to_output_unbuffered(info, test_host: str, bash_cmd: str, port: int, std ssh_client = dcos_test_utils.ssh_client.SshClient(info['ssh_user'], info['ssh_private_key']) ssh_client.wait_for_ssh_connection(test_host, port=port) try: + if stdout is None: + stdout = sys.stdout.buffer ssh_client.command(test_host, ['bash', '-c', bash_cmd], port=port, stdout=stdout) except subprocess.CalledProcessError as e: log.exception('Test run failed!') From 1825c494970192d92bb0585076b326241101ae90 Mon Sep 17 00:00:00 2001 From: Margaret Sy Date: Thu, 7 Dec 2017 15:30:14 -0800 Subject: [PATCH 03/17] test pytest option to write to file --- test/conftest.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/conftest.py b/test/conftest.py index c8832aa7..ce4cd50e 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -44,7 +44,7 @@ def mocked_test_runner(monkeypatch): def mock_ssh_client(monkeypatch): # monkeypatch.setattr(dcos_test_utils.ssh_client, 'Tunnelled', MockTunnelled) monkeypatch.setattr(dcos_test_utils.ssh_client, 'open_tunnel', mocked_context) - monkeypatch.setattr(dcos_test_utils.ssh_client.SshClient, 'command', stub(b'')) + monkeypatch.setattr(dcos_test_utils.ssh_client.SshClient, 'command', stub(b'Hooray, everything worked')) monkeypatch.setattr(dcos_test_utils.ssh_client.SshClient, 'get_home_dir', stub(b'')) # need to nullify platforms.onprem monkeypatch.setattr(dcos_launch.platforms.onprem, 'prepare_bootstrap', stub('foo')) @@ -271,6 +271,13 @@ def check_success(capsys, tmpdir, config_path): launcher.wait() launcher.describe() launcher.test([], {}) + # check that option to write test output to file works + test_log_path = str(tmpdir.join('test_logs.txt')) + with open(test_log_path, 'wr') as log_handle: + launcher.test([], {}, output=log_handle) + logs = log_handle.read() + assert logs == 'Hooray, everything worked' + launcher.delete() info_path = str(tmpdir.join('my_specific_info.json')) # test non-default name From 37237be8cf6549a52988f64eb980250341caf418 Mon Sep 17 00:00:00 2001 From: Margaret Sy Date: Thu, 7 Dec 2017 15:46:18 -0800 Subject: [PATCH 04/17] use correct file mode --- test/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/conftest.py b/test/conftest.py index ce4cd50e..75315c1e 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -273,7 +273,7 @@ def check_success(capsys, tmpdir, config_path): launcher.test([], {}) # check that option to write test output to file works test_log_path = str(tmpdir.join('test_logs.txt')) - with open(test_log_path, 'wr') as log_handle: + with open(test_log_path, 'w+') as log_handle: launcher.test([], {}, output=log_handle) logs = log_handle.read() assert logs == 'Hooray, everything worked' From 558f12d4da58af6711e1cf532cc897203dca1504 Mon Sep 17 00:00:00 2001 From: Margaret Sy Date: Thu, 7 Dec 2017 15:57:43 -0800 Subject: [PATCH 05/17] try separate handle for reading --- test/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 75315c1e..ed889022 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -273,10 +273,10 @@ def check_success(capsys, tmpdir, config_path): launcher.test([], {}) # check that option to write test output to file works test_log_path = str(tmpdir.join('test_logs.txt')) - with open(test_log_path, 'w+') as log_handle: + with open(test_log_path, 'w') as log_handle: launcher.test([], {}, output=log_handle) - logs = log_handle.read() - assert logs == 'Hooray, everything worked' + with open(test_log_path, 'r') as log_handle: + assert log_handle.read() == 'Hooray, everything worked' launcher.delete() From 45fde9f0410bed70e773f84af5e5521f7d640272 Mon Sep 17 00:00:00 2001 From: Margaret Sy Date: Thu, 7 Dec 2017 16:07:25 -0800 Subject: [PATCH 06/17] debugging --- dcos_launch/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dcos_launch/util.py b/dcos_launch/util.py index fc333232..6716873e 100644 --- a/dcos_launch/util.py +++ b/dcos_launch/util.py @@ -140,6 +140,7 @@ def try_to_output_unbuffered(info, test_host: str, bash_cmd: str, port: int, std """ ssh_client = dcos_test_utils.ssh_client.SshClient(info['ssh_user'], info['ssh_private_key']) ssh_client.wait_for_ssh_connection(test_host, port=port) + print('stdout in try_to_output_unbuffered: {}'.format(stdout)) try: if stdout is None: stdout = sys.stdout.buffer From 019d1f754f638f6f6b9642e64366334058b82799 Mon Sep 17 00:00:00 2001 From: Margaret Sy Date: Thu, 7 Dec 2017 16:27:04 -0800 Subject: [PATCH 07/17] verbose pytest --- test/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/conftest.py b/test/conftest.py index ed889022..9019d80f 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -274,7 +274,7 @@ def check_success(capsys, tmpdir, config_path): # check that option to write test output to file works test_log_path = str(tmpdir.join('test_logs.txt')) with open(test_log_path, 'w') as log_handle: - launcher.test([], {}, output=log_handle) + launcher.test(['-s', '-vv'], {}, output=log_handle) with open(test_log_path, 'r') as log_handle: assert log_handle.read() == 'Hooray, everything worked' From fa9dc52715d7e7a7f356892c1b54ffa8ec2f5b35 Mon Sep 17 00:00:00 2001 From: Margaret Sy Date: Thu, 7 Dec 2017 16:30:04 -0800 Subject: [PATCH 08/17] log not print --- dcos_launch/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcos_launch/util.py b/dcos_launch/util.py index 6716873e..af65463d 100644 --- a/dcos_launch/util.py +++ b/dcos_launch/util.py @@ -140,7 +140,7 @@ def try_to_output_unbuffered(info, test_host: str, bash_cmd: str, port: int, std """ ssh_client = dcos_test_utils.ssh_client.SshClient(info['ssh_user'], info['ssh_private_key']) ssh_client.wait_for_ssh_connection(test_host, port=port) - print('stdout in try_to_output_unbuffered: {}'.format(stdout)) + log.info('stdout in try_to_output_unbuffered: {}'.format(stdout)) try: if stdout is None: stdout = sys.stdout.buffer From 4566708d603828b50786f9d3b63bd2a6b7bd33a7 Mon Sep 17 00:00:00 2001 From: Margaret Sy Date: Thu, 7 Dec 2017 16:47:37 -0800 Subject: [PATCH 09/17] change mocked_test_runner not mock_ssh_client --- test/conftest.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 9019d80f..38f176bb 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -37,14 +37,15 @@ def mocked_context(*args, **kwargs): @pytest.fixture def mocked_test_runner(monkeypatch): - monkeypatch.setattr(dcos_launch.util, 'try_to_output_unbuffered', stub(0)) + monkeypatch.setattr(dcos_test_utils.ssh_client.SshClient, 'command', stub(b'Hooray, everything worked')) + # monkeypatch.setattr(dcos_launch.util, 'try_to_output_unbuffered', stub(0)) @pytest.fixture def mock_ssh_client(monkeypatch): # monkeypatch.setattr(dcos_test_utils.ssh_client, 'Tunnelled', MockTunnelled) monkeypatch.setattr(dcos_test_utils.ssh_client, 'open_tunnel', mocked_context) - monkeypatch.setattr(dcos_test_utils.ssh_client.SshClient, 'command', stub(b'Hooray, everything worked')) + monkeypatch.setattr(dcos_test_utils.ssh_client.SshClient, 'command', stub(b'')) monkeypatch.setattr(dcos_test_utils.ssh_client.SshClient, 'get_home_dir', stub(b'')) # need to nullify platforms.onprem monkeypatch.setattr(dcos_launch.platforms.onprem, 'prepare_bootstrap', stub('foo')) From 0830af06be931890bf3d1307cd57af48270432ab Mon Sep 17 00:00:00 2001 From: Margaret Sy Date: Fri, 8 Dec 2017 10:21:22 -0800 Subject: [PATCH 10/17] Revert "change mocked_test_runner not mock_ssh_client" This reverts commit 9b10fd1ac331656390b1797b26ad01928afa88cd. --- test/conftest.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 38f176bb..9019d80f 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -37,15 +37,14 @@ def mocked_context(*args, **kwargs): @pytest.fixture def mocked_test_runner(monkeypatch): - monkeypatch.setattr(dcos_test_utils.ssh_client.SshClient, 'command', stub(b'Hooray, everything worked')) - # monkeypatch.setattr(dcos_launch.util, 'try_to_output_unbuffered', stub(0)) + monkeypatch.setattr(dcos_launch.util, 'try_to_output_unbuffered', stub(0)) @pytest.fixture def mock_ssh_client(monkeypatch): # monkeypatch.setattr(dcos_test_utils.ssh_client, 'Tunnelled', MockTunnelled) monkeypatch.setattr(dcos_test_utils.ssh_client, 'open_tunnel', mocked_context) - monkeypatch.setattr(dcos_test_utils.ssh_client.SshClient, 'command', stub(b'')) + monkeypatch.setattr(dcos_test_utils.ssh_client.SshClient, 'command', stub(b'Hooray, everything worked')) monkeypatch.setattr(dcos_test_utils.ssh_client.SshClient, 'get_home_dir', stub(b'')) # need to nullify platforms.onprem monkeypatch.setattr(dcos_launch.platforms.onprem, 'prepare_bootstrap', stub('foo')) From 4fb6a5d82cc9d857efcdd7e202e0c5e3b1f3310e Mon Sep 17 00:00:00 2001 From: Margaret Sy Date: Fri, 8 Dec 2017 10:21:26 -0800 Subject: [PATCH 11/17] Revert "log not print" This reverts commit cc0108554406448c3afc19ca52d40e4bc3180b83. --- dcos_launch/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcos_launch/util.py b/dcos_launch/util.py index af65463d..6716873e 100644 --- a/dcos_launch/util.py +++ b/dcos_launch/util.py @@ -140,7 +140,7 @@ def try_to_output_unbuffered(info, test_host: str, bash_cmd: str, port: int, std """ ssh_client = dcos_test_utils.ssh_client.SshClient(info['ssh_user'], info['ssh_private_key']) ssh_client.wait_for_ssh_connection(test_host, port=port) - log.info('stdout in try_to_output_unbuffered: {}'.format(stdout)) + print('stdout in try_to_output_unbuffered: {}'.format(stdout)) try: if stdout is None: stdout = sys.stdout.buffer From 3bf1d17eff6b74c27f25b1f688da0473bbe812c1 Mon Sep 17 00:00:00 2001 From: Margaret Sy Date: Fri, 8 Dec 2017 10:21:27 -0800 Subject: [PATCH 12/17] Revert "verbose pytest" This reverts commit b61f36836b8e44d5733effd9bf90745ec4b90058. --- test/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/conftest.py b/test/conftest.py index 9019d80f..ed889022 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -274,7 +274,7 @@ def check_success(capsys, tmpdir, config_path): # check that option to write test output to file works test_log_path = str(tmpdir.join('test_logs.txt')) with open(test_log_path, 'w') as log_handle: - launcher.test(['-s', '-vv'], {}, output=log_handle) + launcher.test([], {}, output=log_handle) with open(test_log_path, 'r') as log_handle: assert log_handle.read() == 'Hooray, everything worked' From bbb463eafece6cc185a1b581354df41c549387aa Mon Sep 17 00:00:00 2001 From: Margaret Sy Date: Fri, 8 Dec 2017 10:21:29 -0800 Subject: [PATCH 13/17] Revert "debugging" This reverts commit 477f57afc86ae534eede3b7e1d05cffd67eba24e. --- dcos_launch/util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dcos_launch/util.py b/dcos_launch/util.py index 6716873e..fc333232 100644 --- a/dcos_launch/util.py +++ b/dcos_launch/util.py @@ -140,7 +140,6 @@ def try_to_output_unbuffered(info, test_host: str, bash_cmd: str, port: int, std """ ssh_client = dcos_test_utils.ssh_client.SshClient(info['ssh_user'], info['ssh_private_key']) ssh_client.wait_for_ssh_connection(test_host, port=port) - print('stdout in try_to_output_unbuffered: {}'.format(stdout)) try: if stdout is None: stdout = sys.stdout.buffer From a67b3f8ec1b4cca59f3630d9f4ae570455268b60 Mon Sep 17 00:00:00 2001 From: Margaret Sy Date: Fri, 8 Dec 2017 10:21:30 -0800 Subject: [PATCH 14/17] Revert "try separate handle for reading" This reverts commit 07f5c6d5c8892c2f50d9015b826cb2bb17dade45. --- test/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index ed889022..75315c1e 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -273,10 +273,10 @@ def check_success(capsys, tmpdir, config_path): launcher.test([], {}) # check that option to write test output to file works test_log_path = str(tmpdir.join('test_logs.txt')) - with open(test_log_path, 'w') as log_handle: + with open(test_log_path, 'w+') as log_handle: launcher.test([], {}, output=log_handle) - with open(test_log_path, 'r') as log_handle: - assert log_handle.read() == 'Hooray, everything worked' + logs = log_handle.read() + assert logs == 'Hooray, everything worked' launcher.delete() From 8de268f6d6349425e77021b897c2e3df681b6d60 Mon Sep 17 00:00:00 2001 From: Margaret Sy Date: Fri, 8 Dec 2017 10:21:36 -0800 Subject: [PATCH 15/17] Revert "use correct file mode" This reverts commit 2143be875eeb0ab2a925edea00f4e04403c8fac3. --- test/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/conftest.py b/test/conftest.py index 75315c1e..ce4cd50e 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -273,7 +273,7 @@ def check_success(capsys, tmpdir, config_path): launcher.test([], {}) # check that option to write test output to file works test_log_path = str(tmpdir.join('test_logs.txt')) - with open(test_log_path, 'w+') as log_handle: + with open(test_log_path, 'wr') as log_handle: launcher.test([], {}, output=log_handle) logs = log_handle.read() assert logs == 'Hooray, everything worked' From 424adff490935fbdfb057a9222da933b33b54bdb Mon Sep 17 00:00:00 2001 From: Margaret Sy Date: Fri, 8 Dec 2017 10:21:45 -0800 Subject: [PATCH 16/17] Revert "test pytest option to write to file" This reverts commit 7af767a80f7c6519dee15a3e39c22b013eff17da. --- test/conftest.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index ce4cd50e..c8832aa7 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -44,7 +44,7 @@ def mocked_test_runner(monkeypatch): def mock_ssh_client(monkeypatch): # monkeypatch.setattr(dcos_test_utils.ssh_client, 'Tunnelled', MockTunnelled) monkeypatch.setattr(dcos_test_utils.ssh_client, 'open_tunnel', mocked_context) - monkeypatch.setattr(dcos_test_utils.ssh_client.SshClient, 'command', stub(b'Hooray, everything worked')) + monkeypatch.setattr(dcos_test_utils.ssh_client.SshClient, 'command', stub(b'')) monkeypatch.setattr(dcos_test_utils.ssh_client.SshClient, 'get_home_dir', stub(b'')) # need to nullify platforms.onprem monkeypatch.setattr(dcos_launch.platforms.onprem, 'prepare_bootstrap', stub('foo')) @@ -271,13 +271,6 @@ def check_success(capsys, tmpdir, config_path): launcher.wait() launcher.describe() launcher.test([], {}) - # check that option to write test output to file works - test_log_path = str(tmpdir.join('test_logs.txt')) - with open(test_log_path, 'wr') as log_handle: - launcher.test([], {}, output=log_handle) - logs = log_handle.read() - assert logs == 'Hooray, everything worked' - launcher.delete() info_path = str(tmpdir.join('my_specific_info.json')) # test non-default name From 33ff9feb889c228942339c35be0dc6ea5601f98f Mon Sep 17 00:00:00 2001 From: Margaret Sy Date: Mon, 18 Dec 2017 17:31:15 -0800 Subject: [PATCH 17/17] acl option, default to public-read --- dcos_launch/platforms/aws.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dcos_launch/platforms/aws.py b/dcos_launch/platforms/aws.py index 75c0cac3..44806edd 100644 --- a/dcos_launch/platforms/aws.py +++ b/dcos_launch/platforms/aws.py @@ -239,6 +239,13 @@ def get_auto_scaling_instances(self, asg_physical_resource_id): AutoScalingGroupNames=[asg_physical_resource_id]) ['AutoScalingGroups'] for i in asg['Instances']] + @retry_boto_rate_limits + def put_s3(self, bucket, keyname, file, acl='public-read'): + return self.resource('s3').Bucket(bucket_id).put_object( + Key=keyname, + Body=file, + ACL=acl) + @retry_boto_rate_limits def empty_and_delete_bucket(self, bucket_id): """ Buckets must be empty to be deleted. Additionally, there is no high-level