Fix compatibility issues

Dockerfile.build → Dockerfile.py27-build

@@ -1,5 +1,4 @@
 FROM python:2.7
-#FROM python:3.6
 
 # Install Docker
 ENV DOCKER_CONFIG=/tmp/

Dockerfile.py36-build

@@ -0,0 +1,14 @@
+FROM python:3.6
+
+# Install Docker
+ENV DOCKER_CONFIG=/tmp/
+RUN curl https://get.docker.com/builds/Linux/x86_64/docker-latest.tgz | tar xz -C /tmp/ \
+    && chmod +x /tmp/docker && mv -f /tmp/docker/* /usr/local/bin/
+
+# Install packages required for the tests
+COPY test-requirements.txt /tmp/test-requirements.txt
+RUN pip install --no-cache-dir -r /tmp/test-requirements.txt
+
+# Install the package requirements
+COPY requirements.txt /tmp/requirements.txt
+RUN pip install --no-cache-dir -r /tmp/requirements.txt

Makefile

@@ -1,46 +1,13 @@
-# *** WARNING: Targets are meant to run in a build container - Use skipper make ***
+all: py27 py36
 
-# Get docker api version and set the compose file version accordingly
-DOCKER_API_VERSION = $(shell python docker_test_tools/api_version.py)
-COMPOSE_FILE_VERSION = $(shell python -c 'print("2.1" if "$(DOCKER_API_VERSION)" >= "1.24" else "2")')
-DTT_COMPOSE_PATH=tests/resources/docker-compose-v$(COMPOSE_FILE_VERSION).yml
+py27:
+	# Run the internal make file using python 2.7 container
+	skipper --build-container-image=py27-build make
 
-all: coverage nose2 pytest dist/docker-test-tools-*.tar.gz
-
-flake8:
-	flake8 docker_test_tools
-
-pylint:
-	mkdir -p build/
-	PYLINTHOME=reports/ pylint -r n docker_test_tools
-
-test:
-	# Run the unittests and create a junit-xml report
-	mkdir -p build/
-	nose2 --config=tests/ut/nose2.cfg --verbose --project-directory .
-
-coverage: test
-	# Create a coverage report and validate the given threshold
-	coverage html --fail-under=69 -d build/coverage
-
-nose2:
-	mkdir -p build/
-
-	# Run the example nose2 tests - validate the package works
-	DTT_COMPOSE_PATH=$(DTT_COMPOSE_PATH) \
-	nose2 --config=tests/integration/nose2.cfg --verbose --project-directory .
-
-pytest:
-	mkdir -p build/
-
-	# Run the example pytest tests - validate the package works
-	DTT_COMPOSE_PATH=$(DTT_COMPOSE_PATH) \
-	pytest -v tests/integration/
-
-dist/docker-test-tools-*.tar.gz:
-	# Create the source distribution
-	python setup.py sdist
+py36:
+	# Run the internal make file using python 3.6 container
+	skipper --build-container-image=py36-build make
 
 clean:
 	# Clean any generated files
-	rm -rf build dist docker_test_tools.egg-info .coverage .cache
+	rm -rf build dist docker_test_tools.egg-info .coverage .cache

Makefile.internal

@@ -0,0 +1,46 @@
+# *** WARNING: Targets are meant to run in a build container - Use skipper make ***
+
+# Get docker api version and set the compose file version accordingly
+DOCKER_API_VERSION = $(shell python docker_test_tools/api_version.py)
+COMPOSE_FILE_VERSION = $(shell python -c 'print("2.1" if "$(DOCKER_API_VERSION)" >= "1.24" else "2")')
+DTT_COMPOSE_PATH=tests/resources/docker-compose-v$(COMPOSE_FILE_VERSION).yml
+
+all: coverage nose2 pytest dist/docker-test-tools-*.tar.gz
+
+flake8:
+	flake8 docker_test_tools
+
+pylint:
+	mkdir -p build/
+	PYLINTHOME=reports/ pylint -r n docker_test_tools
+
+test:
+	# Run the unittests and create a junit-xml report
+	mkdir -p build/
+	nose2 --config=tests/ut/nose2.cfg --verbose --project-directory .
+
+coverage: test
+	# Create a coverage report and validate the given threshold
+	coverage html --fail-under=69 -d build/coverage
+
+nose2:
+	mkdir -p build/
+
+	# Run the example nose2 tests - validate the package works
+	DTT_COMPOSE_PATH=$(DTT_COMPOSE_PATH) \
+	nose2 --config=tests/integration/nose2.cfg --verbose --project-directory .
+
+pytest:
+	mkdir -p build/
+
+	# Run the example pytest tests - validate the package works
+	DTT_COMPOSE_PATH=$(DTT_COMPOSE_PATH) \
+	pytest -v tests/integration/
+
+dist/docker-test-tools-*.tar.gz:
+	# Create the source distribution
+	python setup.py sdist
+
+clean:
+	# Clean any generated files
+	rm -rf build dist docker_test_tools.egg-info .coverage .cache

docker_test_tools/environment.py

@@ -26,7 +26,7 @@ def __init__(self, project_name, compose_path, log_path, reuse_containers=False)
         self.environment_variables = self._get_environment_variables()
         self.services = self.get_services()
 
-        self.encoding = self.environment_variables.get('PYTHONIOENCODING', 'ascii')
+        self.encoding = self.environment_variables.get('PYTHONIOENCODING', 'utf-8')
         self.work_dir = os.path.dirname(self.log_path)
         self.logs_collector = logs.LogCollector(
             log_path=log_path,
@@ -63,7 +63,7 @@ def get_services(self):
         except subprocess.CalledProcessError as error:
             raise RuntimeError("Failed getting environment services, reason: %s" % error.output)
 
-        return self._to_str(services_output).strip().split('\n')
+        return utils.to_str(services_output).strip().split('\n')
 
     def setup(self):
         """Sets up the environment using docker commands.
@@ -240,7 +240,7 @@ def get_container_id(self, name):
         except subprocess.CalledProcessError as error:
             raise RuntimeError("Failed getting container %s id, reason: %s" % (name, error.output))
 
-        return self._to_str(output)
+        return utils.to_str(output)
 
     def is_container_ready(self, name):
         """Return True if the container is in ready state.
@@ -250,7 +250,7 @@ def is_container_ready(self, name):
 
         :param str name: container name as it appears in the docker compose file.
         """
-        status_output = self._inspect(name, format='{{json .State}}')
+        status_output = self._inspect(name, result_format='{{json .State}}')
 
         if '"Health":' in status_output:
             is_ready = '"Status":"healthy"' in status_output
@@ -275,6 +275,7 @@ def wait_for_services(self, services=None, interval=1, timeout=60):
         """
         services = services if services else self.services
         log.info('Waiting for %s to reach the required state', services)
+        # pylint: disable=cell-var-from-loop
         return utils.run_health_checks(checks=[lambda: self.is_container_ready(name) for name in services],
                                        interval=interval, timeout=timeout)
 
@@ -387,38 +388,23 @@ def _get_environment_variables():
     def write_common_log_message(self, message):
         self.logs_collector.write(message=message)
 
-    def _inspect(self, name, format='{{json}}'):
+    def _inspect(self, name, result_format='{{json}}'):
         """
         Returns the inspect content of a container
         :param name: name of container
-        :param format: format of inspect output
+        :param result_format: format of inspect output
         """
         self.validate_service_name(name)
         log.debug("Getting %s container state", name)
         container_id = self.get_container_id(name)
         try:
             inspect_output = subprocess.check_output(
-                r"docker inspect --format='{}' {}".format(format, container_id),
+                r"docker inspect --format='{}' {}".format(result_format, container_id),
                 shell=True, stderr=subprocess.STDOUT, env=self.environment_variables
             )
 
         except subprocess.CalledProcessError as error:
             logging.warning("Failed getting container %s state, reason: %s", name, error.output)
             return ''
 
-        return self._to_str(inspect_output).strip('"\n')
-
-    @staticmethod
-    def _to_str(value):
-        """
-        For each value, return the value as string
-        For python3 compatibility
-        """
-
-        if isinstance(value, str):
-            return value
-
-        if isinstance(value, bytes):
-            return value.decode('utf-8')
-
-        raise Exception("Type {} was not converted to string".format(type(value)))
+        return utils.to_str(inspect_output).strip('"\n')

docker_test_tools/stats.py

@@ -5,6 +5,8 @@
 import subprocess
 import humanfriendly
 
+from docker_test_tools import utils
+
 log = logging.getLogger(__name__)
 
 
@@ -47,10 +49,11 @@ def stop(self):
 
     def _get_filters(self):
         """Return the docker-compose project containers."""
-        return subprocess.check_output(
+        filters_output = subprocess.check_output(
             ['docker', 'ps', '--format', '{{.Names}}',
              '--filter', "label=com.docker.compose.project={project}".format(project=self.project)]
-        ).strip().split('\n')
+        )
+        return utils.to_str(filters_output).strip().split('\n')
 
 
 class ClusterStats(object):

docker_test_tools/utils.py

@@ -63,5 +63,19 @@ def url_health_check():
     return url_health_check
 
 
+def to_str(value):
+    """Return the value as string.
+
+    For python3 compatibility
+    """
+    if isinstance(value, str):
+        return value
+
+    if isinstance(value, bytes):
+        return value.decode('utf-8')
+
+    raise Exception("Type {} was not converted to string".format(type(value)))
+
+
 # For backward compatibility
 get_curl_health_check = get_health_check

skipper.yaml

@@ -1,3 +1,5 @@
 registry: rackattack-nas.dc1:5000
-build-container-image: build
+build-container-image: py27-build
 build-container-net: example_tests-network
+make:
+  makefile: Makefile.internal