diff --git a/afctl/_version.py b/afctl/_version.py index c469247..8f07417 100644 --- a/afctl/_version.py +++ b/afctl/_version.py @@ -495,7 +495,7 @@ def get_versions(): # versionfile_source is the relative path from the top of the source # tree (where the .git directory might live) to this file. Invert # this to find the root from __file__. - for i in cfg.versionfile_source.split('/'): + for i in cfg.versionfile_source.split(os.path.sep): root = os.path.dirname(root) except NameError: return {"version": "0+unknown", "full-revisionid": None, diff --git a/afctl/parser_helpers.py b/afctl/parser_helpers.py index b41b24d..9762118 100644 --- a/afctl/parser_helpers.py +++ b/afctl/parser_helpers.py @@ -1,8 +1,11 @@ from afctl.utils import Utility import os -import subprocess +import shutil from termcolor import colored from afctl.exceptions import AfctlParserException +import git + +SEP = os.path.sep class ParserHelpers(): @@ -10,7 +13,7 @@ class ParserHelpers(): def get_project_file_names(name): try: pwd = os.getcwd() - main_dir = pwd if name == '.' else os.path.join(pwd, name.lstrip('/').rstrip('/')) + main_dir = pwd if name == '.' else os.path.join(pwd, name.lstrip(SEP).rstrip(SEP)) project_name = os.path.basename(main_dir) config_dir = Utility.CONSTS['config_dir'] config_file = Utility.project_config(project_name) @@ -35,11 +38,10 @@ def get_project_file_names(name): @staticmethod def add_git_config(files): try: - origin = subprocess.run(['git', '--git-dir={}'.format(os.path.join(files['main_dir'], '.git')), 'config', - '--get', 'remote.origin.url'],stdout=subprocess.PIPE) - origin = origin.stdout.decode('utf-8')[:-1] - if origin == '': - subprocess.run(['git', 'init', files['main_dir']]) + try: + origin = git.Repo(os.path.join(files['main_dir'])) + except git.exc.InvalidGitRepositoryError: + origin = git.Repo.init(os.path.join(files['main_dir'])) print(colored("Git origin is not set for this repository. Run 'afctl config global -o '", 'yellow')) else: print("Updating git origin.") @@ -71,9 +73,8 @@ def generate_all(files): #STEP - 2: create config file ParserHelpers.generate_config_file(files) - subprocess.run(['cp', '{}/templates/gitignore.txt'.format(os.path.dirname(os.path.abspath(__file__))), - sub_file['.gitignore']]) - + shutil.copyfile('{}/templates/gitignore.txt'.format(os.path.dirname(os.path.abspath(__file__))), + sub_file['.gitignore']) except Exception as e: raise AfctlParserException(e) @@ -81,9 +82,8 @@ def generate_all(files): @staticmethod def generate_config_file(files): try: - subprocess.run(['cp', '{}/plugins/deployments/deployment_config.yml'.format(os.path.dirname(os.path.abspath(__file__))), - files['config_file']]) - + shutil.copyfile('{}/plugins/deployments/deployment_config.yml'.format(os.path.dirname(os.path.abspath(__file__))), + files['config_file']) ParserHelpers.add_git_config(files) except Exception as e: diff --git a/afctl/parsers.py b/afctl/parsers.py index 6d45bda..456e255 100644 --- a/afctl/parsers.py +++ b/afctl/parsers.py @@ -5,7 +5,6 @@ from afctl import __version__ from afctl.utils import Utility from afctl.exceptions import AfctlParserException -import subprocess from afctl.plugins.deployments.deployment_config import DeploymentConfig from afctl.parser_helpers import ParserHelpers from termcolor import colored @@ -123,9 +122,10 @@ def generate(cls, args): elif args.type == "module": path = "{}/{}/dags/{}".format(project_path, project_name, args.n) test_path = "{}/tests/{}".format(project_path, args.n) - mod_val = subprocess.call(['mkdir', path]) - test_val = subprocess.call(['mkdir', test_path]) - if mod_val != 0 or test_val != 0: + try: + os.makedirs(path, exist_ok=True) + os.makedirs(test_path, exist_ok=True) + except: cls.parser.error(colored("Unable to generate.", 'red')) print(colored("Generated successfully.", 'green')) @@ -286,4 +286,4 @@ def act_on_configs(cls, args, project_name): Utility.print_file(Utility.project_config(project_name)) except Exception as e: - AfctlParserException(e) \ No newline at end of file + AfctlParserException(e) diff --git a/afctl/plugins/deployments/docker/deployment_config.py b/afctl/plugins/deployments/docker/deployment_config.py index 51970ce..99023d8 100644 --- a/afctl/plugins/deployments/docker/deployment_config.py +++ b/afctl/plugins/deployments/docker/deployment_config.py @@ -4,7 +4,7 @@ import os import yaml from afctl.utils import Utility -import subprocess +import docker # Yaml Structure # deployment: @@ -47,8 +47,10 @@ def deploy_project(cls, args, config_file): with open(Utility.project_config(config_file)) as file: config = yaml.full_load(file) - val = subprocess.call(['docker', 'info']) - if val != 0: + try: + client = docker.from_env() + client.info() + except: return True, "Docker is not running. Please start docker." if args.d: diff --git a/afctl/plugins/deployments/qubole/qubole_utils.py b/afctl/plugins/deployments/qubole/qubole_utils.py index 656bda5..14af18f 100644 --- a/afctl/plugins/deployments/qubole/qubole_utils.py +++ b/afctl/plugins/deployments/qubole/qubole_utils.py @@ -1,25 +1,17 @@ -import subprocess from afctl.exceptions import AfctlDeploymentException from mako.template import Template from qds_sdk.qubole import Qubole from qds_sdk.commands import ShellCommand from urllib.parse import urlparse - +import git class QuboleUtils(): @staticmethod def fetch_latest_commit(origin, branch): try: - commit = subprocess.run(['git', 'ls-remote', origin, 'refs/heads/{}'.format(branch), '|', 'cut', '-f', '1'], - stdout=subprocess.PIPE) - commit = commit.stdout.decode('utf-8') - - if commit == '': - return None - - return commit.split('\t')[0] - + repo = git.Repo('.') + return repo.remotes.origin.refs[0].commit.hexsha except Exception as e: raise AfctlDeploymentException(e) @@ -95,9 +87,8 @@ def generate_configs(configs, args): env = "{}/api".format(config['env'].rstrip('/')) cluster = config['cluster'] token = config['token'] - branch = subprocess.run(['git', 'symbolic-ref', '--short', 'HEAD'], stdout=subprocess.PIPE).stdout.decode( - 'utf-8')[:-1] - + repo = git.Repo('.') + branch = repo.active_branch.name return { 'name': name, 'env': env, diff --git a/afctl/templates/dag_template.py b/afctl/templates/dag_template.py index cca00fa..eb36e5b 100644 --- a/afctl/templates/dag_template.py +++ b/afctl/templates/dag_template.py @@ -8,8 +8,8 @@ def dag_template(name, config_name): default_args = { 'owner': '${config_name}', +'start_date': datetime.now() - timedelta(days=1), # 'depends_on_past': , -# 'start_date': , # 'email': , # 'email_on_failure': , # 'email_on_retry': , @@ -23,4 +23,4 @@ def dag_template(name, config_name): """ ) - return template.render_unicode(name=name, config_name=config_name) \ No newline at end of file + return template.render_unicode(name=name, config_name=config_name) diff --git a/afctl/tests/deployment_tests/test_local_deployment.py b/afctl/tests/deployment_tests/test_local_deployment.py index 217aa4f..e20547e 100644 --- a/afctl/tests/deployment_tests/test_local_deployment.py +++ b/afctl/tests/deployment_tests/test_local_deployment.py @@ -1,7 +1,9 @@ from afctl.plugins.deployments.docker.deployment_config import DockerDeploymentConfig from afctl.tests.utils import clean_up, PROJECT_NAME, PROJECT_CONFIG_DIR import pytest -import os, subprocess +import os, pathlib, tempfile + +TMP = tempfile.gettempdir() class TestLocalDeployment: @@ -10,12 +12,12 @@ class TestLocalDeployment: def create_project(self): clean_up(PROJECT_NAME) clean_up(PROJECT_CONFIG_DIR) - main_dir = os.path.join('/tmp', PROJECT_NAME) - subprocess.run(['mkdir', main_dir]) - subprocess.run(['mkdir', PROJECT_CONFIG_DIR]) - subprocess.run(['mkdir', os.path.join(main_dir, 'deployments')]) + main_dir = os.path.join(TMP, PROJECT_NAME) + os.makedirs(main_dir, exist_ok=True) + os.makedirs(PROJECT_CONFIG_DIR, exist_ok=True) + os.makedirs(os.path.join(main_dir, 'deployments'), exist_ok=True) config_file = "{}.yml".format(os.path.join(PROJECT_CONFIG_DIR, PROJECT_NAME)) - subprocess.run(['touch', config_file]) + pathlib.Path(config_file).touch() config_file_content = """ global: airflow_version: @@ -51,4 +53,4 @@ def test_docker_compose_generation(self, create_project): current_output = open(config_file).read() expected_output = expected_output.replace(" ", "") current_output = current_output.replace(" ", "") - assert expected_output == current_output \ No newline at end of file + assert expected_output == current_output diff --git a/afctl/tests/parser_tests/test_parser_utils.py b/afctl/tests/parser_tests/test_parser_utils.py index 230944a..469f441 100644 --- a/afctl/tests/parser_tests/test_parser_utils.py +++ b/afctl/tests/parser_tests/test_parser_utils.py @@ -1,13 +1,17 @@ from afctl.utils import Utility import pytest -import os, subprocess +import os, pathlib, shutil, tempfile from afctl.tests.utils import create_path_and_clean, PROJECT_NAME, PROJECT_CONFIG_DIR, clean_up +SEP = os.path.sep +TMP = tempfile.gettempdir() +TMP_NO_SEP = TMP.replace(SEP,'') + class TestUtils: @pytest.fixture(scope='function') def clean_tmp_dir(self): - parent = ['/tmp'] + parent = [TMP] child = ['one', 'two', 'three'] create_path_and_clean(parent, child) yield @@ -15,26 +19,26 @@ def clean_tmp_dir(self): # create_dirs def test_create_dir(self, clean_tmp_dir): - parent = ['/tmp'] + parent = [TMP] child = ['one', 'two', 'three'] dirs = Utility.create_dirs(parent, child) - assert dirs['one'] == '/tmp/one' - assert os.path.exists(dirs['one']) is True - assert dirs['two'] == '/tmp/two' - assert os.path.exists(dirs['two']) is True - assert dirs['three'] == '/tmp/three' - assert os.path.exists(dirs['three']) is True + assert dirs['one'] == SEP.join([TMP, 'one']) + assert os.path.isdir(dirs['one']) + assert dirs['two'] == SEP.join([TMP, 'two']) + assert os.path.isdir(dirs['two']) + assert dirs['three'] == SEP.join([TMP, 'three']) + assert os.path.isdir(dirs['three']) # create_files def test_create_files(self, clean_tmp_dir): - parent = ['/tmp'] + parent = [TMP] child = ['one', 'two', 'three'] dirs = Utility.create_files(parent, child) - assert dirs['one'] == '/tmp/one' + assert dirs['one'] == SEP.join([TMP, 'one']) assert os.path.exists(dirs['one']) is True - assert dirs['two'] == '/tmp/two' + assert dirs['two'] == SEP.join([TMP, 'two']) assert os.path.exists(dirs['two']) is True - assert dirs['three'] == '/tmp/three' + assert dirs['three'] == SEP.join([TMP, 'three']) assert os.path.exists(dirs['three']) is True # project_config @@ -47,7 +51,7 @@ def test_return_project_config_file(self): # generate_dag_template def test_generate_dag_template(self): project_name = "tes_project" - path = "/tmp" + path = TMP dag = "test" Utility.generate_dag_template(project_name, dag, path) expected_output = """ @@ -56,8 +60,8 @@ def test_generate_dag_template(self): default_args = { 'owner': 'tes_project', +'start_date': datetime.now() - timedelta(days=1), # 'depends_on_past': , -# 'start_date': , # 'email': , # 'email_on_failure': , # 'email_on_retry': , @@ -76,26 +80,26 @@ def test_generate_dag_template(self): @pytest.fixture(scope='function') def create_project(self): - path = '/tmp/one/two/three' - subprocess.run(['mkdir', '-p', path]) - file_path = '/tmp/one/two/.afctl_project' - subprocess.run(['touch', file_path]) + path = os.path.sep.join([TMP, 'one', 'two', 'three']) + os.makedirs(path, exist_ok=True) + file_path = os.path.sep.join([TMP, 'one', 'two', '.afctl_project']) + pathlib.Path(file_path).touch() yield - subprocess.run(['rm', '-rf', path]) + shutil.rmtree(path) # find_project def test_find_project(self, create_project): - path = '/tmp/one/two/three' + path = os.path.sep.join([TMP, 'one', 'two', 'three']) project = Utility.find_project(path) assert project[0] == 'two' - assert project[1] == '/tmp/one/two' + assert project[1] == os.path.sep.join([TMP, 'one', 'two']) @pytest.fixture(scope='function') def create_config_file(self): - subprocess.run(['mkdir', PROJECT_CONFIG_DIR]) + os.mkdir(PROJECT_CONFIG_DIR) file_path = os.path.join(PROJECT_CONFIG_DIR, PROJECT_NAME)+'.yml' - subprocess.run(['touch', file_path]) + pathlib.Path(file_path).touch() yml_template = """ parent: child1: @@ -160,4 +164,4 @@ def test_add_and_update_configs(self, create_config_file): current_output = open(config_file).read() expected_output = expected_output.replace(" ", "") current_output = current_output.replace(" ", "") - assert expected_output == current_output \ No newline at end of file + assert expected_output == current_output diff --git a/afctl/tests/utils.py b/afctl/tests/utils.py index 305fc1e..3c70a70 100644 --- a/afctl/tests/utils.py +++ b/afctl/tests/utils.py @@ -1,5 +1,5 @@ import os -import subprocess +import shutil import itertools PROJECT_NAME = 'test_project' @@ -20,7 +20,10 @@ def __init__(self, type=None, origin=None, token=None, version=None, env=None, c def clean_up(project_file): if os.path.exists(project_file): - subprocess.run(['rm', '-rf', project_file]) + if os.path.isdir(project_file): + shutil.rmtree(project_file, ignore_errors=True) + else: + os.unlink(project_file) def check_paths(parent, child): diff --git a/afctl/utils.py b/afctl/utils.py index 876dc1f..cbb3572 100644 --- a/afctl/utils.py +++ b/afctl/utils.py @@ -1,11 +1,13 @@ import os import itertools -import subprocess +import pathlib import yaml from afctl.exceptions import AfctlUtilsException from afctl.templates.dag_template import dag_template from termcolor import colored +SEP = os.path.sep + class Utility(): CONSTS = { @@ -17,11 +19,12 @@ def create_dirs(parent, child): try: dirs = {} for dir1, dir2 in itertools.product(parent, child): - if not os.path.exists(os.path.join(dir1, dir2)): - os.mkdir(os.path.join(dir1, dir2)) + target = os.path.join(dir1, dir2) + if not os.path.exists(target): + os.makedirs(target) else: print("{} already exists. Skipping.".format(dir2)) - dirs[dir2] = os.path.join(dir1, dir2) + dirs[dir2] = target return dirs except Exception as e: raise AfctlUtilsException(e) @@ -33,7 +36,7 @@ def create_files(parent, child): files = {} for dir1, dir2 in itertools.product(parent, child): if not os.path.exists(os.path.join(dir1, dir2)): - subprocess.run(['touch', os.path.join(dir1, dir2)]) + pathlib.Path(os.path.join(dir1, dir2)).touch() else: print("{} already exists. Skipping.".format(dir2)) files[dir2] = os.path.join(dir1, dir2) @@ -65,7 +68,11 @@ def project_config(file): @staticmethod def print_file(file): - subprocess.call(['cat', file]) + try: + with open(file) as fh: + print(fh.read()) + finally: + fh.close() @staticmethod def update_config(file, config): @@ -100,11 +107,11 @@ def crawl_config(crawler, config): @staticmethod def find_project(pwd): - dirs = pwd.lstrip('/').split('/') - for i in range(len(dirs)+1): - path = '/'.join(dirs[:i]) - if os.path.exists(os.path.join('/'+path, '.afctl_project')): - return [dirs[i-1], '/'+path] + dirs = pwd.split(SEP) + for i in range(len(dirs), 0, -1): + path = SEP.join(dirs[:i]) + if os.path.exists(os.path.join(path, '.afctl_project')): + return [dirs[i-1], path] return None @staticmethod diff --git a/requirements.txt b/requirements.txt index de99664..2c07d03 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,6 @@ versioneer==0.18 mako==1.1.0 qds-sdk==1.13.2 termcolor -pytest \ No newline at end of file +pytest +gitpython +docker