diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 636dc76..5ba1fa1 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -22,7 +22,7 @@ jobs: pull_number: context.issue.number }) const isTitleValid = /^\[#\d+\] /.test(pr.data.title) - const isDescriptionValid = /([Ff]ix(es|ed)?|[Cc]lose(s|d)?|[Rr]esolve(s|d)?|[Pp]art [Oo]f) \(.*\)\[.*\]/.test(pr.data.body) + const isDescriptionValid = /([Ff]ix(es|ed)?|[Cc]lose(s|d)?|[Rr]esolve(s|d)?|[Pp]art [Oo]f) \[.*\]\(.*\)/.test(pr.data.body) if (isTitleValid && isDescriptionValid) { return } diff --git a/.gitignore b/.gitignore index ce2d119..034d630 100644 --- a/.gitignore +++ b/.gitignore @@ -130,8 +130,7 @@ dmypy.json # Pyre type checker .pyre/ -my_secret.py -test_hpcc.py +tests/my_secret.py # Miscellaneous folders Dummy/ \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..b72f84b --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,7 @@ +repos: +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: "v0.5.0" + hooks: + - id: ruff + args: ["--fix"] + - id: ruff-format \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 87561a2..75ca21f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,6 +4,26 @@ We value your interest in contributing to `PyHPCC.` Thank you +## Project Structure +``` +. +└── pyhpcc/ + ├── .github # contains build, release, test and other gh actions + ├── docs/ # contains files for documentation + ├── examples/ # contains starter examples + ├── src/ + │ ├── pyhpcc/ + │ │ ├── handlers/ # contains thor and roxie handler + │ │ └── models/ # contains classes auth, workunit submit + │ └── tests/ + │ ├── models/ + │ ├── test_files/ # contains resource files needed for testing + │ └── hanlders/ + ├── pyproject.toml # Project config + ├── CONTRIBUTING.md + └── README.md +``` + ## Set up the repository locally. ## Prerequisites Before starting to develop, make sure you install the following software: @@ -25,6 +45,15 @@ To install the dependencies, run the following command, which downloads the depe poetry install ``` +## How to run tests +Since ecl client tools aren't installed in the GitHub runner, some tests are skipped in the github runner. + +Some tests will fail if `ecl client tools` aren't installed. + +``` +pytest run # Run in project root +``` + ## Linting and Formatting PyHPCC uses [Ruff](https://docs.astral.sh/ruff/) as its formatter and linter. @@ -56,8 +85,4 @@ The base branch is the main repo's main branch. - PR name: copy-and-paste the relevant issue name and include the issue number in front in square brackets, e.g. `[#1020] Make bash_runcommand in WorkUnitSubmit class configurable ` - PR description: mention the issue number in this format: Fixes #1020. Doing so will automatically close the related issue once the PR is merged. - Please Ensure that "Allow edits from maintainers" is ticked. -- Please describe the changes you have made in your branch and how they resolve the issue. - - - - +- Please describe the changes you have made in your branch and how they resolve the issue. \ No newline at end of file diff --git a/README.md b/README.md index 48e162c..e572567 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ To use PyHPCC, you need these
1. [Python3](https://www.python.org/downloads/) 2. [ECL Client Tools](https://hpccsystems.com/download/): Select your operating systems to download client tools + + Download the latest stable build from releases in GitHub.
``` bash @@ -36,5 +38,4 @@ Contributions to PyHPCC are welcomed and encouraged.
For specific contribution guidelines, please take a look at [CONTRIBUTING.md](CONTRIBUTING.md). -For more information about the package, please refer to the detailed documentation - https://upgraded-bassoon-daa9d010.pages.github.io/build/html/index.html - +For more information about the package, please refer to the detailed documentation - https://upgraded-bassoon-daa9d010.pages.github.io/build/html/index.html \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index cb83dd0..12573da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ elementpath = "^3.0.2" # XPath selectors for XML Data structures [tool.poetry.group.dev.dependencies] pytest = "^7.1.3" pytest-mock = "^3.0.2" +pre-commit="^3.7.1" [tool.poetry.group.coverage.dependencies] coverage = "^7.5.3" diff --git a/src/pyhpcc/command_config.py b/src/pyhpcc/command_config.py new file mode 100644 index 0000000..2b444a2 --- /dev/null +++ b/src/pyhpcc/command_config.py @@ -0,0 +1,202 @@ +import logging + +from pyhpcc.config import ( + CLUSTER_OPTION, + COMPILE_OPTIONS, + JOB_NAME_OPTION, + LIMIT_OPTION, + MASKED_PASSWORD, + OUTPUT_FILE_OPTION, + PASSWORD_OPTIONS, + PORT_OPTION, + RUN_AUTH_OPTIONS, + RUN_OPTIONS, + SERVER_OPTIONS, + USER_OPTIONS, +) +from pyhpcc.errors import CompileConfigException, RunConfigException +from pyhpcc.models.auth import Auth + +log = logging.getLogger(__name__) + + +class CompileConfig(object): + """ + Class for eclcc option configuration + + Attributes: + ---------- + options: + Dictionary of keys (option), values (value) + + Methods: + ------- + validate_options: + Validate if the compiler options are supported or not + + set_output_file: + Set name of output file (default a.out if linking to + + create_compile_bash_command: + Generate the eclcc command for the given options and input_file + + get_option: + Retrieves the compile config option + + + """ + + def __init__(self, options: dict): + self.options = options.copy() + + def validate_options(self): + """Validate if the compiler options are supported or not""" + invalid_options = [] + for option in self.options: + if option not in COMPILE_OPTIONS and not ( + option.startswith("-R") or option.startswith("-f") + ): + invalid_options.append(option) + if len(invalid_options) > 0: + raise CompileConfigException(str(invalid_options)) + + def set_output_file(self, output_file): + """Set name of output file (default a.out if linking to""" + self.options[OUTPUT_FILE_OPTION] = output_file + + def create_compile_bash_command(self, input_file): + """Generate the eclcc command for the given options and input_file""" + self.validate_options() + eclcc_command = "eclcc" + for key, value in self.options.items(): + if value is bool: + eclcc_command += f" {key}" + else: + eclcc_command += f" {key} {value}" + eclcc_command = f"{eclcc_command} {input_file}" + return eclcc_command + + def get_option(self, option): + """Get the option available for the option""" + return self.options[option] + + +class RunConfig(object): + """ + Class for ecl run option configuration + + Attributes: + ---------- + options: + Dictionary of keys (option), values (value) + + Methods: + ------- + validate_options: + Validate if the compiler options are supported or not + + create_run_bash_command: + Generate the ecl command for the given options and target_file + + set_auth_params: + Set the auth parameters from the auth object passed + + set_target: + Specify the job name for the workunit + + set_limit: + Sets the result limit for the query + + set_server: + Set IP of server running ecl services (eclwatch) + + set_port: + Set ECL services port + + set_username: + Set username for accessing ecl services + + set_password: + Set password for accessing ecl services + + get_option: + Retrieves the run config option + + """ + + def __init__(self, options: dict): + self.options = options.copy() + + def validate_options(self): + """Validate if the runtime options are supported or not""" + invalid_options = set() + for option in self.options: + if option not in RUN_OPTIONS and not ( + option.startswith("-X") or option.startswith("-f") + ): + invalid_options.add(option) + if len(invalid_options) > 0: + log.error("Entered invalid options %s", str(invalid_options)) + raise RunConfigException( + f"Invalid options not supported by pyhpcc {str(invalid_options)}" + ) + + def create_run_bash_command(self, target_file, password_mask=False): + """Generate the ecl command for the given options and target_file""" + self.validate_options() + ecl_command = "ecl run" + params = "" + for key, value in self.options.items(): + if value is bool: + params += f" {key}" + else: + if key in PASSWORD_OPTIONS and password_mask: + params += f" {key} {MASKED_PASSWORD}" + else: + params += f" {key} {value}" + ecl_command = f"{ecl_command} {target_file}{params}" + log.info(ecl_command) + return ecl_command + + def set_auth_params(self, auth: Auth): + """Set the auth parameters from the auth object passed""" + for option in list(self.options): + if option in RUN_AUTH_OPTIONS: + log.warning(f"Overriding option {option} with Auth object parameters") + del self.options[option] + self.set_server(auth.ip) + self.set_port(auth.port) + self.set_username(auth.oauth[0]) + self.set_password(auth.oauth[1]) + + def set_target(self, target): + """Set the target""" + self.options[CLUSTER_OPTION] = target + + def set_job_name(self, job_name): + """Specify the job name for the workunit""" + self.options[JOB_NAME_OPTION] = job_name + + def set_limit(self, limit): + """Sets the result limit for the query""" + self.options[LIMIT_OPTION] = limit + + def set_server(self, server): + """Set IP of server running ecl services (eclwatch)""" + self.options[SERVER_OPTIONS[0]] = server + + def set_port(self, port): + """Set ECL services port""" + self.options[PORT_OPTION] = port + + def set_username(self, username): + """Set username for accessing ecl services""" + self.options[USER_OPTIONS[0]] = username + + def set_password(self, password): + """Set password for accessing ecl services""" + self.options[PASSWORD_OPTIONS[0]] = password + + def get_option(self, option): + """Get the option available for the option""" + return self.options[option] diff --git a/src/pyhpcc/config.py b/src/pyhpcc/config.py index cafbba7..9b90e82 100644 --- a/src/pyhpcc/config.py +++ b/src/pyhpcc/config.py @@ -26,3 +26,126 @@ "paused": 16, "statesize": 17, } + +DEFAULT_COMPILE_OPTIONS = {"-platform": "thor", "-wu": bool, "-E": bool} +DEFUALT_RUN_OPTIONS = {} + +COMMAND = "command" +CLUSTER_OPTION = "--target" +JOB_NAME_OPTION = "--job-name" +LIMIT_OPTION = "--limit" +DEFAULT_LIMIT = 100 +USER_OPTIONS = ["-u", "--username"] +PASSWORD_OPTIONS = ["-pw", "--password"] +SERVER_OPTIONS = ["-s", "--s"] +PORT_OPTION = "--port" +OUTPUT_FILE_OPTION = "-o" +VERBOSE_OPTIONS = [ + "-v", + "--verbose", +] +MASKED_PASSWORD = "*****" +RUN_AUTH_OPTIONS = {*USER_OPTIONS, *PASSWORD_OPTIONS, *SERVER_OPTIONS, PORT_OPTION} + +COMPILE_OPTIONS = { + "-I", + "-L", + "-manifest", + "--main", + "-syntax", + "-platform", + "-E", + "-q", + "-qa", + "-wu", + "-S", + "-g", + "--debug", + "-Wc", + "-xx", + "-shared", + "-dfs", + "-scope", + "-cluster", + "-user", + "-password", + "-checkDirty", + "--cleanrepos", + "--cleaninvalidrepos", + "--fetchrepos", + "-help", + "--help", + "--logfile", + "--metacache", + "--nosourcepath", + "-specs", + "--updaterepos", + *VERBOSE_OPTIONS, + "-wxxxx", + "--version", + CLUSTER_OPTION, + OUTPUT_FILE_OPTION, +} + +RUN_OPTIONS = { + "--job-name", + "--input", + "-in", + "--wait", + "--poll", + "--exception-level", + "--protect", + "--main", + "--snapshot", + "--ecl-only", + "--limit", + "-Dname", + "-I", + "-L", + "-manifest", + "-g", + "debug", + "--checkDirty", + "--cleanrepos", + "--cleaninvalidrepos", + "--fetchrepos", + "--updaterepos", + "--help", + *SERVER_OPTIONS, + "--ssl", + "-ssl", + "--accept-self-signed", + "--cert", + "--key", + "--cacert", + PORT_OPTION, + *USER_OPTIONS, + *PASSWORD_OPTIONS, + "--wait-connect", + "--wait-read", + CLUSTER_OPTION, + JOB_NAME_OPTION, + *VERBOSE_OPTIONS, +} + + +RUN_ERROR_MSG_PATTERN = [ + "401: Unauthorized", + "Error checking ESP configuration", + "Bad host name/ip:", +] + +COMPILE_ERROR_MIDDLE_PATTERN = [ + r"\(\d+,\d+\): error C([0-9]){3,6}", +] + +COMPILE_ERROR_PATTERN = [ + "Error: ", + "Failed to compile ", +] +FAILED_STATUS = {"failed", "aborted", "aborting"} + +WUID_PATTERN = "^(wuid): (W[0-9]+-[0-9]+)$" +WUID = "wuid" +STATE_PATTERN = f"^(state): ({"|".join(WORKUNIT_STATE_MAP.keys())})$" +STATE = "state" diff --git a/src/pyhpcc/errors.py b/src/pyhpcc/errors.py index 8d4bb86..cf1e0a4 100644 --- a/src/pyhpcc/errors.py +++ b/src/pyhpcc/errors.py @@ -17,8 +17,8 @@ def __init__(self, message: str): super().__init__(self.message) -class TypeError(Error): - """Exception raised for type errors. +class HPCCException(Error): + """Exception raised for HPCC errors. Attributes: message: @@ -30,8 +30,21 @@ def __init__(self, message: str): super().__init__(self.message) -class HPCCException(Error): - """Exception raised for HPCC errors. +class CompileConfigException(Error): + """Exception raised for CompileConfig Errors. + + Attributes: + message: + The error message + """ + + def __init__(self, message: str): + self.message = message + super().__init__(self.message) + + +class RunConfigException(Error): + """Exception raised for RunConfig Errors Attributes: message: diff --git a/src/pyhpcc/handlers/roxie_handler.py b/src/pyhpcc/handlers/roxie_handler.py index c205186..ec73e34 100644 --- a/src/pyhpcc/handlers/roxie_handler.py +++ b/src/pyhpcc/handlers/roxie_handler.py @@ -1,7 +1,7 @@ import logging import pyhpcc.config as conf -from pyhpcc.errors import HPCCAuthenticationError, TypeError +from pyhpcc.errors import HPCCAuthenticationError from pyhpcc.utils import convert_arg_to_utf8_str log = logging.getLogger(__name__) diff --git a/src/pyhpcc/handlers/thor_handler.py b/src/pyhpcc/handlers/thor_handler.py index c09efef..99faf92 100644 --- a/src/pyhpcc/handlers/thor_handler.py +++ b/src/pyhpcc/handlers/thor_handler.py @@ -1,7 +1,7 @@ import logging import pyhpcc.config as conf -from pyhpcc.errors import HPCCAuthenticationError, TypeError +from pyhpcc.errors import HPCCAuthenticationError from pyhpcc.utils import convert_arg_to_utf8_str log = logging.getLogger(__name__) @@ -134,7 +134,9 @@ def execute(self): # self.api.cached_result = True # return result - full_url = self.api.auth.get_url() + self.path + "." + self.response_type + full_url = ( + self.api.auth.get_url() + "/" + self.path + "." + self.response_type + ) # Debugging if conf.DEBUG: diff --git a/src/pyhpcc/models/workunit_submit.py b/src/pyhpcc/models/workunit_submit.py index 2b27a71..3368341 100644 --- a/src/pyhpcc/models/workunit_submit.py +++ b/src/pyhpcc/models/workunit_submit.py @@ -2,12 +2,17 @@ import logging import os import subprocess +from collections import Counter import requests import pyhpcc.config as conf import pyhpcc.utils as utils -from pyhpcc.errors import HPCCException +from pyhpcc.command_config import CompileConfig, RunConfig +from pyhpcc.errors import HPCCException, RunConfigException +from pyhpcc.models.hpcc import HPCC + +log = logging.getLogger(__name__) class WorkunitSubmit(object): @@ -60,12 +65,20 @@ class WorkunitSubmit(object): run_workunit: Legacy function to run the workunit + + configure_run_config: + Creates run config from given options """ - def __init__(self, hpcc, cluster1="", cluster2=""): - self.hpcc = hpcc - self.cluster1 = cluster1 - self.cluster2 = cluster2 + def __init__( + self, + hpcc: HPCC, + clusters: tuple, + ): + if len(clusters) == 0: + raise ValueError("Minimum one cluster should be specified") + self.hpcc: HPCC = hpcc + self.clusters: tuple = clusters self.stateid = conf.WORKUNIT_STATE_MAP def write_file(self, query_text, folder, job_name): @@ -101,15 +114,15 @@ def write_file(self, query_text, folder, job_name): except Exception as e: raise HPCCException("Could not write file: " + str(e)) - def get_bash_command(self, file_name, repository): + def get_bash_command(self, file_name, compile_config: CompileConfig): """Get the bash command to compile the ecl file Parameters ---------- file_name: The name of the ecl file - repository: - Git repository to use + config: + CompileConfig object Returns ------- @@ -124,15 +137,19 @@ def get_bash_command(self, file_name, repository): A generic exception """ try: - output_file = utils.create_compile_file_name(file_name) - bash_command = utils.create_compile_bash_command( - repository, output_file, file_name - ) + if conf.OUTPUT_FILE_OPTION not in compile_config.options: + output_file = utils.create_compile_file_name(file_name) + compile_config.set_output_file(output_file) + else: + output_file = compile_config.get_option(conf.OUTPUT_FILE_OPTION) + log.info(compile_config.options) + bash_command = compile_config.create_compile_bash_command(file_name) + log.info(bash_command) return bash_command, output_file except Exception as e: raise HPCCException("Could not get bash command: " + str(e)) - def get_work_load(self): + def get_least_active_cluster(self): """Get the workload for the given two HPCC clusters Parameters @@ -151,23 +168,41 @@ def get_work_load(self): A generic exception """ try: + if len(self.clusters) == 1: + return self.clusters[0] payload = {"SortBy": "Name", "Descending": 1} + return self.get_cluster_from_response(self.hpcc.activity(**payload).json()) + except Exception as e: + raise HPCCException("Could not get workload: " + str(e)) + + def get_cluster_from_response(self, resp): + """Extract the cluster from the Activity API Response - resp = self.hpcc.activity(**payload).json() - len1 = 0 - len2 = 0 - if "Running" in list(resp["ActivityResponse"].keys()): - workunits = resp["ActivityResponse"]["Running"]["ActiveWorkunit"] - for workunit in workunits: - if workunit["TargetClusterName"] == self.cluster1: - len1 = len1 + 1 - if workunit["TargetClusterName"] == self.cluster2: - len2 = len2 + 1 + Parameters + ---------- + self: + The object pointer + resp: + Activity API response - return len1, len2 + Returns + ------- + str + Cluster with least activity - except Exception as e: - raise HPCCException("Could not get workload: " + str(e)) + Raises + ------ + HPCCException: + A generic exception + """ + cluster_activity = Counter(self.clusters) + if "Running" in list(resp["ActivityResponse"].keys()): + workunits = resp["ActivityResponse"]["Running"]["ActiveWorkunit"] + for workunit in workunits: + cluster = workunit["TargetClusterName"] + if cluster in cluster_activity: + cluster_activity[cluster] -= 1 + return cluster_activity.most_common(1)[0][0] def create_file_name(self, query_text, working_folder, job_name): """Create a filename for the ecl file @@ -197,15 +232,15 @@ def create_file_name(self, query_text, working_folder, job_name): except Exception as e: raise HPCCException("Could not create file name: " + str(e)) - def bash_compile(self, file_name, git_repository): + def bash_compile(self, file_name: str, options: dict = None): """Compile the ecl file Parameters ---------- file_name: The name of the ecl file - git_repository: - Git repository to use + options: + dictionary of eclcc compiler options Returns ------- @@ -220,16 +255,21 @@ def bash_compile(self, file_name, git_repository): A generic exception """ try: - bash_command, output_file = self.get_bash_command(file_name, git_repository) + if options is None: + options = conf.DEFAULT_COMPILE_OPTIONS + compile_config = CompileConfig(options) + bash_command, output_file = self.get_bash_command(file_name, compile_config) + print(bash_command) process = subprocess.Popen( bash_command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) output, error = process.communicate() - return output, output_file + parsed_output = utils.parse_bash_compile_output(output) + return parsed_output, output_file except Exception as e: raise HPCCException("Could not compile: " + str(e)) - def bash_run(self, compiled_file, cluster): + def bash_run(self, compiled_file, options: dict = None): """Run the compiled ecl file Parameters @@ -250,33 +290,44 @@ def bash_run(self, compiled_file, cluster): A generic exception """ try: - # Select the cluster to run the query on - if cluster == "": - len1, len2 = self.get_work_load() - if len2 > len1: - cluster = self.cluster1 - else: - cluster = self.cluster2 - - self.job_name = self.job_name.replace(" ", "_") - bash_command = utils.create_run_bash_command( - compiled_file, - cluster, - self.hpcc.auth.ip, - self.hpcc.auth.port, - self.hpcc.auth.oauth[0], - self.hpcc.auth.oauth[1], - self.job_name, - ) + run_config = self.configure_run_config(options) + bash_command = run_config.create_run_bash_command(compiled_file) process = subprocess.Popen( bash_command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) output, error = process.communicate() - - return output, error + return utils.parse_bash_run_output(output) + except RunConfigException: + raise except Exception as e: raise HPCCException("Could not run: " + str(e)) + def configure_run_config(self, options: dict) -> RunConfig: + """Creates run config from given options + + Parameters + ---------- + options: + dict of run config options + + Returns + ------- + run_config: + Returns RunConfig object configured with additional options + """ + if options is None: + options = conf.DEFUALT_RUN_OPTIONS + run_config = RunConfig(options) + if conf.CLUSTER_OPTION not in run_config.options: + run_config.set_target(self.get_least_active_cluster()) + if conf.JOB_NAME_OPTION not in run_config.options: + self.job_name = self.job_name.replace(" ", "_") + run_config.set_job_name(self.job_name) + if conf.LIMIT_OPTION not in run_config.options: + run_config.set_limit(conf.DEFAULT_LIMIT) + run_config.set_auth_params(self.hpcc.auth) + return run_config + def compile_workunit(self, wuid, cluster=""): """Legacy function to compile a workunit - use bash_compile instead @@ -288,11 +339,8 @@ def compile_workunit(self, wuid, cluster=""): The HPCC cluster to run the query on """ if cluster == "": - len1, len2 = self.get_work_load() - if len2 > len1: - cluster = self.cluster1 - else: - cluster = self.cluster2 + cluster = self.get_least_active_cluster() + self.hpcc.wu_submit(Wuid=wuid, Cluster=cluster) try: w3 = self.hpcc.wu_wait_compiled(Wuid=wuid) @@ -325,12 +373,7 @@ def create_workunit( The data to pass to the query """ if cluster_orig == "": - len1, len2 = self.get_work_load() - - if len2 > len1: - cluster_orig = self.cluster1 - else: - cluster_orig = self.cluster2 + cluster_orig = self.get_least_active_cluster() if query_text is None: data = {"QueryText": data} kwargs = {"data": data} @@ -407,12 +450,7 @@ def run_workunit(self, wuid, cluster=""): The HPCC cluster to run the query on """ if cluster == "": - len1, len2 = self.get_work_load() - - if len2 > len1: - cluster = self.cluster1 - else: - cluster = self.cluster2 + cluster = self.get_least_active_cluster() try: w4 = self.hpcc.wu_run(Wuid=wuid, Cluster=cluster, Variables=[]) except requests.exceptions.Timeout: diff --git a/src/pyhpcc/utils.py b/src/pyhpcc/utils.py index 7717070..5af0c91 100644 --- a/src/pyhpcc/utils.py +++ b/src/pyhpcc/utils.py @@ -1,3 +1,4 @@ +import re import sys import xml.etree.ElementTree as ET @@ -8,6 +9,16 @@ from StringIO import StringIO else: from io import StringIO +from pyhpcc.config import ( + COMPILE_ERROR_MIDDLE_PATTERN, + COMPILE_ERROR_PATTERN, + FAILED_STATUS, + RUN_ERROR_MSG_PATTERN, + STATE, + STATE_PATTERN, + WUID, + WUID_PATTERN, +) from pyhpcc.errors import HPCCException """ @@ -49,37 +60,6 @@ def convert_arg_to_utf8_str(arg): raise e -def create_compile_bash_command(repository, output_file, file_name): - """ - Create a bash command to compile a file. - - Parameters - ---------- - repository : str - The repository to compile the file from. - output_file : str - The output file to write the compiled code to. - file_name : str - The filename to compile. - - Returns - ------- - str - The bash command to compile the file. - - Raises - ------ - HPCCException - A generic exception. - """ - try: - return """ eclcc -legacy -I {0} -platform=thor -E -o {1} {2} -wu""".format( - repository, output_file, file_name - ) - except HPCCException as e: - raise e - - def create_compile_file_name(file_name): """ Create a compiled file name from a filename. @@ -105,47 +85,6 @@ def create_compile_file_name(file_name): raise e -def create_run_bash_command( - compiled_file, cluster, ip, port, user_name, password, job_name -): - """ - Create a bash command to run a compiled file. - - Parameters - ---------- - compiled_file : str - The compiled file to run. - cluster : str - The cluster to run the compiled file on. - ip : str - The ip address of the HPCC cluster. - port : str - The port of the HPCC cluster. - user_name : str - The username to use to connect to the HPCC cluster. - password : str - The password to use to connect to the HPCC cluster. - job_name : str - The name of the job to run. - - Returns - ------- - str - The bash command to run a compiled file. - - Raises - ------ - HPCCException - A generic exception. - """ - try: - return """ecl run {0} --limit=100 --wait=0 --target={1} --server={2} --ssl --port={3} -u={4} -pw={5} --name={6} -v""".format( - compiled_file, cluster, ip, port, user_name, password, job_name - ) - except HPCCException as e: - raise e - - def get_graph_skew(response): """ Get the graph skew from the response of a WUInfo call. @@ -476,3 +415,104 @@ def despray_file(hpcc, query_text, cluster, jobn): except HPCCException as e: raise e + + +RUN_UNWANTED_PATTERNS = [ + r"jsocket\([0-9]+,[0-9]+\) ", + "deploying", + "Deployed", + "Running", + "Using eclcc path ", +] + + +def parse_bash_run_output(response: bytes): + """ + Parse raw run output to user-friendly JSON format + + Parameters + ---------- + response : binary string + ecl run output + + Returns + ------- + response: dict + parsed run output + """ + parsed_response = {} + wu_info = {WUID: None, STATE: None} + misc_info = {"message": []} + error = {} + messages = [] + error_messages = [] + response = response.decode() + raw_output = response + response = response.split("\n") + wuid_found = False + state_found = False + for line in response: + line = line.strip() + if line == "" or re.match("|".join(RUN_UNWANTED_PATTERNS), line, re.IGNORECASE): + continue + if not wuid_found: + if wuid_match := re.match(WUID_PATTERN, line): + wu_info[WUID] = wuid_match.group(2) + continue + if not state_found: + if state_match := re.match(STATE_PATTERN, line): + wu_info[STATE] = state_match.group(2) + continue + if re.match("|".join(RUN_ERROR_MSG_PATTERN), line, re.IGNORECASE): + error_messages.append(line) + continue + messages.append(line) + if ( + (state_found and wu_info[STATE] in FAILED_STATUS) or wu_info[STATE] is None + ) and len(error_messages) > 0: + error["message"] = error_messages + parsed_response.update(error=error) + misc_info["message"] = messages + parsed_response.update(raw_output=raw_output) + parsed_response.update(wu_info=wu_info) + parsed_response.update(misc_info=misc_info) + return parsed_response + + +def parse_bash_compile_output(response): + """ + Parse raw compiler output to user-friendly JSON format + + Parameters + ---------- + response : binary string + eclcc compiler output + + Returns + ------- + response: dict + parsed compiler output + """ + errors = [] + parsed_response = {} + response = response.decode() + raw_output = response + response = response.split("\n") + for line in response: + line = line.strip() + if line == "": + continue + + line = line.strip() + if re.match("|".join(COMPILE_ERROR_PATTERN), line) or re.search( + "|".join(COMPILE_ERROR_MIDDLE_PATTERN), line + ): + errors.append(line) + continue + if len(errors) == 0: + parsed_response["status"] = "success" + else: + parsed_response["status"] = "error" + parsed_response["errors"] = errors + parsed_response["raw_output"] = raw_output + return parsed_response diff --git a/tests/config.py b/tests/conftest.py similarity index 92% rename from tests/config.py rename to tests/conftest.py index 35debb1..cc9a214 100644 --- a/tests/config.py +++ b/tests/conftest.py @@ -13,13 +13,14 @@ class DUMMY_SECRETS: DUMMY_HPCC_PORT = 0 WUID = "" DEBUG = False + ENV = "LOCAL" try: import my_secret except Exception: my_secret = DUMMY_SECRETS - +ENV_VAR = "ENV" ## HPCC Config HPCC_USERNAME = os.environ.get("HPCC_USERNAME") or my_secret.HPCC_USERNAME HPCC_PASSWORD = os.environ.get("HPCC_PASSWORD") or my_secret.HPCC_PASSWORD @@ -31,3 +32,4 @@ class DUMMY_SECRETS: DUMMY_HPCC_HOST = os.environ.get("DUMMY_HPCC_HOST") or my_secret.DUMMY_HPCC_HOST DUMMY_HPCC_PORT = os.environ.get("DUMMY_HPCC_PORT") or my_secret.DUMMY_HPCC_PORT WUID = os.environ.get("WUID") or my_secret.WUID +ENV = os.environ.get(ENV_VAR) or my_secret.ENV diff --git a/tests/models/test_auth.py b/tests/models/test_auth.py index 409f598..81df690 100644 --- a/tests/models/test_auth.py +++ b/tests/models/test_auth.py @@ -1,102 +1,95 @@ # Unit tests to test the authentication module -import unittest - -import config +import conftest +import pytest from pyhpcc.errors import HPCCAuthenticationError from pyhpcc.models.auth import Auth +HPCC_HOST = conftest.HPCC_HOST +HPCC_PORT = conftest.HPCC_PORT +HPCC_USERNAME = conftest.HPCC_USERNAME +HPCC_PASSWORD = conftest.HPCC_PASSWORD +HPCC_PROTOCOL = conftest.HPCC_PROTOCOL +DUMMY_USERNAME = conftest.DUMMY_USERNAME +DUMMY_PASSWORD = conftest.DUMMY_PASSWORD +DUMMY_HPCC_HOST = conftest.DUMMY_HPCC_HOST +DUMMY_HPCC_PORT = conftest.DUMMY_HPCC_PORT + + +# Test the get_username method +def test_get_username(): + test = Auth( + HPCC_HOST, + HPCC_PORT, + HPCC_USERNAME, + HPCC_PASSWORD, + True, + HPCC_PROTOCOL, + ) + test.get_username() == HPCC_USERNAME -class TestAuth(unittest.TestCase): - HPCC_HOST = config.HPCC_HOST - HPCC_PORT = config.HPCC_PORT - HPCC_USERNAME = config.HPCC_USERNAME - HPCC_PASSWORD = config.HPCC_PASSWORD - DUMMY_USERNAME = config.DUMMY_USERNAME - DUMMY_PASSWORD = config.DUMMY_PASSWORD - DUMMY_HPCC_HOST = config.DUMMY_HPCC_HOST - DUMMY_HPCC_PORT = config.DUMMY_HPCC_PORT - # Test the get_url method - def test_get_url(self): - test = Auth( - self.HPCC_HOST, - self.HPCC_PORT, - self.HPCC_USERNAME, - self.HPCC_PASSWORD, - True, - "https", - ) - self.assertEqual( - test.get_url(), "https://" + self.HPCC_HOST + ":" + str(self.HPCC_PORT) - ) +# Test the get_verified method +def test_get_verified(): + test = Auth( + HPCC_HOST, + HPCC_PORT, + HPCC_USERNAME, + HPCC_PASSWORD, + True, + HPCC_PROTOCOL, + ) + assert test.get_verified() - # Test the get_username method - def test_get_username(self): - test = Auth( - self.HPCC_HOST, - self.HPCC_PORT, - self.HPCC_USERNAME, - self.HPCC_PASSWORD, - True, - "https", - ) - self.assertEqual(test.get_username(), self.HPCC_USERNAME) - # Test the get_verified method - def test_get_verified(self): - test = Auth( - self.HPCC_HOST, - self.HPCC_PORT, - self.HPCC_USERNAME, - self.HPCC_PASSWORD, - True, - "https", - ) - self.assertTrue(test.get_verified()) +# Test the get_verified method with an invalid username and password +def test_get_verified_invalid_username_password(): + test = Auth( + HPCC_HOST, + HPCC_PORT, + DUMMY_USERNAME, + DUMMY_PASSWORD, + True, + HPCC_PROTOCOL, + ) + with pytest.raises(HPCCAuthenticationError): + test.get_verified() - # Test the get_verified method with an invalid username and password - def test_get_verified_invalid_username_password(self): - test = Auth( - self.HPCC_HOST, - self.HPCC_PORT, - self.DUMMY_USERNAME, - self.DUMMY_PASSWORD, - True, - "https", - ) - self.assertRaises(HPCCAuthenticationError, test.get_verified) - # Test the get_verified method with an invalid IP address - def test_get_verified_invalid_ip(self): - test = Auth( - self.DUMMY_HPCC_HOST, - self.HPCC_PORT, - self.HPCC_USERNAME, - self.HPCC_PASSWORD, - True, - "https", - ) - self.assertRaises(HPCCAuthenticationError, test.get_verified) +# Test the get_verified method with only required parameters +def test_get_verified_only_required_parameters(): + test = Auth( + HPCC_HOST, + HPCC_PORT, + HPCC_USERNAME, + HPCC_PASSWORD, + protocol=HPCC_PROTOCOL, + ) + assert test.get_verified() - # Test the get_verified method with an invalid port - def test_get_verified_invalid_port(self): - test = Auth( - self.HPCC_HOST, - self.DUMMY_HPCC_PORT, - self.HPCC_USERNAME, - self.HPCC_PASSWORD, - True, - "https", - ) - self.assertRaises(HPCCAuthenticationError, test.get_verified) - # Test the get_verified method with only required parameters - def test_get_verified_only_required_parameters(self): - test = Auth( - self.HPCC_HOST, self.HPCC_PORT, self.HPCC_USERNAME, self.HPCC_PASSWORD - ) - self.assertTrue(test.get_verified()) +# Test the get_verified method with an invalid IP address +def test_get_verified_invalid_ip(): + test = Auth( + DUMMY_HPCC_HOST, + HPCC_PORT, + HPCC_USERNAME, + HPCC_PASSWORD, + True, + HPCC_PROTOCOL, + ) + with pytest.raises(HPCCAuthenticationError): + test.get_verified() -if __name__ == "__main__": - unittest.main() +# Test the get_verified method with an invalid port +def test_get_verified_invalid_port(): + test = Auth( + HPCC_HOST, + DUMMY_HPCC_PORT, + HPCC_USERNAME, + HPCC_PASSWORD, + True, + HPCC_PROTOCOL, + ) + with pytest.raises(HPCCAuthenticationError): + test.get_verified() diff --git a/tests/models/test_hpcc_api.py b/tests/models/test_hpcc_api.py index ae0f1a3..76d11bb 100644 --- a/tests/models/test_hpcc_api.py +++ b/tests/models/test_hpcc_api.py @@ -3,21 +3,21 @@ import unittest from datetime import datetime # used by test_upload_file -import config +import conftest from pyhpcc.errors import HPCCException from pyhpcc.models.auth import Auth from pyhpcc.models.hpcc import HPCC class TestHPCCAPI(unittest.TestCase): - HPCC_HOST = config.HPCC_HOST - HPCC_PORT = config.HPCC_PORT - HPCC_USERNAME = config.HPCC_USERNAME - HPCC_PASSWORD = config.HPCC_PASSWORD - DUMMY_USERNAME = config.DUMMY_USERNAME - DUMMY_PASSWORD = config.DUMMY_PASSWORD - DUMMY_HPCC_HOST = config.DUMMY_HPCC_HOST - DUMMY_HPCC_PORT = config.DUMMY_HPCC_PORT + HPCC_HOST = conftest.HPCC_HOST + HPCC_PORT = conftest.HPCC_PORT + HPCC_USERNAME = conftest.HPCC_USERNAME + HPCC_PASSWORD = conftest.HPCC_PASSWORD + DUMMY_USERNAME = conftest.DUMMY_USERNAME + DUMMY_PASSWORD = conftest.DUMMY_PASSWORD + DUMMY_HPCC_HOST = conftest.DUMMY_HPCC_HOST + DUMMY_HPCC_PORT = conftest.DUMMY_HPCC_PORT AUTH_OBJ = Auth(HPCC_HOST, HPCC_PORT, HPCC_USERNAME, HPCC_PASSWORD, True, "https") HPCC_OBJ = HPCC(AUTH_OBJ) diff --git a/tests/models/test_workunit_submit.py b/tests/models/test_workunit_submit.py new file mode 100644 index 0000000..2a5898e --- /dev/null +++ b/tests/models/test_workunit_submit.py @@ -0,0 +1,183 @@ +import copy +import os + +import conftest +import pytest +from pyhpcc.command_config import CompileConfig +from pyhpcc.config import OUTPUT_FILE_OPTION +from pyhpcc.models.auth import Auth +from pyhpcc.models.hpcc import HPCC +from pyhpcc.models.workunit_submit import WorkunitSubmit + +DUMMY_OUTPUT = "dummy_output" + +HPCC_HOST = conftest.HPCC_HOST +HPCC_PASSWORD = conftest.HPCC_PASSWORD +HPCC_PORT = conftest.HPCC_PORT +HPCC_USERNAME = conftest.HPCC_USERNAME +ENV = "LOCAL" + + +@pytest.fixture +def auth(): + return Auth(HPCC_HOST, HPCC_PORT, HPCC_USERNAME, HPCC_PASSWORD) + + +@pytest.fixture +def hpcc(auth): + return HPCC(auth) + + +@pytest.fixture +def clusters(): + return ("thor", "hthor") + + +@pytest.fixture +def ws(hpcc, clusters): + return WorkunitSubmit(hpcc, clusters) + + +# Test if creation of WorkUnitSubmit raises error if no clusters are provided +def test_work_unit_creation_error(hpcc): + with pytest.raises(ValueError): + WorkunitSubmit(hpcc, ()) + + +# Test WorkUnitSubmit creation with correct parameters +def test_work_unit_creation_noerror(hpcc, clusters): + try: + WorkunitSubmit(hpcc, clusters) + except Exception as error: + pytest.fail( + f"Faced with exception while creating WorkunitSubmit object {str(error)}" + ) + + +# Test if get_bash_command produces correct output with and without output file in CompileConfig +@pytest.mark.parametrize( + "config, input_file, expected_output", + [ + ({}, "a.ecl", ("eclcc -o a.eclxml a.ecl", "a.eclxml")), + ( + {OUTPUT_FILE_OPTION: "hello.eclxml"}, + "a.ecl", + ("eclcc -o hello.eclxml a.ecl", "hello.eclxml"), + ), + ], +) +def test_get_bash_command_output_file(ws, config, input_file, expected_output): + compile_config = CompileConfig(config) + assert expected_output == ws.get_bash_command(input_file, compile_config) + + +activity_response_skeleton = {"ActivityResponse": {"Running": {"ActiveWorkunit": []}}} + + +def create_tests(): + active_workunits = { + ("thor", "hthor", "thor", "dthor"): "hthor", + ("dthor", "thor"): "hthor", + ("dthor", "hthor", "hthor", "thor"): "thor", + } + inputs = [] + for resp_clusters, answer in active_workunits.items(): + temp = [] + for cluster in resp_clusters: + temp.append({"TargetClusterName": cluster}) + input = copy.deepcopy(activity_response_skeleton) + input["ActivityResponse"]["Running"]["ActiveWorkunit"] = temp + inputs.append((input, answer)) + return inputs + + +# Test if WorkunitSubmit get_cluster_from_response cluster selection is correct from Activity API response +@pytest.mark.parametrize("activity, expected_output", create_tests()) +def test_get_cluster_from_response(activity, expected_output, ws): + output = ws.get_cluster_from_response(activity) + assert output == expected_output + + +# Test if get_least_active_cluster return least active cluster directly returns if only one cluster is configured +def test_get_cluster_when_one_cluster(hpcc): + clusters = ("thor",) + ws = WorkunitSubmit(hpcc, clusters) + ws.get_least_active_cluster() == clusters[0] + + +# Test if file is created with the contents for create_file_name function +@pytest.mark.parametrize( + "content, job_name, expected_file", + [("OUTPUT('HELLO WORLD!');", "Basic Job", "Basic_Job.ecl")], +) +def test_create_file(tmp_path, ws, content, job_name, expected_file): + output = ws.create_file_name(content, tmp_path, job_name) + ecl_file_path = tmp_path / expected_file + assert output == str(ecl_file_path) + assert ecl_file_path.read_text() == content + + +# Test if compilation is working for bash_compile: Runs only in local +@pytest.mark.skipif( + conftest.ENV != "LOCAL", + reason="ECL Client Tools required. Can't run on github runner", +) +@pytest.mark.parametrize( + "job_name, expected_file, options, content, status", + [ + ( + "Basic Job", + "Basic_job.eclxml", + {"-E": bool}, + "OUTPUT('HELLO WORLD!');", + "success", + ), + # ("Basic Job", "Basic_job.eclxml", {"-E": True}, "OUTPUT('HELLO WORL", 185), + ("Basic Job", "Basic_job.eclxml", None, "OUTPUT('HELLO WORLD!');", "success"), + ], +) +def test_bash_compile_full( + tmp_path, ws, job_name, options, expected_file, content, status +): + output_file = ws.create_file_name(content, tmp_path, job_name) + output, error = ws.bash_compile(output_file, options) + assert os.path.exists(tmp_path / expected_file) + assert output["status"] == status + + +# Test if RunConfig options are properly instantiated. +@pytest.mark.parametrize( + "options, expected_options", + [ + ( + None, + { + "--target": "thor", + "--job-name": "Basic_Job", + "--limit": 100, + "-s": HPCC_HOST, + "--port": f"{HPCC_PORT}", + "-u": HPCC_USERNAME, + "-pw": HPCC_PASSWORD, + }, + ), + ( + {"--target": "hthor", "--limit": 1000, "--job-name": "Basic Job 2"}, + { + "--target": "hthor", + "--job-name": "Basic Job 2", + "--limit": 1000, + "-s": HPCC_HOST, + "--port": f"{HPCC_PORT}", + "-u": HPCC_USERNAME, + "-pw": HPCC_PASSWORD, + }, + ), + ], +) +def test_configure_run_config(hpcc, options, expected_options): + clusters = ("thor",) + ws = WorkunitSubmit(hpcc, clusters) + ws.job_name = "Basic Job" + run_config = ws.configure_run_config(options) + assert run_config.options == expected_options diff --git a/tests/test_command_config.py b/tests/test_command_config.py new file mode 100644 index 0000000..61056c1 --- /dev/null +++ b/tests/test_command_config.py @@ -0,0 +1,273 @@ +import conftest +import pytest +from pyhpcc.command_config import CompileConfig, RunConfig +from pyhpcc.config import ( + MASKED_PASSWORD, + PASSWORD_OPTIONS, + PORT_OPTION, + SERVER_OPTIONS, + USER_OPTIONS, +) +from pyhpcc.errors import CompileConfigException, RunConfigException +from pyhpcc.models.auth import Auth + + +@pytest.fixture +def config_option(): + return {"--username": "testuser"} + + +@pytest.fixture +def auth(): + return Auth("university.hpccsystems.io", "8010", "testuser", "password") + + +# Test if copy is made out of provided options for creating CompileConfig +def test_compile_config_option_copy(config_option): + compile_config = CompileConfig(config_option) + config_option["-dfs"] = "website.com" + assert compile_config.options != config_option + + +# Test if compile config validation is not raising exception for correct options +@pytest.mark.parametrize( + "options", + [ + {"-user": "testuser"}, + {"-R": "r"}, + {"-Rconfig": "config"}, + {"-f": "feature"}, + {"-fconf": "fconf"}, + {"--debug": bool}, + {"--updaterepos": bool}, + {"-user": "testuser", "-R": "r", "-fconf": "feature", "--debug": bool}, + ], +) +def test_compile_config_validation_no_errors(options): + compile_config = CompileConfig(options) + try: + compile_config.validate_options() + except CompileConfigException as error: + pytest.fail(f"Faced with CompileConfig exception {str(error)}") + + +# Test if CompileConfig validate_options is raising exception for incorrect options +@pytest.mark.parametrize( + "options", + [ + {"-u": "testuser"}, + {"--manifest": "r"}, + {"--Rconfig": "config"}, + {"-userd": "testuser", "-R": "r", "-fconf": "feature", "--debug": bool}, + ], +) +def test_compile_config_validation_errors(options): + compile_config = CompileConfig(options) + with pytest.raises(CompileConfigException): + compile_config.validate_options() + + +# Test if CompileConfig create_compile_bash_command returns correct bash command +@pytest.mark.parametrize( + "file_name, options, expected_output", + [ + ( + "abc.xml", + {"--target": "thor", "-f": "dsafda"}, + "eclcc --target thor -f dsafda abc.xml", + ), + ("a.xml", {}, "eclcc a.xml"), + ( + "/usr/loc/Basic_job_submission.ecl", + { + "-platform": "thor", + "-wu": bool, + "-E": bool, + "-o": "/usr/loc/Basic_job_submission.eclxml", + }, + "eclcc -platform thor -wu -E -o /usr/loc/Basic_job_submission.eclxml /usr/loc/Basic_job_submission.ecl", + ), + ( + "/usr/loc/Basic_job_submission.ecl", + { + "-platform": "thor", + "-wu": bool, + "-E": bool, + "-o": "/usr/loc/Basic_job_submission.eclxml", + }, + "eclcc -platform thor -wu -E -o /usr/loc/Basic_job_submission.eclxml /usr/loc/Basic_job_submission.ecl", + ), + ], +) +def test_create_compile_bash_command(file_name, options, expected_output): + compile_config = CompileConfig(options) + cmd = compile_config.create_compile_bash_command(file_name) + assert set(cmd.split(" ")) == set(expected_output.split(" ")) + + +# Test if copy is made out of provided options for creating CompileConfig +def test_run_config_option_copy(config_option): + run_config = RunConfig(config_option) + config_option["--v"] = bool + assert run_config.options != config_option + + +# Test if RunConfig validate_options is raising exception for incorrect options +@pytest.mark.parametrize( + "options", + [ + {"--username": "testuser"}, + {"-X": "x"}, + {"-v": bool}, + {"--verbose": "bool"}, + {"-fconf": "fconf"}, + {"--updaterepos": bool}, + { + "--username": "testuser", + "-X": "thor", + "-fconf": "feature", + "--fetchrepos": bool, + }, + ], +) +def test_run_config_validation_no_errors(options): + run_config = RunConfig(options) + try: + run_config.validate_options() + except RunConfigException as error: + pytest.fail(f"Faced with CompileConfig exception {str(error)}") + + +# Test if RunConfig validate_options is raising errors for incorrect options +@pytest.mark.parametrize( + "options", + [ + {"--user": "testuser"}, + {"-R": "x"}, + {"--v": bool}, + {"-cacert": "bool"}, + {"-Fconf": "fconf"}, + {"abcd": bool}, + { + "--user": "testuser", + "-x": "thor", + "-Fconf": "feature", + "--Dname": "name", + }, + ], +) +def test_run_config_validation_errors(options): + run_config = RunConfig(options) + with pytest.raises(RunConfigException): + run_config.validate_options() + + +# Test if RunConfig create_run_bash_command is creating correct command +@pytest.mark.parametrize( + "file_name, options, expected_output", + [ + ( + "/usr/loc/Basic_job_submission.eclxml", + { + "--target": "thor", + "--job-name": "my_custom_workunit", + "--limit": 100, + "-s": conftest.HPCC_HOST, + "--port": conftest.HPCC_PORT, + "-u": conftest.HPCC_USERNAME, + "-pw": conftest.HPCC_PASSWORD, + }, + f"ecl run /usr/loc/Basic_job_submission.eclxml --target thor --job-name my_custom_workunit --limit 100 -s {conftest.HPCC_HOST} --port {conftest.HPCC_PORT} -u {conftest.HPCC_USERNAME} -pw {conftest.HPCC_PASSWORD}", + ), + ( + "/usr/loc/Basic_job_submission.eclxml", + { + "--target": "thor", + "--job-name": "Basic_job_submission", + "--limit": 100, + "-s": conftest.HPCC_HOST, + "--port": conftest.HPCC_PORT, + "-u": conftest.HPCC_USERNAME, + "-pw": conftest.HPCC_PASSWORD, + }, + f"ecl run /usr/loc/Basic_job_submission.eclxml --target thor --job-name Basic_job_submission --limit 100 -s {conftest.HPCC_HOST} --port {conftest.HPCC_PORT} -u {conftest.HPCC_USERNAME} -pw {conftest.HPCC_PASSWORD}", + ), + ( + "/usr/loc/Basic_job_submission.eclxml", + { + "--target": "thor", + "-v": bool, + "--job-name": "Basic_job_submission", + "--limit": 100, + "-s": conftest.HPCC_HOST, + "--port": conftest.HPCC_PORT, + "-u": conftest.HPCC_USERNAME, + "-pw": conftest.HPCC_PASSWORD, + }, + f"ecl run /usr/loc/Basic_job_submission.eclxml --target thor -v --job-name Basic_job_submission --limit 100 -s {conftest.HPCC_HOST} --port {conftest.HPCC_PORT} -u {conftest.HPCC_USERNAME} -pw {conftest.HPCC_PASSWORD}", + ), + ], +) +def test_create_run_bash_command(file_name, options, expected_output): + run_config = RunConfig(options) + cmd = run_config.create_run_bash_command(file_name) + assert set(cmd.split(" ")) == set(expected_output.split(" ")) + + +@pytest.mark.parametrize( + "file_name, options, expected_output", + [ + ( + "/usr/loc/Basic_job_submission.eclxml", + { + "--target": "thor", + "--job-name": "my_custom_workunit", + "--limit": 100, + "-s": conftest.HPCC_HOST, + "--port": conftest.HPCC_PORT, + "-u": conftest.HPCC_USERNAME, + "-pw": conftest.HPCC_PASSWORD, + }, + f"ecl run /usr/loc/Basic_job_submission.eclxml --target thor --job-name my_custom_workunit --limit 100 -s {conftest.HPCC_HOST} --port {conftest.HPCC_PORT} -u {conftest.HPCC_USERNAME} -pw {MASKED_PASSWORD}", + ), + ], +) +def test_create_run_bash_command_mask_password(file_name, options, expected_output): + run_config = RunConfig(options) + cmd = run_config.create_run_bash_command(file_name, password_mask=True) + assert set(cmd.split(" ")) == set(expected_output.split(" ")) + + +# Test if RunConfig create_run_bash_command raises error for incorrect options +@pytest.mark.parametrize( + "file_name, options", + [ + ( + "/usr/loc/Basic_job_submission.eclxml", + { + "--target": "thor", + "--name": "Basic_job_submission", + "--limit": 100, + "-s": conftest.HPCC_HOST, + "--port": conftest.HPCC_PORT, + "-u": conftest.HPCC_USERNAME, + "-pwd": conftest.HPCC_PASSWORD, + }, + ), + ], +) +def test_create_run_bash_command_errors(file_name, options): + run_config = RunConfig(options) + with pytest.raises(RunConfigException): + run_config.create_run_bash_command(file_name) + + +# Test RunConfig set_auth_params set the auth options properly +def test_set_auth_params(config_option, auth: Auth): + run_config = RunConfig(config_option) + run_config.set_auth_params(auth) + assert run_config.get_option(SERVER_OPTIONS[0]) == auth.ip + assert run_config.get_option(PASSWORD_OPTIONS[0]) == auth.password + assert run_config.get_option(PORT_OPTION) == auth.port + assert run_config.get_option(USER_OPTIONS[0]) == auth.username + assert len(run_config.options) == 4 diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..0d025c0 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,85 @@ +import pyhpcc.utils as utils +import pytest + + +@pytest.mark.parametrize( + "resp, expected_output", + [ + (b"", {"status": "success", "raw_output": ""}), + ( + b"Error: File 'Basic_job_submission.' does not exist\nNo input files could be opened\n", + { + "status": "error", + "errors": ["Error: File 'Basic_job_submission.' does not exist"], + }, + ), + ( + b"/Users/rohithsuryapodugu/Documents/GitHub/pyhpcc-internal/examples/Basic_job_submission.ecl(1,8): error C2195: String constant is not terminated: \"'HELLO WORLD;\"\n/Users/rohithsuryapodugu/Documents/GitHub/pyhpcc-internal/examples/Basic_job_submission.ecl(1,21): error C3002: syntax error : expected ')'\n2 errors, 0 warning\n", + { + "status": "error", + "errors": [ + '/Users/rohithsuryapodugu/Documents/GitHub/pyhpcc-internal/examples/Basic_job_submission.ecl(1,8): error C2195: String constant is not terminated: "\'HELLO WORLD;"', + "/Users/rohithsuryapodugu/Documents/GitHub/pyhpcc-internal/examples/Basic_job_submission.ecl(1,21): error C3002: syntax error : expected ')'", + ], + }, + ), + ( + b"Failed to compile Basic_job_submission.eclxml\nBasic_job_submission.eclxml.res.s(5,2): error C6003: unknown directive\nBasic_job_submission.eclxml.res.s(10,2): error C6003: unknown directive\nBasic_job_submission.eclxml.res.s(15,2): error C6003: unknown directive\nBasic_job_submission.eclxml(0,0): error C3000: Compile/Link failed for Basic_job_submission.eclxml (see //192.168.86.46/Users/rohithsuryapodugu/Documents/GitHub/pyhpcc-internal/examples/eclcc.log for details)\n\n---------- compiler output --------------\nBasic_job_submission.eclxml.res.s:5:2: error: unknown directive\n .size RESULT_XSD_1001_txt_start,339\n ^\nBasic_job_submission.eclxml.res.s:10:2: error: unknown directive\n .size BINWORKUNIT_1000_txt_start,1125\n ^\nBasic_job_submission.eclxml.res.s:15:2: error: unknown directive\n .size MANIFEST_1000_txt_start,182\n ^\n\n--------- end compiler output -----------\n4 errors, 0 warning\n", + { + "status": "error", + "errors": [ + "Failed to compile Basic_job_submission.eclxml", + "Basic_job_submission.eclxml.res.s(5,2): error C6003: unknown directive", + "Basic_job_submission.eclxml.res.s(10,2): error C6003: unknown directive", + "Basic_job_submission.eclxml.res.s(15,2): error C6003: unknown directive", + "Basic_job_submission.eclxml(0,0): error C3000: Compile/Link failed for Basic_job_submission.eclxml (see //192.168.86.46/Users/rohithsuryapodugu/Documents/GitHub/pyhpcc-internal/examples/eclcc.log for details)", + ], + }, + ), + ], +) +def test_bash_compile(resp, expected_output): + output = utils.parse_bash_compile_output(resp) + expected_output["raw_output"] = resp.decode() + assert output == expected_output + + +@pytest.mark.parametrize( + "resp, expected_output", + [ + ( + b"EXEC: Creating PIPE program process : '/opt/HPCCSystems/9.6.14/clienttools/bin/eclcc -E \"Basic_job_submission.ecl\" -fapplyInstantEclTransformations=1 -fapplyInstantEclTransformationsLimit=100' - hasinput=0, hasoutput=1 stderrbufsize=0 [] in ()\nEXEC: Pipe: process 19789 complete 0\njsocket(9,2566) shutdown err = 57 : C!:52923 -> university.us-hpccsystems-dev.azure.lnrsg.io:8010 (5)\njsocket(9,2566) shutdown err = 57 : C!:52924 -> university.us-hpccsystems-dev.azure.lnrsg.io:8010 (5)\njsocket(9,2566) shutdown err = 57 : C!:52925 -> university.us-hpccsystems-dev.azure.lnrsg.io:8010 (5)\nUsing eclcc path /opt/HPCCSystems/9.6.14/clienttools/bin/eclcc\n\nDeploying ECL Archive Basic_job_submission.ecl\n\nDeployed\n wuid: W20240701-115916\n state: compiled\n\nRunning deployed workunit W20240701-115916\n\n\n HELLO WORLD\n\n\n", + { + "wu_info": {"wuid": "W20240701-115916", "state": "compiled"}, + "misc_info": { + "message": [ + "EXEC: Creating PIPE program process : '/opt/HPCCSystems/9.6.14/clienttools/bin/eclcc -E \"Basic_job_submission.ecl\" -fapplyInstantEclTransformations=1 -fapplyInstantEclTransformationsLimit=100' - hasinput=0, hasoutput=1 stderrbufsize=0 [] in ()", + "EXEC: Pipe: process 19789 complete 0", + "", + "", + "HELLO WORLD", + "", + "", + ] + }, + }, + ), + ( + b"EXEC: Creating PIPE program process : '/opt/HPCCSystems/9.6.14/clienttools/bin/eclcc -E \"Basic_job_submission.ecl\" -fapplyInstantEclTransformations=1 -fapplyInstantEclTransformationsLimit=100' - hasinput=0, hasoutput=1 stderrbufsize=0 [] in ()\nEXEC: Pipe: process 21114 complete 0\njsocket(9,2566) shutdown err = 57 : C!:54803 -> university.us-hpccsystems-dev.azure.lnrsg.io:8010 (5)\njsocket(9,2566) shutdown err = 57 : C!:54805 -> university.us-hpccsystems-dev.azure.lnrsg.io:8010 (5)\n\n401: Unauthorized Access\nUsing eclcc path /opt/HPCCSystems/9.6.14/clienttools/bin/eclcc\n\nDeploying ECL Archive Basic_job_submission.ecl\n", + { + "error": {"message": ["401: Unauthorized Access"]}, + "wu_info": {"wuid": None, "state": None}, + "misc_info": { + "message": [ + "EXEC: Creating PIPE program process : '/opt/HPCCSystems/9.6.14/clienttools/bin/eclcc -E \"Basic_job_submission.ecl\" -fapplyInstantEclTransformations=1 -fapplyInstantEclTransformationsLimit=100' - hasinput=0, hasoutput=1 stderrbufsize=0 [] in ()", + "EXEC: Pipe: process 21114 complete 0", + ] + }, + }, + ), + ], +) +def test_bash_run(resp, expected_output): + output = utils.parse_bash_run_output(resp) + expected_output["raw_output"] = resp.decode() + assert output == expected_output