[#32100] Customize bash_run and bash_compile. Add support for multi cluster options

.github/workflows/pr.yaml

@@ -22,7 +22,7 @@ jobs:
             pull_number: context.issue.number
           })
           const isTitleValid = /^\[#\d+\] /.test(pr.data.title)
-          const isDescriptionValid = /([Ff]ix(es|ed)?|[Cc]lose(s|d)?|[Rr]esolve(s|d)?|[Pp]art [Oo]f) \(.*\)\[.*\]/.test(pr.data.body)
+          const isDescriptionValid = /([Ff]ix(es|ed)?|[Cc]lose(s|d)?|[Rr]esolve(s|d)?|[Pp]art [Oo]f) \[.*\]\(.*\)/.test(pr.data.body)
           if (isTitleValid && isDescriptionValid) {
             return
           }

.gitignore

@@ -130,8 +130,7 @@ dmypy.json
 # Pyre type checker
 .pyre/
 
-my_secret.py
-test_hpcc.py
+tests/my_secret.py
 
 # Miscellaneous folders
 Dummy/

.pre-commit-config.yaml

@@ -0,0 +1,7 @@
+repos:
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: "v0.5.0"
+  hooks:
+    - id: ruff
+      args: ["--fix"]
+    - id: ruff-format

CONTRIBUTING.md

@@ -4,6 +4,26 @@ We value your interest in contributing to `PyHPCC.`
 
 Thank you
 
+## Project Structure
+```
+.
+└── pyhpcc/
+    ├── .github # contains build, release, test and other gh actions
+    ├── docs/ # contains files for documentation
+    ├── examples/ # contains starter examples
+    ├── src/
+    │   ├── pyhpcc/
+    │   │   ├── handlers/ # contains thor and roxie handler
+    │   │   └── models/ # contains classes auth, workunit submit
+    │   └── tests/
+    │       ├── models/
+    │       ├── test_files/ # contains resource files needed for testing
+    │       └── hanlders/
+    ├── pyproject.toml # Project config
+    ├── CONTRIBUTING.md
+    └── README.md
+```
+
 ## Set up the repository locally.
 ## Prerequisites
 Before starting to develop, make sure you install the following software:
@@ -25,6 +45,15 @@ To install the dependencies, run the following command, which downloads the depe
 poetry install
 ```
 
+## How to run tests  
+Since ecl client tools aren't installed in the GitHub runner, some tests are skipped in the github runner.
+
+Some tests will fail if `ecl client tools` aren't installed.
+
+```
+pytest run # Run in project root
+```
+
 ## Linting and Formatting
 PyHPCC uses [Ruff](https://docs.astral.sh/ruff/) as its formatter and linter.
 
@@ -56,8 +85,4 @@ The base branch is the main repo's main branch.
 - PR name: copy-and-paste the relevant issue name and include the issue number in front in square brackets, e.g. `[#1020] Make bash_runcommand in WorkUnitSubmit class configurable `
 - PR description: mention the issue number in this format: Fixes #1020. Doing so will automatically close the related issue once the PR is merged.
 - Please Ensure that "Allow edits from maintainers" is ticked.
-- Please describe the changes you have made in your branch and how they resolve the issue.  
-
-
-
-
+- Please describe the changes you have made in your branch and how they resolve the issue.  

README.md

@@ -17,6 +17,8 @@ To use PyHPCC, you need these<br>
 1. [Python3](https://www.python.org/downloads/)
 2. [ECL Client Tools](https://hpccsystems.com/download/): Select your operating systems to download client tools
 
+
+
 Download the latest stable build from releases in GitHub.<br>
 
 ``` bash
@@ -36,5 +38,4 @@ Contributions to PyHPCC are welcomed and encouraged.<br>
 For specific contribution guidelines, please take a look at [CONTRIBUTING.md](CONTRIBUTING.md).
 
 
-For more information about the package, please refer to the detailed documentation - https://upgraded-bassoon-daa9d010.pages.github.io/build/html/index.html
-
+For more information about the package, please refer to the detailed documentation - https://upgraded-bassoon-daa9d010.pages.github.io/build/html/index.html

pyproject.toml

@@ -19,6 +19,7 @@ elementpath = "^3.0.2" # XPath selectors for XML Data structures
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.1.3"
 pytest-mock = "^3.0.2"
+pre-commit="^3.7.1"
 
 [tool.poetry.group.coverage.dependencies]
 coverage = "^7.5.3"

src/pyhpcc/command_config.py

@@ -0,0 +1,202 @@
+import logging
+
+from pyhpcc.config import (
+    CLUSTER_OPTION,
+    COMPILE_OPTIONS,
+    JOB_NAME_OPTION,
+    LIMIT_OPTION,
+    MASKED_PASSWORD,
+    OUTPUT_FILE_OPTION,
+    PASSWORD_OPTIONS,
+    PORT_OPTION,
+    RUN_AUTH_OPTIONS,
+    RUN_OPTIONS,
+    SERVER_OPTIONS,
+    USER_OPTIONS,
+)
+from pyhpcc.errors import CompileConfigException, RunConfigException
+from pyhpcc.models.auth import Auth
+
+log = logging.getLogger(__name__)
+
+
+class CompileConfig(object):
+    """
+    Class for eclcc option configuration
+
+    Attributes:
+    ----------
+        options:
+            Dictionary of keys (option), values (value)
+
+    Methods:
+    -------
+        validate_options:
+            Validate if the compiler options are supported or not
+
+        set_output_file:
+            Set name of output file (default a.out if linking to
+
+        create_compile_bash_command:
+            Generate the eclcc command for the given options and input_file
+
+        get_option:
+            Retrieves the compile config option
+
+
+    """
+
+    def __init__(self, options: dict):
+        self.options = options.copy()
+
+    def validate_options(self):
+        """Validate if the compiler options are supported or not"""
+        invalid_options = []
+        for option in self.options:
+            if option not in COMPILE_OPTIONS and not (
+                option.startswith("-R") or option.startswith("-f")
+            ):
+                invalid_options.append(option)
+        if len(invalid_options) > 0:
+            raise CompileConfigException(str(invalid_options))
+
+    def set_output_file(self, output_file):
+        """Set name of output file (default a.out if linking to"""
+        self.options[OUTPUT_FILE_OPTION] = output_file
+
+    def create_compile_bash_command(self, input_file):
+        """Generate the eclcc command for the given options and input_file"""
+        self.validate_options()
+        eclcc_command = "eclcc"
+        for key, value in self.options.items():
+            if value is bool:
+                eclcc_command += f" {key}"
+            else:
+                eclcc_command += f" {key} {value}"
+        eclcc_command = f"{eclcc_command} {input_file}"
+        return eclcc_command
+
+    def get_option(self, option):
+        """Get the option available for the option"""
+        return self.options[option]
+
+
+class RunConfig(object):
+    """
+    Class for ecl run option configuration
+
+    Attributes:
+    ----------
+        options:
+            Dictionary of keys (option), values (value)
+
+    Methods:
+    -------
+        validate_options:
+            Validate if the compiler options are supported or not
+
+        create_run_bash_command:
+            Generate the ecl command for the given options and target_file
+
+        set_auth_params:
+            Set the auth parameters from the auth object passed
+
+        set_target:
+            Specify the job name for the workunit
+
+        set_limit:
+            Sets the result limit for the query
+
+        set_server:
+            Set IP of server running ecl services (eclwatch)
+
+        set_port:
+            Set ECL services port
+
+        set_username:
+            Set username for accessing ecl services
+
+        set_password:
+            Set password for accessing ecl services
+
+        get_option:
+            Retrieves the run config option
+
+    """
+
+    def __init__(self, options: dict):
+        self.options = options.copy()
+
+    def validate_options(self):
+        """Validate if the runtime options are supported or not"""
+        invalid_options = set()
+        for option in self.options:
+            if option not in RUN_OPTIONS and not (
+                option.startswith("-X") or option.startswith("-f")
+            ):
+                invalid_options.add(option)
+        if len(invalid_options) > 0:
+            log.error("Entered invalid options %s", str(invalid_options))
+            raise RunConfigException(
+                f"Invalid options not supported by pyhpcc {str(invalid_options)}"
+            )
+
+    def create_run_bash_command(self, target_file, password_mask=False):
+        """Generate the ecl command for the given options and target_file"""
+        self.validate_options()
+        ecl_command = "ecl run"
+        params = ""
+        for key, value in self.options.items():
+            if value is bool:
+                params += f" {key}"
+            else:
+                if key in PASSWORD_OPTIONS and password_mask:
+                    params += f" {key} {MASKED_PASSWORD}"
+                else:
+                    params += f" {key} {value}"
+        ecl_command = f"{ecl_command} {target_file}{params}"
+        log.info(ecl_command)
+        return ecl_command
+
+    def set_auth_params(self, auth: Auth):
+        """Set the auth parameters from the auth object passed"""
+        for option in list(self.options):
+            if option in RUN_AUTH_OPTIONS:
+                log.warning(f"Overriding option {option} with Auth object parameters")
+                del self.options[option]
+        self.set_server(auth.ip)
+        self.set_port(auth.port)
+        self.set_username(auth.oauth[0])
+        self.set_password(auth.oauth[1])
+
+    def set_target(self, target):
+        """Set the target"""
+        self.options[CLUSTER_OPTION] = target
+
+    def set_job_name(self, job_name):
+        """Specify the job name for the workunit"""
+        self.options[JOB_NAME_OPTION] = job_name
+
+    def set_limit(self, limit):
+        """Sets the result limit for the query"""
+        self.options[LIMIT_OPTION] = limit
+
+    def set_server(self, server):
+        """Set IP of server running ecl services (eclwatch)"""
+        self.options[SERVER_OPTIONS[0]] = server
+
+    def set_port(self, port):
+        """Set ECL services port"""
+        self.options[PORT_OPTION] = port
+
+    def set_username(self, username):
+        """Set username for accessing ecl services"""
+        self.options[USER_OPTIONS[0]] = username
+
+    def set_password(self, password):
+        """Set password for accessing ecl services"""
+        self.options[PASSWORD_OPTIONS[0]] = password
+
+    def get_option(self, option):
+        """Get the option available for the option"""
+        return self.options[option]