LIU-420: Protoype config.INI and Slurm script templates

daliuge-engine/dlg/deploy/configs/__init__.py

@@ -52,6 +52,16 @@
 
 $EXEC_PREFIX $PY_BIN -m dlg.deploy.start_dlg_cluster --log_dir $LOG_DIR $GRAPH_PAR $PROXY_PAR $GRAPH_VIS_PAR $LOGV_PAR $ZERORUN_PAR $MAXTHREADS_PAR $SNC_PAR $NUM_ISLANDS_PAR $ALL_NICS $CHECK_WITH_SESSION --ssid $SESSION_ID
 """
+
+__slurm_shebang = "#!/bin/bash --login"
+
+dlg_exec_str = (
+    "$EXEC_PREFIX $PY_BIN -m dlg.deploy.start_dlg_cluster" 
+    " --log_dir $LOG_DIR $GRAPH_PAR $PROXY_PAR $GRAPH_VIS_PAR $LOGV_PAR $ZERORUN_PAR"
+    " $MAXTHREADS_PAR $SNC_PAR $NUM_ISLANDS_PAR $ALL_NICS $CHECK_WITH_SESSION"
+    " --ssid $SESSION_ID"
+)
+
 init_tpl = string.Template(__sub_tpl_str)
 
 

daliuge-engine/dlg/deploy/configs/default.ini

@@ -0,0 +1,11 @@
+; This will be replaced by the above definition when processed
+[ENVIRONMENT]
+USER = 
+ACCOUNT =   
+LOGIN_NODE = 
+HOME_DIR = 
+DLG_ROOT = ${HOME_DIR}/dlg 
+LOG_DIR = ${DLG_ROOT}/log
+MODULES = 
+VENV = ${DLG_ROOT}/venv
+EXEC_PREFIX = "srun -l"

daliuge-engine/dlg/deploy/configs/default.slurm

@@ -0,0 +1,11 @@
+#!/bin/bash --login
+
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=2
+#SBATCH --job-name=DALiuGE-$SESSION_ID
+#SBATCH --time=30
+#SBATCH --error=err-%j.log
+export DLG_ROOT=$DLG_ROOT
+
+VENV=$DLG_ROOT/venv

daliuge-engine/dlg/deploy/configs/setonix.ini

@@ -0,0 +1,11 @@
+; This will be replaced by the above definition when processed
+[ENVIRONMENT]
+USER = rbunney
+ACCOUNT = pawsey0411
+LOGIN_NODE = setonix.pawsey.org.au
+HOME_DIR = /scratch/${ACCOUNT}
+DLG_ROOT = ${HOME_DIR}/${USER}/dlg
+LOG_DIR = ${DLG_ROOT}/log
+MODULES = 
+VENV = source /software/projects/${ACCOUNT}/venv/bin/activate
+EXEC_PREFIX = srun -l

daliuge-engine/dlg/deploy/configs/setonix.slurm

@@ -0,0 +1,11 @@
+#!/bin/bash --login
+
+#SBATCH --nodes=2
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=2
+#SBATCH --job-name=DALiuGE-$SESSION_ID
+#SBATCH --time=00:45:00
+#SBATCH --error=err-%j.log
+
+export DLG_ROOT=$DLG_ROOT
+source /software/projects/pawsey0411/venv/bin/activate

daliuge-engine/dlg/deploy/create_dlg_job.py

@@ -37,6 +37,8 @@
 import time
 import os
 
+from pathlib import Path
+
 from dlg.deploy.configs import (
     ConfigFactory,
 )  # get all available configurations
@@ -403,6 +405,59 @@ def check_log_dir(self, log_dir):
                 return True
         return False
 
+def process_config(config_file: str):
+    """
+    Use configparser to process INI file
+
+    Current functionality: 
+        - Returns remote environment config (e.g. DLG_ROOT, HOME etc.)
+
+    Future Functionality: 
+        - Graph translation parameters
+        - Engine parameters
+
+    :returns: dict, config information
+    """
+    from configparser import ConfigParser, ExtendedInterpolation
+    parser = ConfigParser(interpolation=ExtendedInterpolation())
+    parser.read(config_file)
+    return (dict(parser["ENVIRONMENT"]))
+
+def process_slurm_template(template_file: str):
+    template = Path(template_file)
+    with template.open('r') as fp:
+        return fp.read()
+
+def create_experiment_group(parser: optparse.OptionParser):
+    from optparse import OptionGroup
+    group=OptionGroup(parser, "Experimental Options", 
+                      "Caution: These are not properly tested and likely to"
+                      "be rough around the edges.")
+
+    group.add_option(
+        "--config_file",
+        dest="config_file", 
+        type="string", 
+        action="store", 
+        help="Use INI configuration file.",
+        default=None
+    )
+    group.add_option(
+        "--slurm_template", 
+        dest="slurm_template",
+        type="string", 
+        action="store", 
+        help="Use SLURM template file for job submission. WARNING: Using this command will over-write other job-parameters passed here.", 
+        default=None
+    )
+    return group
+
+def create_job_group():
+    pass
+
+def create_graph_group():
+    pass
+
 
 def main():
     parser = optparse.OptionParser(
@@ -623,7 +678,7 @@ def main():
         "--configs",
         dest="configs",
         action="store_true",
-        help="Display the available configurations  and exit",
+        help="Display the available configurations and exit",
         default=False,
     )
     parser.add_option(
@@ -636,6 +691,8 @@ def main():
         default=None,
     )
 
+    parser.add_option_group(create_experiment_group(parser))
+
     (opts, _) = parser.parse_args(sys.argv)
     if opts.configs:
         print(f"Available facilities: {FACILITIES}")
@@ -715,6 +772,10 @@ def main():
             else:
                 pgt_file = path_to_graph_file
 
+        config = process_config(opts.config_file) if opts.config_file else None
+        template = process_slurm_template(
+            opts.slurm_template) if opts.slurm_template else None
+
         client = SlurmClient(
             dlg_root=opts.dlg_root,
             log_root=opts.log_root,
@@ -734,7 +795,10 @@ def main():
             submit=opts.submit,
             remote=opts.remote,
             username=opts.username,
+            config=config,
+            slurm_template=template
         )
+
         client._visualise_graph = opts.visualise_graph
         client.submit_job()
     else:

daliuge-engine/dlg/deploy/deployment_utils.py

@@ -122,6 +122,9 @@ def find_numislands(physical_graph_template_file):
     init already.
     TODO: We will probably need to do the same with job duration and CPU number
     """
+    if not physical_graph_template_file:
+        return None, None, physical_graph_template_file
+
     with open(physical_graph_template_file, "r") as f:
         pgt_data = json.load(f, strict=False)
     try: