feat: refactor script templates

Use Jinja2 templates for rendering scripts
Deprecate and remove use of ScriptTemplates

examples/empty_config.yaml

@@ -29,9 +29,6 @@
           prod_area: 'output/archive'
           data_query: "instrument = 'HSC'"
           lsst_version: "${WEEKLY}"
-          bps_script_template: bps_panda_script_template
-          bps_yaml_template: bps_yaml_template
-          manifest_script_template: manifest_script_template
 - Specification:
       name: base
       spec_aliases:

examples/example_standard_elements.yaml

@@ -145,10 +145,6 @@
                     inputs: ["{campaign_output}", "{campaign_input}", "{campaign_resource_usage}"]
                     output: "{campaign_public_output}"
       data:
-          bps_core_script_template: bps_core_script_template
-          bps_core_yaml_template: bps_core_yaml_template
-          manifest_script_template: stack_script_template
-          resource_usage_script_template: stack_script_template
           lsst_version: MUST_OVERRIDE
           lsst_distrib_dir: /cvmfs/sw.lsst.eu/linux-x86_64/lsst_distrib
 # Campagin Template for fixed inputs
@@ -189,10 +185,6 @@
                     inputs: ["{campaign_output}", "{campaign_input}"]
                     output: "{campaign_public_output}"
       data:
-          bps_core_script_template: bps_core_script_template
-          bps_core_yaml_template: bps_core_yaml_template
-          manifest_script_template: stack_script_template
-          resource_usage_script_template: stack_script_template
           lsst_version: MUST_OVERRIDE
           lsst_distrib_dir: /cvmfs/sw.lsst.eu/linux-x86_64/lsst_distrib
 # Campaign template for base campaign+hips maps
@@ -263,11 +255,6 @@
                     inputs: ["{campaign_output}", "{campaign_input}", "{campaign_resource_usage}", "{campaign_hips_maps}"]
                     output: "{campaign_public_output}"
       data:
-          bps_core_script_template: bps_core_script_template
-          bps_core_yaml_template: bps_core_yaml_template
-          manifest_script_template: stack_script_template
-          resource_usage_script_template: stack_script_template
-          hips_maps_script_template: stack_script_template
           lsst_version: MUST_OVERRIDE
           lsst_distrib_dir: /cvmfs/sw.lsst.eu/linux-x86_64/lsst_distrib
 # Campaign Template for fixed inputs which also generates HIPS maps
@@ -315,10 +302,5 @@
                     inputs: ["{campaign_output}", "{campaign_input}", "{campaign_resource_usage}", "{campaign_hips_maps}"]
                     output: "{campaign_public_output}"
       data:
-          bps_core_script_template: bps_core_script_template
-          bps_core_yaml_template: bps_core_yaml_template
-          manifest_script_template: stack_script_template
-          resource_usage_script_template: stack_script_template
-          hips_maps_script_template: stack_script_template
           lsst_version: MUST_OVERRIDE
           lsst_distrib_dir: /cvmfs/sw.lsst.eu/linux-x86_64/lsst_distrib

examples/example_standard_scripts.yaml

@@ -1,23 +1,3 @@
-# Template shell script fragment for running BPS
-- ScriptTemplate:
-      name: bps_core_script_template
-      file_path: ${CM_CONFIGS}/templates/example_bps_core_script_template.yaml
-# Template shell script fragment for running BPS under PANDA
-- ScriptTemplate:
-      name: bps_panda_script_template
-      file_path: ${CM_CONFIGS}/templates/example_bps_panda_script_template.yaml
-# Template shell script fragment for running BPS under HTCondor
-- ScriptTemplate:
-      name: bps_htcondor_script_template
-      file_path: ${CM_CONFIGS}/templates/example_bps_htcondor_script_template.yaml
-# Template shell script fragment for using stack
-- ScriptTemplate:
-      name: stack_script_template
-      file_path: ${CM_CONFIGS}/templates/example_stack_script_template.yaml
-# Template yaml config for running bps
-- ScriptTemplate:
-      name: bps_core_yaml_template
-      file_path: ${CM_CONFIGS}/templates/example_bps_core_yaml_template.yaml
 # define the null Script
 - SpecBlock:
       name: null_script
@@ -69,8 +49,8 @@
       name: bps_panda_submit_script
       handler: lsst.cmservice.handlers.jobs.PandaScriptHandler
       data:
+          wms: panda
           bps_wms_yaml_file: "${CTRL_BPS_PANDA_DIR}/config/bps_usdf.yaml"
-          bps_wms_script_template: bps_panda_script_template
 # Run a bps report script
 - SpecBlock:
       name: bps_panda_report_script
@@ -80,8 +60,7 @@
       name: bps_htcondor_submit_script
       handler: lsst.cmservice.handlers.jobs.HTCondorScriptHandler
       data:
-          bps_wms_yaml_file: "${CM_CONFIGS}/stack_files/bps_htcondor_usdf.yaml"
-          bps_wms_script_template: bps_htcondor_script_template
+          wms: htcondor
 # Run a bps report script
 - SpecBlock:
       name: bps_htcondor_report_script

src/lsst/cmservice/common/bash.py

@@ -7,6 +7,7 @@
 import yaml
 from anyio import Path, open_file, open_process
 from anyio.streams.text import TextReceiveStream
+from jinja2 import Environment, PackageLoader
 
 from ..config import config
 from .enums import StatusEnum
@@ -169,6 +170,7 @@ async def check_stamp_file(
 async def write_bash_script(
     script_url: str | Path,
     command: str,
+    values: dict,
     **kwargs: Any,
 ) -> Path:
     """Utility function to write a bash script for later execution.
@@ -181,34 +183,28 @@ async def write_bash_script(
     command: `str`
         Main command line(s) in the script
 
+    values: `dict`
+        Mapping of potential template variables to values.
+
     Keywords
     --------
-    prepend: `str | None`
-        Text to prepend before command
-
-    append: `str | None`
-        Test to append after command
-
-    stamp: `str | None`
-        Text to echo to stamp file when script completes
-
-    stamp_url: `str | None`
-        Stamp file to write to when script completes
-
     fake: `str | None`
         Echo command instead of running it
 
     rollback: `str | Path | None`
         Prefix to script_url used when rolling back
         processing. Will default to CWD (".").
 
+
     Returns
     -------
     script_url : `anyio.Path`
         The path to the newly written script
     """
-    prepend = kwargs.get("prepend")
-    append = kwargs.get("append")
+    # Get the yaml template using package lookup
+    template_environment = Environment(loader=PackageLoader("lsst.cmservice"))
+    bash_template = template_environment.get_template("wms_submit_sh.j2")
+
     fake = kwargs.get("fake")
     rollback_prefix = Path(kwargs.get("rollback", "."))
 
@@ -218,8 +214,17 @@ async def write_bash_script(
         command = f"echo '{command}'"
 
     await script_path.parent.mkdir(parents=True, exist_ok=True)
-    contents = (prepend if prepend else "") + "\n" + command + "\n" + (append if append else "")
 
-    async with await open_file(script_path, "w") as fout:
-        await fout.write(contents)
+    template_values = {
+        "command": command,
+        **values,
+    }
+
+    try:
+        # Render bash script template to `script_path`
+        bash_output = bash_template.render(template_values)
+        await Path(script_path).write_text(bash_output)
+    except Exception as e:
+        raise RuntimeError(f"Error writing a script to run BPS job {script_url}; threw {e}")
+
     return script_path