feat: Deprecate and remove use of ScriptTemplates

examples/empty_config.yaml

@@ -29,9 +29,6 @@
           prod_area: 'output/archive'
           data_query: "instrument = 'HSC'"
           lsst_version: "${WEEKLY}"
-          bps_script_template: bps_panda_script_template
-          bps_yaml_template: bps_yaml_template
-          manifest_script_template: manifest_script_template
 - Specification:
       name: base
       spec_aliases:

examples/example_standard_elements.yaml

@@ -145,10 +145,6 @@
                     inputs: ["{campaign_output}", "{campaign_input}", "{campaign_resource_usage}"]
                     output: "{campaign_public_output}"
       data:
-          bps_core_script_template: stack_script_template
-          bps_core_yaml_template: bps_core_yaml_template
-          manifest_script_template: stack_script_template
-          resource_usage_script_template: stack_script_template
           lsst_version: MUST_OVERRIDE
           lsst_distrib_dir: /cvmfs/sw.lsst.eu/linux-x86_64/lsst_distrib
 # Campagin Template for fixed inputs
@@ -189,10 +185,6 @@
                     inputs: ["{campaign_output}", "{campaign_input}"]
                     output: "{campaign_public_output}"
       data:
-          bps_core_script_template: stack_script_template
-          bps_core_yaml_template: bps_core_yaml_template
-          manifest_script_template: stack_script_template
-          resource_usage_script_template: stack_script_template
           lsst_version: MUST_OVERRIDE
           lsst_distrib_dir: /cvmfs/sw.lsst.eu/linux-x86_64/lsst_distrib
 # Campaign template for base campaign+hips maps
@@ -263,11 +255,6 @@
                     inputs: ["{campaign_output}", "{campaign_input}", "{campaign_resource_usage}", "{campaign_hips_maps}"]
                     output: "{campaign_public_output}"
       data:
-          bps_core_script_template: stack_script_template
-          bps_core_yaml_template: bps_core_yaml_template
-          manifest_script_template: stack_script_template
-          resource_usage_script_template: stack_script_template
-          hips_maps_script_template: stack_script_template
           lsst_version: MUST_OVERRIDE
           lsst_distrib_dir: /cvmfs/sw.lsst.eu/linux-x86_64/lsst_distrib
 # Campaign Template for fixed inputs which also generates HIPS maps
@@ -315,10 +302,5 @@
                     inputs: ["{campaign_output}", "{campaign_input}", "{campaign_resource_usage}", "{campaign_hips_maps}"]
                     output: "{campaign_public_output}"
       data:
-          bps_core_script_template: stack_script_template
-          bps_core_yaml_template: bps_core_yaml_template
-          manifest_script_template: stack_script_template
-          resource_usage_script_template: stack_script_template
-          hips_maps_script_template: stack_script_template
           lsst_version: MUST_OVERRIDE
           lsst_distrib_dir: /cvmfs/sw.lsst.eu/linux-x86_64/lsst_distrib

examples/example_standard_scripts.yaml

@@ -1,15 +1,3 @@
-# Template shell script fragment for running BPS under PANDA
-- ScriptTemplate:
-      name: bps_panda_script_template
-      file_path: ${CM_CONFIGS}/templates/example_bps_panda_script_template.yaml
-# Template shell script fragment for using stack
-- ScriptTemplate:
-      name: stack_script_template
-      file_path: ${CM_CONFIGS}/templates/example_stack_script_template.yaml
-# Template yaml config for running bps
-- ScriptTemplate:
-      name: bps_core_yaml_template
-      file_path: ${CM_CONFIGS}/templates/example_bps_core_yaml_template.yaml
 # define the null Script
 - SpecBlock:
       name: null_script
@@ -61,8 +49,8 @@
       name: bps_panda_submit_script
       handler: lsst.cmservice.handlers.jobs.PandaScriptHandler
       data:
+          wms: panda
           bps_wms_yaml_file: "${CTRL_BPS_PANDA_DIR}/config/bps_usdf.yaml"
-          bps_wms_script_template: bps_panda_script_template
 # Run a bps report script
 - SpecBlock:
       name: bps_panda_report_script
@@ -72,7 +60,7 @@
       name: bps_htcondor_submit_script
       handler: lsst.cmservice.handlers.jobs.HTCondorScriptHandler
       data:
-          bps_wms_yaml_file: "${CM_CONFIGS}/stack_files/bps_htcondor_usdf.yaml"
+          wms: htcondor
 # Run a bps report script
 - SpecBlock:
       name: bps_htcondor_report_script

src/lsst/cmservice/common/bash.py

@@ -7,6 +7,7 @@
 import yaml
 from anyio import Path, open_file, open_process
 from anyio.streams.text import TextReceiveStream
+from jinja2 import Environment, PackageLoader
 
 from ..config import config
 from .enums import StatusEnum
@@ -169,6 +170,7 @@ async def check_stamp_file(
 async def write_bash_script(
     script_url: str | Path,
     command: str,
+    values: dict,
     **kwargs: Any,
 ) -> Path:
     """Utility function to write a bash script for later execution.
@@ -181,34 +183,28 @@ async def write_bash_script(
     command: `str`
         Main command line(s) in the script
 
+    values: `dict`
+        Mapping of potential template variables to values.
+
     Keywords
     --------
-    prepend: `str | None`
-        Text to prepend before command
-
-    append: `str | None`
-        Test to append after command
-
-    stamp: `str | None`
-        Text to echo to stamp file when script completes
-
-    stamp_url: `str | None`
-        Stamp file to write to when script completes
-
     fake: `str | None`
         Echo command instead of running it
 
     rollback: `str | Path | None`
         Prefix to script_url used when rolling back
         processing. Will default to CWD (".").
 
+
     Returns
     -------
     script_url : `anyio.Path`
         The path to the newly written script
     """
-    prepend = kwargs.get("prepend")
-    append = kwargs.get("append")
+    # Get the yaml template using package lookup
+    template_environment = Environment(loader=PackageLoader("lsst.cmservice"))
+    bash_template = template_environment.get_template("wms_submit_sh.j2")
+
     fake = kwargs.get("fake")
     rollback_prefix = Path(kwargs.get("rollback", "."))
 
@@ -218,8 +214,17 @@ async def write_bash_script(
         command = f"echo '{command}'"
 
     await script_path.parent.mkdir(parents=True, exist_ok=True)
-    contents = (prepend if prepend else "") + "\n" + command + "\n" + (append if append else "")
 
-    async with await open_file(script_path, "w") as fout:
-        await fout.write(contents)
+    template_values = {
+        "command": command,
+        **values,
+    }
+
+    try:
+        # Render bash script template to `script_path`
+        bash_output = bash_template.render(template_values)
+        await Path(script_path).write_text(bash_output)
+    except Exception as e:
+        raise RuntimeError(f"Error writing a script to run BPS job {script_url}; threw {e}")
+
     return script_path

src/lsst/cmservice/handlers/jobs.py

@@ -24,6 +24,7 @@
     CMMissingScriptInputError,
     test_type_and_raise,
 )
+from ..common.logging import LOGGER
 from ..config import config
 from ..db.element import ElementMixin
 from ..db.job import Job
@@ -47,6 +48,8 @@
     WmsStates.PRUNED: TaskStatusEnum.failed,
 }
 
+logger = LOGGER.bind(module=__name__)
+
 
 class BpsScriptHandler(ScriptHandler):
     """Write a script to run bps jobs
@@ -62,20 +65,15 @@ async def _write_script(
         parent: ElementMixin,
         **kwargs: Any,
     ) -> StatusEnum:
-        specification = await script.get_specification(session)
         resolved_cols = await script.resolve_collections(session)
         data_dict = await script.data_dict(session)
         try:
             prod_area = os.path.expandvars(data_dict["prod_area"])
             butler_repo = os.path.expandvars(data_dict["butler_repo"])
             lsst_version = os.path.expandvars(data_dict["lsst_version"])
-            lsst_distrib_dir = os.path.expandvars(data_dict["lsst_distrib_dir"])
             pipeline_yaml = os.path.expandvars(data_dict["pipeline_yaml"])
             run_coll = resolved_cols["run"]
             input_colls = resolved_cols["inputs"]
-            # bps_core_yaml_template = data_dict["bps_core_yaml_template"]
-            bps_core_script_template = data_dict["bps_core_script_template"]
-            bps_wms_script_template = data_dict["bps_wms_script_template"]
         except KeyError as msg:
             raise CMMissingScriptInputError(f"{script.fullname} missing an input: {msg}") from msg
 
@@ -100,19 +98,6 @@ async def _write_script(
         ).resolve()
         log_url = await Path(os.path.expandvars(f"{prod_area}/{script.fullname}.log")).resolve()
 
-        # get the requested templates
-        bps_core_script_template_ = await specification.get_script_template(
-            session,
-            bps_core_script_template,
-        )
-        bps_wms_script_template_ = await specification.get_script_template(
-            session,
-            bps_wms_script_template,
-        )
-
-        # Template rendering
-        # - config_url <- output of rendered bps_submit_yaml.j2
-
         config_path = await Path(config_url).resolve()
         submit_path = await Path(f"{prod_area}/{parent.fullname}/submit").resolve()
         # Clean up any existing artifacts in the target submit_path
@@ -124,14 +109,7 @@ async def _write_script(
         # build up the bps wrapper script
         command = f"{config.bps.bps_bin} --log-file {json_url} --no-log-tty submit {config_path} > {log_url}"
 
-        prepend = bps_core_script_template_.data["text"].replace("{lsst_version}", lsst_version)  # type: ignore
-        prepend = prepend.replace("{lsst_distrib_dir}", lsst_distrib_dir)
-        # Add custom_lsst_setup to the bps submit script
-        # in case it is a change to bps itself
-        if custom_lsst_setup:  # pragma: no cover
-            prepend += f"\n{custom_lsst_setup}\n"
-        prepend += bps_wms_script_template_.data["text"]  # type: ignore
-        await write_bash_script(script_url, command, prepend=prepend)
+        await write_bash_script(script_url, command, values=data_dict)
 
         # Collect values for and render bps submit yaml from template
         await session.refresh(parent, attribute_names=["c_", "p_"])
@@ -181,9 +159,6 @@ async def _write_script(
         workflow_config["include_configs"] = include_configs
         # \\\\\\\\ INCLUDE_CONFIGS REGION
 
-        # //////// CLUSTERING_CONFIG REGION
-        # \\\\\\\\ CLUSTERING_CONFIG REGION
-
         # //////// PAYLOAD REGION
         if isinstance(input_colls, list):  # pragma: no cover
             in_collection = ",".join(input_colls)
@@ -491,37 +466,21 @@ async def _write_script(
         parent: ElementMixin,
         **kwargs: Any,
     ) -> StatusEnum:
-        specification = await script.get_specification(session)
         resolved_cols = await script.resolve_collections(session)
         data_dict = await script.data_dict(session)
         prod_area = os.path.expandvars(data_dict["prod_area"])
         script_url = await self._set_script_files(session, script, prod_area)
         butler_repo = data_dict["butler_repo"]
-        lsst_distrib_dir = data_dict["lsst_distrib_dir"]
-        lsst_version = data_dict["lsst_version"]
         job_run_coll = resolved_cols["job_run"]
         qgraph_file = f"{job_run_coll}.qgraph".replace("/", "_")
 
         graph_url = await Path(f"{prod_area}/{parent.fullname}/submit/{qgraph_file}").resolve()
         report_url = await Path(f"{prod_area}/{parent.fullname}/submit/manifest_report.yaml").resolve()
 
-        manifest_script_template = await specification.get_script_template(
-            session,
-            data_dict["manifest_script_template"],
-        )
-        prepend = manifest_script_template.data["text"].replace("{lsst_version}", lsst_version)  # type: ignore
-        prepend = prepend.replace("{lsst_distrib_dir}", lsst_distrib_dir)
-        if "custom_lsst_setup" in data_dict:  # pragma: no cover
-            custom_lsst_setup = data_dict["custom_lsst_setup"]
-            prepend += f"\n{custom_lsst_setup}"
-
-        # Strip leading/trailing spaces just in case
-        prepend = "\n".join([line.strip() for line in prepend.splitlines()])
-
         command = (
             f"{config.bps.pipetask_bin} report --full-output-filename {report_url} {butler_repo} {graph_url}"
         )
-        await write_bash_script(script_url, command, prepend=prepend)
+        await write_bash_script(script_url, command, values=data_dict)
 
         return StatusEnum.prepared
 

src/lsst/cmservice/handlers/script_handler.py

@@ -420,13 +420,6 @@ async def _check_htcondor_job(
         await script.update_values(session, status=status)
         return status
 
-    @staticmethod
-    def _prepend_htcondor_job(*, setup_stack: bool = False) -> str:
-        prepend = "#!/usr/bin/env bash\n"
-        if setup_stack:
-            prepend += "source ${LSST_DISTRIB_DIR}/${LSST_VERSION}/loadLSST.bash\nsetup lsst_distrib\n"
-        return prepend
-
     async def prepare(
         self,
         session: async_scoped_session,