diff --git a/examples/empty_config.yaml b/examples/empty_config.yaml index 8f7febeae..49a0bd4f1 100644 --- a/examples/empty_config.yaml +++ b/examples/empty_config.yaml @@ -29,9 +29,6 @@ prod_area: 'output/archive' data_query: "instrument = 'HSC'" lsst_version: "${WEEKLY}" - bps_script_template: bps_panda_script_template - bps_yaml_template: bps_yaml_template - manifest_script_template: manifest_script_template - Specification: name: base spec_aliases: diff --git a/examples/example_standard_elements.yaml b/examples/example_standard_elements.yaml index b9e122905..2f54e86e6 100644 --- a/examples/example_standard_elements.yaml +++ b/examples/example_standard_elements.yaml @@ -145,10 +145,6 @@ inputs: ["{campaign_output}", "{campaign_input}", "{campaign_resource_usage}"] output: "{campaign_public_output}" data: - bps_core_script_template: stack_script_template - bps_core_yaml_template: bps_core_yaml_template - manifest_script_template: stack_script_template - resource_usage_script_template: stack_script_template lsst_version: MUST_OVERRIDE lsst_distrib_dir: /cvmfs/sw.lsst.eu/linux-x86_64/lsst_distrib # Campagin Template for fixed inputs @@ -189,10 +185,6 @@ inputs: ["{campaign_output}", "{campaign_input}"] output: "{campaign_public_output}" data: - bps_core_script_template: stack_script_template - bps_core_yaml_template: bps_core_yaml_template - manifest_script_template: stack_script_template - resource_usage_script_template: stack_script_template lsst_version: MUST_OVERRIDE lsst_distrib_dir: /cvmfs/sw.lsst.eu/linux-x86_64/lsst_distrib # Campaign template for base campaign+hips maps @@ -263,11 +255,6 @@ inputs: ["{campaign_output}", "{campaign_input}", "{campaign_resource_usage}", "{campaign_hips_maps}"] output: "{campaign_public_output}" data: - bps_core_script_template: stack_script_template - bps_core_yaml_template: bps_core_yaml_template - manifest_script_template: stack_script_template - resource_usage_script_template: stack_script_template - hips_maps_script_template: stack_script_template lsst_version: MUST_OVERRIDE lsst_distrib_dir: /cvmfs/sw.lsst.eu/linux-x86_64/lsst_distrib # Campaign Template for fixed inputs which also generates HIPS maps @@ -315,10 +302,5 @@ inputs: ["{campaign_output}", "{campaign_input}", "{campaign_resource_usage}", "{campaign_hips_maps}"] output: "{campaign_public_output}" data: - bps_core_script_template: stack_script_template - bps_core_yaml_template: bps_core_yaml_template - manifest_script_template: stack_script_template - resource_usage_script_template: stack_script_template - hips_maps_script_template: stack_script_template lsst_version: MUST_OVERRIDE lsst_distrib_dir: /cvmfs/sw.lsst.eu/linux-x86_64/lsst_distrib diff --git a/examples/example_standard_scripts.yaml b/examples/example_standard_scripts.yaml index 99e59122f..8f57b6237 100644 --- a/examples/example_standard_scripts.yaml +++ b/examples/example_standard_scripts.yaml @@ -1,15 +1,3 @@ -# Template shell script fragment for running BPS under PANDA -- ScriptTemplate: - name: bps_panda_script_template - file_path: ${CM_CONFIGS}/templates/example_bps_panda_script_template.yaml -# Template shell script fragment for using stack -- ScriptTemplate: - name: stack_script_template - file_path: ${CM_CONFIGS}/templates/example_stack_script_template.yaml -# Template yaml config for running bps -- ScriptTemplate: - name: bps_core_yaml_template - file_path: ${CM_CONFIGS}/templates/example_bps_core_yaml_template.yaml # define the null Script - SpecBlock: name: null_script @@ -61,8 +49,8 @@ name: bps_panda_submit_script handler: lsst.cmservice.handlers.jobs.PandaScriptHandler data: + wms: panda bps_wms_yaml_file: "${CTRL_BPS_PANDA_DIR}/config/bps_usdf.yaml" - bps_wms_script_template: bps_panda_script_template # Run a bps report script - SpecBlock: name: bps_panda_report_script @@ -72,7 +60,7 @@ name: bps_htcondor_submit_script handler: lsst.cmservice.handlers.jobs.HTCondorScriptHandler data: - bps_wms_yaml_file: "${CM_CONFIGS}/stack_files/bps_htcondor_usdf.yaml" + wms: htcondor # Run a bps report script - SpecBlock: name: bps_htcondor_report_script diff --git a/examples/templates/example_bps_core_yaml_template.yaml b/examples/templates/example_bps_core_yaml_template.yaml deleted file mode 100644 index b56e35813..000000000 --- a/examples/templates/example_bps_core_yaml_template.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# turn off the -o option in pipetask commands -pipetaskOutput: "" - -executionButler: - requestMemory: 64000 - queue: "SLAC_Rubin_Merge" - # turn off chaining in the execution butler because - # no output collection - command2: "" - command3: "" diff --git a/examples/templates/example_bps_panda_script_template.yaml b/examples/templates/example_bps_panda_script_template.yaml deleted file mode 100644 index 09776db7c..000000000 --- a/examples/templates/example_bps_panda_script_template.yaml +++ /dev/null @@ -1,10 +0,0 @@ -text: "# setup PanDA env.\n -latest_panda=$(ls -td /cvmfs/sw.lsst.eu/linux-x86_64/panda_env/v* | head -1)\n -setupScript=${latest_panda}/setup_panda_usdf.sh\n -source $setupScript ${WEEKLY}\n - -env | grep PANDA\n - -# let's drop a panda_auth status here for kicks\n -panda_auth status\n -" diff --git a/examples/templates/example_stack_script_template.yaml b/examples/templates/example_stack_script_template.yaml deleted file mode 100644 index c40025a82..000000000 --- a/examples/templates/example_stack_script_template.yaml +++ /dev/null @@ -1,9 +0,0 @@ -text: "#!/usr/bin/env bash\n - -# setup LSST env.\n -export LSST_VERSION='{lsst_version}'\n -export LSST_DISTRIB_DIR='{lsst_distrib_dir}'\n -source ${LSST_DISTRIB_DIR}/${LSST_VERSION}/loadLSST.bash\n -setup lsst_distrib\n - -" diff --git a/src/lsst/cmservice/common/bash.py b/src/lsst/cmservice/common/bash.py index 332ef6f0f..866da8dbb 100644 --- a/src/lsst/cmservice/common/bash.py +++ b/src/lsst/cmservice/common/bash.py @@ -7,6 +7,7 @@ import yaml from anyio import Path, open_file, open_process from anyio.streams.text import TextReceiveStream +from jinja2 import Environment, PackageLoader from ..config import config from .enums import StatusEnum @@ -169,6 +170,7 @@ async def check_stamp_file( async def write_bash_script( script_url: str | Path, command: str, + values: dict, **kwargs: Any, ) -> Path: """Utility function to write a bash script for later execution. @@ -181,20 +183,11 @@ async def write_bash_script( command: `str` Main command line(s) in the script + values: `dict` + Mapping of potential template variables to values. + Keywords -------- - prepend: `str | None` - Text to prepend before command - - append: `str | None` - Test to append after command - - stamp: `str | None` - Text to echo to stamp file when script completes - - stamp_url: `str | None` - Stamp file to write to when script completes - fake: `str | None` Echo command instead of running it @@ -202,13 +195,16 @@ async def write_bash_script( Prefix to script_url used when rolling back processing. Will default to CWD ("."). + Returns ------- script_url : `anyio.Path` The path to the newly written script """ - prepend = kwargs.get("prepend") - append = kwargs.get("append") + # Get the yaml template using package lookup + template_environment = Environment(loader=PackageLoader("lsst.cmservice")) + bash_template = template_environment.get_template("wms_submit_sh.j2") + fake = kwargs.get("fake") rollback_prefix = Path(kwargs.get("rollback", ".")) @@ -218,8 +214,17 @@ async def write_bash_script( command = f"echo '{command}'" await script_path.parent.mkdir(parents=True, exist_ok=True) - contents = (prepend if prepend else "") + "\n" + command + "\n" + (append if append else "") - async with await open_file(script_path, "w") as fout: - await fout.write(contents) + template_values = { + "command": command, + **values, + } + + try: + # Render bash script template to `script_path` + bash_output = bash_template.render(template_values) + await Path(script_path).write_text(bash_output) + except Exception as e: + raise RuntimeError(f"Error writing a script to run BPS job {script_url}; threw {e}") + return script_path diff --git a/src/lsst/cmservice/handlers/jobs.py b/src/lsst/cmservice/handlers/jobs.py index e9527b06c..31ac01f3b 100644 --- a/src/lsst/cmservice/handlers/jobs.py +++ b/src/lsst/cmservice/handlers/jobs.py @@ -24,6 +24,7 @@ CMMissingScriptInputError, test_type_and_raise, ) +from ..common.logging import LOGGER from ..config import config from ..db.element import ElementMixin from ..db.job import Job @@ -47,6 +48,8 @@ WmsStates.PRUNED: TaskStatusEnum.failed, } +logger = LOGGER.bind(module=__name__) + class BpsScriptHandler(ScriptHandler): """Write a script to run bps jobs @@ -62,20 +65,15 @@ async def _write_script( parent: ElementMixin, **kwargs: Any, ) -> StatusEnum: - specification = await script.get_specification(session) resolved_cols = await script.resolve_collections(session) data_dict = await script.data_dict(session) try: prod_area = os.path.expandvars(data_dict["prod_area"]) butler_repo = os.path.expandvars(data_dict["butler_repo"]) lsst_version = os.path.expandvars(data_dict["lsst_version"]) - lsst_distrib_dir = os.path.expandvars(data_dict["lsst_distrib_dir"]) pipeline_yaml = os.path.expandvars(data_dict["pipeline_yaml"]) run_coll = resolved_cols["run"] input_colls = resolved_cols["inputs"] - # bps_core_yaml_template = data_dict["bps_core_yaml_template"] - bps_core_script_template = data_dict["bps_core_script_template"] - bps_wms_script_template = data_dict["bps_wms_script_template"] except KeyError as msg: raise CMMissingScriptInputError(f"{script.fullname} missing an input: {msg}") from msg @@ -100,19 +98,6 @@ async def _write_script( ).resolve() log_url = await Path(os.path.expandvars(f"{prod_area}/{script.fullname}.log")).resolve() - # get the requested templates - bps_core_script_template_ = await specification.get_script_template( - session, - bps_core_script_template, - ) - bps_wms_script_template_ = await specification.get_script_template( - session, - bps_wms_script_template, - ) - - # Template rendering - # - config_url <- output of rendered bps_submit_yaml.j2 - config_path = await Path(config_url).resolve() submit_path = await Path(f"{prod_area}/{parent.fullname}/submit").resolve() # Clean up any existing artifacts in the target submit_path @@ -124,14 +109,7 @@ async def _write_script( # build up the bps wrapper script command = f"{config.bps.bps_bin} --log-file {json_url} --no-log-tty submit {config_path} > {log_url}" - prepend = bps_core_script_template_.data["text"].replace("{lsst_version}", lsst_version) # type: ignore - prepend = prepend.replace("{lsst_distrib_dir}", lsst_distrib_dir) - # Add custom_lsst_setup to the bps submit script - # in case it is a change to bps itself - if custom_lsst_setup: # pragma: no cover - prepend += f"\n{custom_lsst_setup}\n" - prepend += bps_wms_script_template_.data["text"] # type: ignore - await write_bash_script(script_url, command, prepend=prepend) + await write_bash_script(script_url, command, values=data_dict) # Collect values for and render bps submit yaml from template await session.refresh(parent, attribute_names=["c_", "p_"]) @@ -181,9 +159,6 @@ async def _write_script( workflow_config["include_configs"] = include_configs # \\\\\\\\ INCLUDE_CONFIGS REGION - # //////// CLUSTERING_CONFIG REGION - # \\\\\\\\ CLUSTERING_CONFIG REGION - # //////// PAYLOAD REGION if isinstance(input_colls, list): # pragma: no cover in_collection = ",".join(input_colls) @@ -491,37 +466,21 @@ async def _write_script( parent: ElementMixin, **kwargs: Any, ) -> StatusEnum: - specification = await script.get_specification(session) resolved_cols = await script.resolve_collections(session) data_dict = await script.data_dict(session) prod_area = os.path.expandvars(data_dict["prod_area"]) script_url = await self._set_script_files(session, script, prod_area) butler_repo = data_dict["butler_repo"] - lsst_distrib_dir = data_dict["lsst_distrib_dir"] - lsst_version = data_dict["lsst_version"] job_run_coll = resolved_cols["job_run"] qgraph_file = f"{job_run_coll}.qgraph".replace("/", "_") graph_url = await Path(f"{prod_area}/{parent.fullname}/submit/{qgraph_file}").resolve() report_url = await Path(f"{prod_area}/{parent.fullname}/submit/manifest_report.yaml").resolve() - manifest_script_template = await specification.get_script_template( - session, - data_dict["manifest_script_template"], - ) - prepend = manifest_script_template.data["text"].replace("{lsst_version}", lsst_version) # type: ignore - prepend = prepend.replace("{lsst_distrib_dir}", lsst_distrib_dir) - if "custom_lsst_setup" in data_dict: # pragma: no cover - custom_lsst_setup = data_dict["custom_lsst_setup"] - prepend += f"\n{custom_lsst_setup}" - - # Strip leading/trailing spaces just in case - prepend = "\n".join([line.strip() for line in prepend.splitlines()]) - command = ( f"{config.bps.pipetask_bin} report --full-output-filename {report_url} {butler_repo} {graph_url}" ) - await write_bash_script(script_url, command, prepend=prepend) + await write_bash_script(script_url, command, values=data_dict) return StatusEnum.prepared diff --git a/src/lsst/cmservice/handlers/script_handler.py b/src/lsst/cmservice/handlers/script_handler.py index a4ad8bf25..f104410fe 100644 --- a/src/lsst/cmservice/handlers/script_handler.py +++ b/src/lsst/cmservice/handlers/script_handler.py @@ -420,13 +420,6 @@ async def _check_htcondor_job( await script.update_values(session, status=status) return status - @staticmethod - def _prepend_htcondor_job(*, setup_stack: bool = False) -> str: - prepend = "#!/usr/bin/env bash\n" - if setup_stack: - prepend += "source ${LSST_DISTRIB_DIR}/${LSST_VERSION}/loadLSST.bash\nsetup lsst_distrib\n" - return prepend - async def prepare( self, session: async_scoped_session, diff --git a/src/lsst/cmservice/handlers/scripts.py b/src/lsst/cmservice/handlers/scripts.py index d79779f08..52e5de6c2 100644 --- a/src/lsst/cmservice/handlers/scripts.py +++ b/src/lsst/cmservice/handlers/scripts.py @@ -48,7 +48,7 @@ async def _write_script( raise CMMissingScriptInputError(f"{script.fullname} missing an input: {e}") from e command = f"echo trivial {butler_repo} {output_coll}" - await write_bash_script(script_url, command, prepend="#!/usr/bin/env bash\n", **data_dict) + await write_bash_script(script_url, command, values=data_dict) await script.update_values(session, script_url=script_url, status=StatusEnum.prepared) return StatusEnum.prepared @@ -83,7 +83,6 @@ async def _write_script( parent: ElementMixin, **kwargs: Any, ) -> StatusEnum: - setup_stack = kwargs.get("setup_stack", False) resolved_cols = await script.resolve_collections(session) data_dict = await script.data_dict(session) try: @@ -93,7 +92,6 @@ async def _write_script( butler_repo = data_dict["butler_repo"] except KeyError as msg: raise CMMissingScriptInputError(f"{script.fullname} missing an input: {msg}") from msg - prepend = self._prepend_htcondor_job(setup_stack=setup_stack) command = f"{config.butler.butler_bin} collection-chain {butler_repo} {output_coll}" # This is here out of paranoia. # script.resolved_collections should convert the list to a string @@ -102,7 +100,7 @@ async def _write_script( command += f" {input_coll}" else: command += f" {input_colls}" - await write_bash_script(script_url, command, prepend=prepend, **data_dict) + await write_bash_script(script_url, command, values=data_dict) await script.update_values(session, script_url=script_url, status=StatusEnum.prepared) return StatusEnum.prepared @@ -144,7 +142,6 @@ async def _write_script( parent: ElementMixin, **kwargs: Any, ) -> StatusEnum: - setup_stack = kwargs.get("setup_stack", False) resolved_cols = await script.resolve_collections(session) data_dict = await script.data_dict(session) try: @@ -154,12 +151,11 @@ async def _write_script( butler_repo = data_dict["butler_repo"] except KeyError as msg: raise CMMissingScriptInputError(f"{script.fullname} missing an input: {msg}") from msg - prepend = self._prepend_htcondor_job(setup_stack=setup_stack) command = ( f"{config.butler.butler_bin} collection-chain " f"{butler_repo} {output_coll} --mode prepend {input_coll}" ) - await write_bash_script(script_url, command, prepend=prepend, **data_dict) + await write_bash_script(script_url, command, values=data_dict) await script.update_values(session, script_url=script_url, status=StatusEnum.prepared) return StatusEnum.prepared @@ -202,7 +198,6 @@ async def _write_script( parent: ElementMixin, **kwargs: Any, ) -> StatusEnum: - setup_stack = kwargs.get("setup_stack", False) resolved_cols = await script.resolve_collections(session) data_dict = await script.data_dict(session) try: @@ -228,13 +223,12 @@ async def _write_script( ) script_url = await self._set_script_files(session, script, data_dict["prod_area"]) butler_repo = data_dict["butler_repo"] - prepend = self._prepend_htcondor_job(setup_stack=setup_stack) command = f"{config.butler.butler_bin} collection-chain {butler_repo} {output_coll}" for collect_coll_ in collect_colls: command += f" {collect_coll_}" for input_coll_ in input_colls: command += f" {input_coll_}" - await write_bash_script(script_url, command, prepend=prepend, **data_dict) + await write_bash_script(script_url, command, values=data_dict) await script.update_values(session, script_url=script_url, status=StatusEnum.prepared) return StatusEnum.prepared @@ -276,7 +270,6 @@ async def _write_script( parent: ElementMixin, **kwargs: Any, ) -> StatusEnum: - setup_stack = kwargs.get("setup_stack", False) resolved_cols = await script.resolve_collections(session) data_dict = await script.data_dict(session) try: @@ -287,11 +280,10 @@ async def _write_script( data_query = data_dict.get("data_query") except KeyError as msg: raise CMMissingScriptInputError(f"{script.fullname} missing an input: {msg}") from msg - prepend = self._prepend_htcondor_job(setup_stack=setup_stack) command = f"{config.butler.butler_bin} associate {butler_repo} {output_coll}" command += f" --collections {input_coll}" command += f' --where "{data_query}"' if data_query else "" - await write_bash_script(script_url, command, prepend=prepend, **data_dict) + await write_bash_script(script_url, command, values=data_dict) await script.update_values(session, script_url=script_url, status=StatusEnum.prepared) return StatusEnum.prepared @@ -329,7 +321,6 @@ async def _write_script( parent: ElementMixin, **kwargs: Any, ) -> StatusEnum: - setup_stack = kwargs.get("setup_stack", False) resolved_cols = await script.resolve_collections(session) data_dict = await script.data_dict(session) try: @@ -338,9 +329,8 @@ async def _write_script( butler_repo = data_dict["butler_repo"] except KeyError as msg: raise CMMissingScriptInputError(f"{script.fullname} missing an input: {msg}") from msg - prepend = self._prepend_htcondor_job(setup_stack=setup_stack) command = f"{config.butler.butler_bin} associate {butler_repo} {output_coll}" - await write_bash_script(script_url, command, prepend=prepend, **data_dict) + await write_bash_script(script_url, command, values=data_dict) await script.update_values(session, status=StatusEnum.prepared) return StatusEnum.prepared @@ -380,7 +370,6 @@ async def _write_script( parent: ElementMixin, **kwargs: Any, ) -> StatusEnum: - setup_stack = kwargs.get("setup_stack", False) resolved_cols = await script.resolve_collections(session) data_dict = await script.data_dict(session) try: @@ -390,10 +379,9 @@ async def _write_script( butler_repo = data_dict["butler_repo"] except KeyError as msg: raise CMMissingScriptInputError(f"{script.fullname} missing an input: {msg}") from msg - prepend = self._prepend_htcondor_job(setup_stack=setup_stack) command = f"{config.butler.butler_bin} associate {butler_repo} {output_coll}" command += f" --collections {input_coll}" - await write_bash_script(script_url, command, prepend=prepend, **data_dict) + await write_bash_script(script_url, command, values=data_dict) await script.update_values(session, script_url=script_url, status=StatusEnum.prepared) return StatusEnum.prepared @@ -443,7 +431,6 @@ async def _write_script( ) -> StatusEnum: test_type_and_raise(parent, Step, "PrepareStepScriptHandler._write_script parent") - setup_stack = kwargs.get("setup_stack", False) resolved_cols = await script.resolve_collections(session) data_dict = await script.data_dict(session) try: @@ -463,11 +450,10 @@ async def _write_script( if not prereq_colls: prereq_colls.append(resolved_cols["global_inputs"]) - prepend = self._prepend_htcondor_job(setup_stack=setup_stack) command = f"{config.butler.butler_bin} collection-chain {butler_repo} {output_coll}" for prereq_coll_ in prereq_colls: command += f" {prereq_coll_}" - await write_bash_script(script_url, command, prepend=prepend, **data_dict) + await write_bash_script(script_url, command, values=data_dict) await script.update_values(session, script_url=script_url, status=StatusEnum.prepared) return StatusEnum.prepared @@ -500,32 +486,13 @@ async def _write_script( parent: ElementMixin, **kwargs: Any, ) -> StatusEnum: - specification = await script.get_specification(session) resolved_cols = await script.resolve_collections(session) data_dict = await script.data_dict(session) prod_area = os.path.expandvars(data_dict["prod_area"]) script_url = await self._set_script_files(session, script, prod_area) butler_repo = data_dict["butler_repo"] - lsst_distrib_dir = data_dict["lsst_distrib_dir"] - lsst_version = data_dict["lsst_version"] usage_graph_url = os.path.expandvars(f"{prod_area}/{parent.fullname}/resource_usage.qgraph") - resource_usage_script_template = await specification.get_script_template( - session, - data_dict["resource_usage_script_template"], - ) - prepend = resource_usage_script_template.data["text"].replace( # type: ignore - "{lsst_version}", - lsst_version, - ) - prepend = prepend.replace("{lsst_distrib_dir}", lsst_distrib_dir) - if "custom_lsst_setup" in data_dict: # pragma: no cover - custom_lsst_setup = data_dict["custom_lsst_setup"] - prepend += f"\n{custom_lsst_setup}" - - # Strip leading/trailing spaces just in case - prepend = "\n".join([line.strip() for line in prepend.splitlines()]) - command = ( f"{config.bps.resource_usage_bin} {butler_repo} {usage_graph_url} " f"{resolved_cols['campaign_output']} --output {resolved_cols['campaign_resource_usage']};" @@ -533,7 +500,7 @@ async def _write_script( f"-o {resolved_cols['campaign_resource_usage']} --register-dataset-types -j {config.bps.n_jobs}" ) - await write_bash_script(script_url, command, prepend=prepend) + await write_bash_script(script_url, command, values=data_dict) return StatusEnum.prepared @@ -571,32 +538,13 @@ async def _write_script( parent: ElementMixin, **kwargs: Any, ) -> StatusEnum: - specification = await script.get_specification(session) resolved_cols = await script.resolve_collections(session) data_dict = await script.data_dict(session) prod_area = os.path.expandvars(data_dict["prod_area"]) script_url = await self._set_script_files(session, script, prod_area) butler_repo = data_dict["butler_repo"] - lsst_distrib_dir = data_dict["lsst_distrib_dir"] - lsst_version = data_dict["lsst_version"] hips_maps_graph_url = os.path.expandvars(f"{prod_area}/{parent.fullname}/hips_maps.qgraph") - hips_maps_script_template = await specification.get_script_template( - session, - data_dict["hips_maps_script_template"], - ) - prepend = hips_maps_script_template.data["text"].replace( # type: ignore - "{lsst_version}", - lsst_version, - ) - prepend = prepend.replace("{lsst_distrib_dir}", lsst_distrib_dir) - if "custom_lsst_setup" in data_dict: # pragma: no cover - custom_lsst_setup = data_dict["custom_lsst_setup"] - prepend += f"\n{custom_lsst_setup}" - - # Strip leading/trailing spaces just in case - prepend = "\n".join([line.strip() for line in prepend.splitlines()]) - hips_pipeline_yaml = await Path( os.path.expandvars("${CM_CONFIGS}") + data_dict["hips_pipeline_yaml_path"] ).resolve() @@ -639,7 +587,7 @@ async def _write_script( # Strip leading/trailing spaces just in case command = "\n".join([line.strip() for line in command.splitlines()]) - await write_bash_script(script_url, command, prepend=prepend) + await write_bash_script(script_url, command, values=data_dict) return StatusEnum.prepared @@ -684,7 +632,6 @@ async def _write_script( parent: ElementMixin, **kwargs: Any, ) -> StatusEnum: - setup_stack = kwargs.get("setup_stack", False) resolved_cols = await script.resolve_collections(session) data_dict = await script.data_dict(session) try: @@ -694,9 +641,8 @@ async def _write_script( butler_repo = data_dict["butler_repo"] except KeyError as msg: raise CMMissingScriptInputError(f"{script.fullname} missing an input: {msg}") from msg - prepend = self._prepend_htcondor_job(setup_stack=setup_stack) command = f"{config.bps.pipetask_bin} validate {butler_repo} {input_coll} {output_coll}" - await write_bash_script(script_url, command, prepend=prepend, **data_dict) + await write_bash_script(script_url, command, values=data_dict) await script.update_values(session, script_url=script_url, status=StatusEnum.prepared) return StatusEnum.prepared diff --git a/src/lsst/cmservice/templates/bps_submit_yaml.j2 b/src/lsst/cmservice/templates/bps_submit_yaml.j2 index ca9bfd483..998da66f6 100644 --- a/src/lsst/cmservice/templates/bps_submit_yaml.j2 +++ b/src/lsst/cmservice/templates/bps_submit_yaml.j2 @@ -37,8 +37,9 @@ extraQgraphOptions: {{ extra_qgraph_options | replace("\n", " ") | trim }} {{ yaml_literal }} {%- endfor %} {%- endif %} -{%- if compute_site == "usdf" %} {%- if wms == "htcondor" %} +wmsServiceClass: lsst.ctrl.bps.htcondor.HTCondorService +{%- if compute_site == "usdf" %} site: s3df: profile: @@ -47,6 +48,5 @@ site: memoryMultiplier: 4. numberOfRetries: 3 memoryLimit: 400000 -wmsServiceClass: lsst.ctrl.bps.htcondor.HTCondorService {%- endif %} {%- endif %} diff --git a/src/lsst/cmservice/templates/wms_submit_sh.j2 b/src/lsst/cmservice/templates/wms_submit_sh.j2 new file mode 100644 index 000000000..bfedeec0a --- /dev/null +++ b/src/lsst/cmservice/templates/wms_submit_sh.j2 @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +{%- if wms == "htcondor" %} + +# setup LSST env. +export LSST_VERSION='{{ lsst_version }}' +export LSST_DISTRIB_DIR='{{ lsst_distrib_dir }}' +source ${LSST_DISTRIB_DIR}/${LSST_VERSION}/loadLSST.bash +setup lsst_distrib +{%- endif %} +{%- if wms == "panda" %} + +# setup PanDA env. +latest_panda=$(ls -td /cvmfs/sw.lsst.eu/linux-x86_64/panda_env/v* | head -1) +source ${latest_panda}/setup_panda_usdf.sh ${WEEKLY} +panda_auth status +{%- endif %} +{{ custom_lsst_setup }} + +{{ command }} + +{%- if append %} + +{{ append }} +{%- endif %} diff --git a/tests/common/test_common.py b/tests/common/test_common.py index 7c3c634f7..b4b925ce6 100644 --- a/tests/common/test_common.py +++ b/tests/common/test_common.py @@ -31,9 +31,8 @@ async def test_common_bash() -> None: the_script = await write_bash_script( "temp.sh", "ls", - prepend="#!/usr/bin/env bash", - append="# have a nice day", fake=True, + values=dict(append="# have a nice day"), ) await run_bash_job(the_script, "temp.log", "temp.stamp") diff --git a/tests/db/test_handlers.py b/tests/db/test_handlers.py index 02182eb20..99048c692 100644 --- a/tests/db/test_handlers.py +++ b/tests/db/test_handlers.py @@ -122,8 +122,6 @@ async def test_handlers_campaign_level_db( data = dict( lsst_distrib_dir="lsst_distrib_dir", - resource_usage_script_template="stack_script_template", - hips_maps_script_template="stack_script_template", hips_pipeline_yaml_path="/stack_files/highres_hips_rc2.yaml", hips_pipeline_config_path="/stack_files/gen_hips_both_rc2.yaml", ) @@ -245,10 +243,6 @@ async def test_handlers_group_level_db( dict( lsst_distrib_dir="lsst_distrib_dir", - bps_core_yaml_template="bps_core_yaml_template", - bps_core_script_template="stack_script_template", - bps_panda_script_template="bps_panda_script_template", - manifest_script_template="stack_script_template", ) run_jobs = await check_run_script(session, group, "run", "run_jobs", collections=collections) @@ -302,10 +296,6 @@ async def test_handlers_job_level_db( data = dict( lsst_distrib_dir="lsst_distrib_dir", - bps_core_yaml_template="bps_core_yaml_template", - bps_core_script_template="stack_script_template", - bps_panda_script_template="bps_panda_script_template", - manifest_script_template="stack_script_template", ) await check_script( diff --git a/tests/db/test_micro.py b/tests/db/test_micro.py index e9c5d4b40..c890dbdb0 100644 --- a/tests/db/test_micro.py +++ b/tests/db/test_micro.py @@ -45,15 +45,6 @@ async def test_micro_db( with pytest.raises(errors.CMSpecificationError): await specification.get_script_template(session, "bad") - script_template = await specification.get_script_template(session, "stack_script_template") - assert script_template.name == "stack_script_template", "Script template name mismatch" - - await script_template.update_from_file( - session, - script_template.name, - "examples/templates/example_stack_script_template.yaml", - ) - campaign = await interface.load_and_create_campaign( session, "examples/example_hsc_micro.yaml",