From 701ef6602d35cf615547ef04f2574e42dec24fdc Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Sun, 25 Aug 2024 23:29:38 -0400 Subject: [PATCH 01/19] feat: add separate callback module Signed-off-by: Jason C. Nucciarone --- slurmutils/models/callback.py | 78 +++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 slurmutils/models/callback.py diff --git a/slurmutils/models/callback.py b/slurmutils/models/callback.py new file mode 100644 index 0000000..8bf9a4d --- /dev/null +++ b/slurmutils/models/callback.py @@ -0,0 +1,78 @@ +# Copyright 2024 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +"""Callbacks for parsing and marshalling Slurm data models.""" + +__all__ = [ + "Callback", + "CommaSeparatorCallback", + "ColonSeparatorCallback", + "SlurmDictCallback", + "ReasonCallback", +] + +from typing import Any, Callable, Dict, NamedTuple, Optional + + +class Callback(NamedTuple): + """Callbacks for parsing and marshalling Slurm data model values. + + Args: + parser: Callback that parses the value as read in from Slurm configuration. + marshaller: Callback that marshals the value back into a valid Slurm configuration value. + """ + + parser: Optional[Callable[[str], Any]] = None + marshaller: Optional[Callable[[Any], str]] = None + + +def from_slurm_dict(value: str) -> Dict[str, Any]: + """Create dictionary from Slurm dictionary. + + Notes: + key=value,key2 -> {"key": "value", "key2": True} + """ + result = {} + for opt in value.split(","): + if "=" not in opt: + result[opt] = True + continue + + k, v = opt.split("=", maxsplit=1) + result[k] = v + + return result + + +def to_slurm_dict(value: Dict[str, Any]) -> str: + """Convert dictionary into Slurm dictionary. + + Notes: + {"key": "value", "key2": True} -> key=value,key2 + """ + result = [] + for k, v in value.items(): + if isinstance(v, bool) and v: + result.append(v) + continue + + result.append(f"{k}={v}") + + return ",".join(result) + + +CommaSeparatorCallback = Callback(lambda v: v.split(","), lambda v: ",".join(v)) +ColonSeparatorCallback = Callback(lambda v: v.split(":"), lambda v: ":".join(v)) +SlurmDictCallback = Callback(from_slurm_dict, to_slurm_dict) +ReasonCallback = Callback(None, lambda v: f'"{v}"') # Ensure that 'Reason=...' is quoted properly. From 41c043e8b347aa1afe6d011dc23d0631c947b800 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Mon, 26 Aug 2024 10:34:03 -0400 Subject: [PATCH 02/19] chore(deps): update tox.ini to use new ruff command syntax Signed-off-by: Jason C. Nucciarone --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 37b1aac..ce0c397 100644 --- a/tox.ini +++ b/tox.ini @@ -35,7 +35,7 @@ deps = ruff commands = black {[vars]all_path} - ruff --fix {[vars]all_path} + ruff check --fix {[vars]all_path} [testenv:lint] description = Check code against coding style standards. @@ -45,7 +45,7 @@ deps = codespell commands = codespell {[vars]all_path} - ruff {[vars]all_path} + ruff check {[vars]all_path} black --check --diff {[vars]all_path} [testenv:unit] From 4a671642c1edf6c1b773363f2306c0f83135ba65 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Mon, 26 Aug 2024 10:35:20 -0400 Subject: [PATCH 03/19] chore(deps): update pyproject.toml to use new ruff syntax Signed-off-by: Jason C. Nucciarone --- pyproject.toml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 50aba78..099b3bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,9 @@ target-version = ["py38"] # Linting tools configuration [tool.ruff] line-length = 99 +extend-exclude = ["__pycache__", "*.egg_info"] + +[tool.ruff.lint] select = ["E", "W", "F", "C", "N", "D", "I001"] extend-ignore = [ "D203", @@ -80,9 +83,6 @@ extend-ignore = [ "D409", "D413", ] -ignore = ["E501", "D105", "D107"] -extend-exclude = ["__pycache__", "*.egg_info", "__init__.py"] +ignore = ["E501", "D107"] per-file-ignores = {"tests/*" = ["D100","D101","D102","D103","D104"]} - -[tool.ruff.mccabe] -max-complexity = 15 +mccabe = { "max-complexity" = 15 } From 8e58a9dc19cf8836f9870689e1739095f4ab8e82 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Mon, 26 Aug 2024 10:41:36 -0400 Subject: [PATCH 04/19] feat: simplify base editor implementation * Changes: * Remove unneccessary sections for rendered config files. Header and sections add unneccessary amount of time to marshalling * Don't compose methods using `functools.partial` and instead use decorator to supply functionality we need * Simplify parsing methods to more straightforward and less complex Signed-off-by: Jason C. Nucciarone --- slurmutils/editors/_editor.py | 229 ---------------------------------- slurmutils/editors/editor.py | 131 +++++++++++++++++++ 2 files changed, 131 insertions(+), 229 deletions(-) delete mode 100644 slurmutils/editors/_editor.py create mode 100644 slurmutils/editors/editor.py diff --git a/slurmutils/editors/_editor.py b/slurmutils/editors/_editor.py deleted file mode 100644 index 634213a..0000000 --- a/slurmutils/editors/_editor.py +++ /dev/null @@ -1,229 +0,0 @@ -# Copyright 2024 Canonical Ltd. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License version 3 as published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . - -"""Base methods for Slurm workload manager configuration file editors.""" - -import logging -import re -import shlex -from collections import deque -from os import PathLike -from pathlib import Path -from typing import Deque, Dict, List, Optional, Set, Union - -from slurmutils.exceptions import EditorError - -_logger = logging.getLogger(__name__) - - -def dump_base(content, file: Union[str, PathLike], marshaller): - """Dump configuration into file using provided marshalling function. - - Do not use this function directly. - """ - if (loc := Path(file)).exists(): - _logger.warning("Overwriting contents of %s file located at %s.", loc.name, loc) - - _logger.debug("Marshalling configuration into %s file located at %s.", loc.name, loc) - return loc.write_text(marshaller(content), encoding="ascii") - - -def dumps_base(content, marshaller) -> str: - """Dump configuration into Python string using provided marshalling function. - - Do not use this function directly. - """ - return marshaller(content) - - -def load_base(file: Union[str, PathLike], parser): - """Load configuration from file using provided parsing function. - - Do not use this function directly. - """ - if (file := Path(file)).exists(): - _logger.debug("Parsing contents of %s located at %s.", file.name, file) - config = file.read_text(encoding="ascii") - return parser(config) - else: - msg = "Unable to locate file" - _logger.error(msg + " %s.", file) - raise FileNotFoundError(msg + f" {file}") - - -def loads_base(content: str, parser): - """Load configuration from Python String using provided parsing function. - - Do not use this function directly. - """ - return parser(content) - - -# Helper functions for parsing and marshalling Slurm configuration data. - -_loose_pascal_filter = re.compile(r"(.)([A-Z][a-z]+)") -_snakecase_convert = re.compile(r"([a-z0-9])([A-Z])") - - -def _pascal2snake(v: str) -> str: - """Convert string in loose PascalCase to snakecase. - - This private method takes in Slurm configuration knob keys and converts - them to snakecase. The returned snakecase representation is used to - dynamically access Slurm data model attributes and retrieve callbacks. - """ - # The precompiled regex filters do a good job of converting Slurm's - # loose PascalCase to snakecase, however, there are still some tokens - # that slip through such as `CPUs`. This filter identifies those problematic - # tokens and converts them into tokens that can be easily processed by the - # compiled regex expressions. - if "CPUs" in v: - v = v.replace("CPUs", "Cpus") - holder = _loose_pascal_filter.sub(r"\1_\2", v) - return _snakecase_convert.sub(r"\1_\2", holder).lower() - - -def clean(config: Deque[str]) -> Deque[str]: - """Clean loaded configuration file before parsing. - - Cleaning tasks include: - 1. Stripping away comments (#) in configuration. Slurm does not - support octothorpes in strings; only for inline and standalone - comments. **Do not use** octothorpes in Slurm configuration knob - values as Slurm will treat anything proceeding an octothorpe as a comment. - 2. Strip away any extra whitespace at the end of each line. - - Args: - config: Loaded configuration file. Split by newlines. - """ - processed = deque() - while config: - line = config.popleft() - if line.startswith("#"): - # Skip comment lines as they're not necessary for configuration. - continue - elif "#" in line: - # Slice off inline comment and strip away extra whitespace. - processed.append(line[: line.index("#")].strip()) - else: - processed.append(line.strip()) - - return processed - - -def header(msg: str) -> str: - """Generate header for marshalled configuration file. - - Args: - msg: Message to put into header. - """ - return "#\n" + "".join(f"# {line}\n" for line in msg.splitlines()) + "#\n" - - -def parse_repeating_config(__key, __value, pocket: Dict) -> None: - """Parse `slurm.conf` configuration knobs with keys that can repeat. - - Args: - __key: Configuration knob key that can repeat. - __value: Value of the current configuration knob. - pocket: Dictionary to add parsed configuration knob to. - """ - if __key not in pocket: - pocket[__key] = [__value] - else: - pocket[__key].append(__value) - - -def parse_model(line: str, pocket: Union[Dict, List], model) -> None: - """Parse configuration knobs based on Slurm models. - - Model callbacks will be used for invoking special - parsing if required for the configuration value in line. - - Args: - line: Configuration line to parse. - pocket: Dictionary to add parsed configuration knob to. - model: Slurm data model to use for invoking callbacks and validating knob keys. - """ - holder = {} - for token in shlex.split(line): # Use `shlex.split(...)` to preserve quotation blocks. - # Word in front of the first `=` denotes the parent configuration knob key. - option, value = token.split("=", maxsplit=1) - if hasattr(model, attr := _pascal2snake(option)): - if attr in model.callbacks and (callback := model.callbacks[attr].parse) is not None: - holder.update({option: callback(value)}) - else: - holder.update({option: value}) - else: - raise EditorError( - f"{option} is not a valid configuration option for {model.__name__}." - ) - - # Use temporary model object to update pocket with a Python dictionary - # in the format that we want the dictionary to be. - if isinstance(pocket, list): - pocket.append(model(**holder).dict()) - else: - pocket.update(model(**holder).dict()) - - -def marshal_model( - model, ignore: Optional[Set] = None, inline: bool = False -) -> Union[List[str], str]: - """Marshal a Slurm model back into its Slurm configuration syntax. - - Args: - model: Slurm model object to marshal into Slurm configuration syntax. - ignore: Set of keys to ignore on model object when marshalling. Useful for models that - have child models under certain keys that are directly handled. Default is None. - inline: If True, marshal object into single line rather than multiline. Default is False. - """ - marshalled = [] - if ignore is None: - # Create an empty set if not ignores are specified. Prevents us from needing to - # rely on a mutable default in the function signature. - ignore = set() - - if primary_key := model.primary_key: - attr = _pascal2snake(primary_key) - primary_value = getattr(model, attr) - data = {primary_key: primary_value, **model.dict()[primary_value]} - else: - data = model.dict() - - for option, value in data.items(): - if option not in ignore: - if hasattr(model, attr := _pascal2snake(option)): - if ( - attr in model.callbacks - and (callback := model.callbacks[attr].marshal) is not None - ): - value = callback(value) - - marshalled.append(f"{option}={value}") - else: - raise EditorError( - f"{option} is not a valid configuration option for {model.__class__.__name__}." - ) - else: - _logger.debug("Ignoring option %s. Option is present in ignore set %s", option, ignore) - - if inline: - # Whitespace is the separator in Slurm configuration syntax. - marshalled = " ".join(marshalled) + "\n" - else: - # Append newline character so that each configuration is on its own line. - marshalled = [line + "\n" for line in marshalled] - - return marshalled diff --git a/slurmutils/editors/editor.py b/slurmutils/editors/editor.py new file mode 100644 index 0000000..e91a525 --- /dev/null +++ b/slurmutils/editors/editor.py @@ -0,0 +1,131 @@ +# Copyright 2024 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +"""Base methods for Slurm workload manager configuration file editors.""" + +import logging +import shlex +from functools import wraps +from os import path +from typing import Any, Dict, List, Tuple + +from ..exceptions import EditorError + +_logger = logging.getLogger("slurmutils") + + +def _is_comment(line: str) -> bool: + """Check if line is a comment.""" + return line.startswith("#") + + +def _contains_comment(line: str) -> bool: + """Check if line contains an inline comment.""" + return "#" in line + + +def _slice_comment(line: str) -> str: + """Slice inline comment off of line.""" + return line.split("#", maxsplit=1)[0] + + +def clean(line: str) -> Tuple[str, bool]: + """Clean line before further processing. + + Returns: + Returns the cleaned line and False if it should be ignored. + If True, then the processors should ignore the line. + """ + if _is_comment(line): + return "", True + + return (_slice_comment(line) if _contains_comment(line) else line).strip(), False + + +def parse_line(options, line: str) -> Dict[str, Any]: + """Parse configuration line. + + Args: + options: Available options for line. + line: Configuration line to parse. + """ + data = {} + opts = shlex.split(line) # Use `shlex.split(...)` to preserve quotation strings. + for opt in opts: + k, v = opt.split("=", maxsplit=1) + if not hasattr(options, k): + raise EditorError( + ( + f"unable to parse configuration option {k}. " + + f"valid configuration options are {[option.name for option in options]}" + ) + ) + + parse = getattr(options, k).parser + data[k] = parse(v) if parse else v + + return data + + +def marshall_content(options, line: Dict[str, Any]) -> List[str]: + """Marshall data model content back into configuration line. + + Args: + options: Available options for line. + line: Data model to marshall into line. + """ + result = [] + for k, v in line.items(): + if not hasattr(options, k): + raise EditorError( + ( + f"unable to marshall configuration option {k}. " + + f"valid configuration options are {[option.name for option in options]}" + ) + ) + + marshall = getattr(options, k).marshaller + result.append(f"{k}={marshall(v) if marshall else v}") + + return result + + +def loader(func): + """Wrap function that loads configuration data from file.""" + + @wraps(func) + def wrapper(*args, **kwargs): + fin = args[0] + if not path.exists(fin): + raise FileNotFoundError("could not locate %s", fin) + + _logger.debug("reading contents of %s", fin) + return func(*args, **kwargs) + + return wrapper + + +def dumper(func): + """Wrap function that dumps configuration data to file.""" + + @wraps(func) + def wrapper(*args, **kwargs): + fout = args[1] + if path.exists(fout): + _logger.debug("overwriting current contents of %s", fout) + + _logger.debug("updating contents of %s", fout) + return func(*args, **kwargs) + + return wrapper From 1189f8500e4a32f8d612689763e54f64bfb897e3 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Mon, 26 Aug 2024 10:46:48 -0400 Subject: [PATCH 05/19] refactor: have config editors use explicitly declared methods rather than partial functions Signed-off-by: Jason C. Nucciarone --- slurmutils/editors/slurmconfig.py | 259 +++++++++++++-------------- slurmutils/editors/slurmdbdconfig.py | 99 +++++----- 2 files changed, 178 insertions(+), 180 deletions(-) diff --git a/slurmutils/editors/slurmconfig.py b/slurmutils/editors/slurmconfig.py index a9df6cc..7aeac91 100644 --- a/slurmutils/editors/slurmconfig.py +++ b/slurmutils/editors/slurmconfig.py @@ -16,155 +16,45 @@ __all__ = ["dump", "dumps", "load", "loads", "edit"] -import functools +import logging import os -from collections import deque from contextlib import contextmanager -from datetime import datetime +from pathlib import Path from typing import Union -from slurmutils.models import DownNodes, FrontendNode, Node, NodeSet, Partition, SlurmConfig - -from ._editor import ( +from ..models import DownNodes, FrontendNode, Node, NodeSet, Partition, SlurmConfig +from ..models.option import SlurmConfigOptions +from .editor import ( clean, - dump_base, - dumps_base, - header, - load_base, - loads_base, - marshal_model, - parse_model, - parse_repeating_config, + dumper, + loader, + marshall_content, + parse_line, ) +_logger = logging.getLogger("slurmutils") -def _marshaller(config: SlurmConfig) -> str: - """Marshal Python object into slurm.conf configuration file. - Args: - config: `SlurmConfig` object to convert to slurm.conf configuration file. - """ - marshalled = [header(f"`slurm.conf` file generated at {datetime.now()} by slurmutils.")] - - if config.include: - marshalled.append(header("Included configuration files")) - marshalled.extend([f"Include {i}\n" for i in config.include] + ["\n"]) - if config.slurmctld_host: - marshalled.extend([f"SlurmctldHost={host}\n" for host in config.slurmctld_host] + ["\n"]) - - # Marshal the SlurmConfig object into Slurm configuration format. - # Ignore pockets containing child models as they will be marshalled inline. - marshalled.extend( - marshal_model( - config, - ignore={ - "Includes", - "SlurmctldHost", - "nodes", - "frontend_nodes", - "down_nodes", - "node_sets", - "partitions", - }, - ) - + ["\n"] - ) - - if len(config.nodes) != 0: - marshalled.extend( - [header("Node configurations")] - + [marshal_model(node, inline=True) for node in config.nodes] - + ["\n"] - ) - - if len(config.frontend_nodes) != 0: - marshalled.extend( - [header("Frontend node configurations")] - + [marshal_model(frontend, inline=True) for frontend in config.frontend_nodes] - + ["\n"] - ) - - if len(config.down_nodes) != 0: - marshalled.extend( - [header("Down node configurations")] - + [marshal_model(down_node, inline=True) for down_node in config.down_nodes] - + ["\n"] - ) - - if len(config.node_sets) != 0: - marshalled.extend( - [header("Node set configurations")] - + [marshal_model(node_set, inline=True) for node_set in config.node_sets] - + ["\n"] - ) - - if len(config.partitions) != 0: - marshalled.extend( - [header("Partition configurations")] - + [marshal_model(part, inline=True) for part in config.partitions] - ) - - return "".join(marshalled) - - -def _parser(config: str) -> SlurmConfig: - """Parse slurm.conf configuration file into Python object. +@loader +def load(file: Union[str, os.PathLike]) -> SlurmConfig: + """Load `slurm.conf` data model from slurm.conf file.""" + return loads(Path(file).read_text()) + + +def loads(content: str) -> SlurmConfig: + """Load `slurm.conf` data model from string.""" + return _parse(content) - Args: - config: Content of slurm.conf configuration file. - """ - slurm_conf = {} - nodes = {} - frontend_nodes = {} - down_nodes = [] - node_sets = {} - partitions = {} - - config = clean(deque(config.splitlines())) - while config: - line = config.popleft() - # slurm.conf `Include` is the only configuration knob whose - # separator is whitespace rather than `=`. - if line.startswith("Include"): - option, value = line.split(maxsplit=1) - parse_repeating_config(option, value, pocket=slurm_conf) - - # `SlurmctldHost` is the same as `Include` where it can - # be specified on multiple lines. - elif line.startswith("SlurmctldHost"): - option, value = line.split("=", 1) - parse_repeating_config(option, value, pocket=slurm_conf) - - # Check if option maps to slurm.conf data model. If so, invoke parsing - # rules for that specific data model and enter its parsed information - # into the appropriate pocket. - elif line.startswith("NodeName"): - parse_model(line, pocket=nodes, model=Node) - elif line.startswith("FrontendNode"): - parse_model(line, pocket=frontend_nodes, model=FrontendNode) - elif line.startswith("DownNodes"): - parse_model(line, pocket=down_nodes, model=DownNodes) - elif line.startswith("NodeSet"): - parse_model(line, pocket=node_sets, model=NodeSet) - elif line.startswith("PartitionName"): - parse_model(line, pocket=partitions, model=Partition) - else: - parse_model(line, pocket=slurm_conf, model=SlurmConfig) - return SlurmConfig( - **slurm_conf, - nodes=nodes, - frontend_nodes=frontend_nodes, - down_nodes=down_nodes, - node_sets=node_sets, - partitions=partitions, - ) +@dumper +def dump(config: SlurmConfig, file: Union[str, os.PathLike]) -> None: + """Dump `slurm.conf` data model into slurm.conf file.""" + Path(file).write_text(dumps(config)) -dump = functools.partial(dump_base, marshaller=_marshaller) -dumps = functools.partial(dumps_base, marshaller=_marshaller) -load = functools.partial(load_base, parser=_parser) -loads = functools.partial(loads_base, parser=_parser) +def dumps(config: SlurmConfig) -> str: + """Dump `slurm.conf` data model into a string.""" + return _marshall(config) @contextmanager @@ -176,10 +66,101 @@ def edit(file: Union[str, os.PathLike]) -> SlurmConfig: not exist at the specified file path, it will be created. """ if not os.path.exists(file): - # Create an empty SlurmConfig that can be populated. + _logger.warning("file %s not found. creating new empty slurm.conf configuration", file) config = SlurmConfig() else: - config = load(file=file) + config = load(file) yield config - dump(content=config, file=file) + dump(config, file) + + +def _parse(content: str) -> SlurmConfig: + """Parse contents of `slurm.conf`. + + Args: + content: Contents of `slurm.conf`. + """ + data = {} + lines = content.splitlines() + for index, line in enumerate(lines): + config, ignore = clean(line) + if ignore: + _logger.warning("ignoring line %s at index %s in slurm.conf", line, index) + continue + + if config.startswith("Include"): + _, v = config.split(maxsplit=1) + data["Include"] = data.get("Include", []) + [v] + elif config.startswith("SlurmctldHost"): + _, v = config.split("=", maxsplit=1) + data["SlurmctldHost"] = data.get("SlurmctldHost", []) + [v] + elif config.startswith("NodeName"): + nodes = data.get("Nodes", {}) + nodes.update(Node.from_str(config).dict()) + data["Nodes"] = nodes + elif config.startswith("DownNodes"): + data["DownNodes"] = data.get("DownNodes", []) + [DownNodes.from_str(config).dict()] + elif config.startswith("FrontendNode"): + frontend_nodes = data.get("FrontendNodes", {}) + frontend_nodes.update(FrontendNode.from_str(config).dict()) + data["FrontendNodes"] = frontend_nodes + elif config.startswith("NodeSet"): + node_sets = data.get("NodeSets", {}) + node_sets.update(NodeSet.from_str(config).dict()) + data["NodeSets"] = node_sets + elif config.startswith("PartitionName"): + partitions = data.get("Partitions", {}) + partitions.update(Partition.from_str(config).dict()) + data["Partitions"] = partitions + else: + data.update(parse_line(SlurmConfigOptions, config)) + + return SlurmConfig.from_dict(data) + + +def _marshall(config: SlurmConfig) -> str: + """Marshall `slurm.conf` data model back into slurm.conf format. + + Args: + config: `slurm.conf` data model to marshall. + """ + result = [] + data = config.dict() + include = data.pop("Include", None) + slurmctld_host = data.pop("SlurmctldHost", None) + nodes = data.pop("Nodes", {}) + down_nodes = data.pop("DownNodes", []) + frontend_nodes = data.pop("FrontendNodes", {}) + node_sets = data.pop("NodeSets", {}) + partitions = data.pop("Partitions", {}) + + if include: + result.extend([f"Include {i}" for i in include]) + + if slurmctld_host: + result.extend([f"SlurmctldHost={host}" for host in slurmctld_host]) + + result.extend(marshall_content(SlurmConfigOptions, data)) + + if nodes: + for k, v in nodes.items(): + result.append(str(Node(NodeName=k, **v))) + + if down_nodes: + for entry in down_nodes: + result.append(str(DownNodes(**entry))) + + if frontend_nodes: + for k, v in frontend_nodes.items(): + result.append(str(FrontendNode(FrontendName=k, **v))) + + if node_sets: + for k, v in node_sets.items(): + result.append(str(NodeSet(NodeSet=k, **v))) + + if partitions: + for k, v in partitions.items(): + result.append(str(Partition(PartitionName=k, **v))) + + return "\n".join(result) diff --git a/slurmutils/editors/slurmdbdconfig.py b/slurmutils/editors/slurmdbdconfig.py index 0c5a9b7..6be5989 100644 --- a/slurmutils/editors/slurmdbdconfig.py +++ b/slurmutils/editors/slurmdbdconfig.py @@ -16,74 +16,91 @@ __all__ = ["dump", "dumps", "load", "loads", "edit"] -import functools +import logging import os -from collections import deque from contextlib import contextmanager -from datetime import datetime +from pathlib import Path from typing import Union from slurmutils.models import SlurmdbdConfig -from ._editor import ( +from ..models.option import SlurmdbdConfigOptions +from .editor import ( clean, - dump_base, - dumps_base, - header, - load_base, - loads_base, - marshal_model, - parse_model, + dumper, + loader, + marshall_content, + parse_line, ) +_logger = logging.getLogger("slurmutils") -def _marshaller(config: SlurmdbdConfig) -> str: - """Marshal Python object into slurmdbd.conf configuration file. - Args: - config: `SlurmdbdConfig` object to convert to slurmdbd.conf configuration file. - """ - marshalled = [header(f"`slurmdbd.conf` file generated at {datetime.now()} by slurmutils.")] - marshalled.extend(marshal_model(config)) - - return "".join(marshalled) +@loader +def load(file: Union[str, os.PathLike]) -> SlurmdbdConfig: + """Load `slurm.conf` data model from slurm.conf file.""" + return loads(Path(file).read_text()) -def _parser(config: str) -> SlurmdbdConfig: - """Parse slurmdbd.conf configuration file into Python object. +def loads(content: str) -> SlurmdbdConfig: + """Load `slurm.conf` data model from string.""" + return _parse(content) - Args: - config: Content of slurmdbd.conf configuration file. - """ - slurmdbd_conf = {} - config = clean(deque(config.splitlines())) - while config: - line = config.popleft() - parse_model(line, pocket=slurmdbd_conf, model=SlurmdbdConfig) +@dumper +def dump(config: SlurmdbdConfig, file: Union[str, os.PathLike]) -> None: + """Dump `slurm.conf` data model into slurm.conf file.""" + Path(file).write_text(dumps(config)) - return SlurmdbdConfig(**slurmdbd_conf) - -dump = functools.partial(dump_base, marshaller=_marshaller) -dumps = functools.partial(dumps_base, marshaller=_marshaller) -load = functools.partial(load_base, parser=_parser) -loads = functools.partial(loads_base, parser=_parser) +def dumps(config: SlurmdbdConfig) -> str: + """Dump `slurm.conf` data model into a string.""" + return _marshall(config) @contextmanager def edit(file: Union[str, os.PathLike]) -> SlurmdbdConfig: - """Edit a slurmdbd.conf file. + """Edit a slurm.conf file. Args: - file: Path to slurmdbd.conf file to edit. If slurmdbd.conf does + file: Path to slurm.conf file to edit. If slurm.conf does not exist at the specified file path, it will be created. """ if not os.path.exists(file): - # Create an empty SlurmConfig that can be populated. + _logger.warning("file %s not found. creating new empty slurmdbd.conf configuration", file) config = SlurmdbdConfig() else: - config = load(file=file) + config = load(file) yield config - dump(content=config, file=file) + dump(config, file) + + +def _parse(content: str) -> SlurmdbdConfig: + """Parse contents of `slurmdbd.conf`. + + Args: + content: Contents of `slurmdbd.conf`. + """ + data = {} + lines = content.splitlines() + for index, line in enumerate(lines): + config, ignore = clean(line) + if ignore: + _logger.debug("ignoring line %s at index %s in slurmdbd.conf", line, index) + continue + + data.update(parse_line(SlurmdbdConfigOptions, config)) + + return SlurmdbdConfig.from_dict(data) + + +def _marshall(config: SlurmdbdConfig) -> str: + """Marshall `slurmdbd.conf` data model back into slurmdbd.conf format. + + Args: + config: `slurmdbd.conf` data model to marshall. + """ + result = [] + result.extend(marshall_content(SlurmdbdConfigOptions, config.dict())) + return "\n".join(result) From 2863e1e58b37ad21759c17d9ea5a7515e294b430 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Mon, 26 Aug 2024 10:50:04 -0400 Subject: [PATCH 06/19] feat!: add `from_*` methods for flexible creation of data models * Dynamically defines descriptors for accessing configuration options. * Less LOC and more maintainable as it is easier to modify callbacks for attributes. * Adds methods that make it much easier to convert between string, dict, json, etc representations. * Refactors camelizer to be more efficient and faster. * Uncomplicates descriptor generation by using only the `generate_descriptors` function. * Adds `option` module to store valid configuration options and validate inputs to data models. BREAKING CHANGES: Removes custom data structures for manipulating groups of Nodes, Partitions, etc. Signed-off-by: Jason C. Nucciarone --- slurmutils/models/__init__.py | 5 - slurmutils/models/_model.py | 275 --------- slurmutils/models/model.py | 116 ++++ slurmutils/models/option.py | 449 +++++++++++++++ slurmutils/models/slurm.py | 1014 +++++++++------------------------ slurmutils/models/slurmdbd.py | 85 +-- 6 files changed, 828 insertions(+), 1116 deletions(-) delete mode 100644 slurmutils/models/_model.py create mode 100644 slurmutils/models/model.py create mode 100644 slurmutils/models/option.py diff --git a/slurmutils/models/__init__.py b/slurmutils/models/__init__.py index d23eea4..65a8922 100644 --- a/slurmutils/models/__init__.py +++ b/slurmutils/models/__init__.py @@ -21,10 +21,5 @@ NodeSet, Partition, SlurmConfig, - NodeMap, - FrontendNodeMap, - DownNodesList, - NodeSetMap, - PartitionMap, ) from .slurmdbd import SlurmdbdConfig diff --git a/slurmutils/models/_model.py b/slurmutils/models/_model.py deleted file mode 100644 index be112da..0000000 --- a/slurmutils/models/_model.py +++ /dev/null @@ -1,275 +0,0 @@ -# Copyright 2024 Canonical Ltd. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License version 3 as published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . - -"""Macros for Slurm workload manager data models.""" - -import copy -import functools -import inspect -import json -from abc import ABC, abstractmethod -from types import MappingProxyType -from typing import Any, Callable, Dict, NamedTuple, Optional - - -# Simple type checking decorator; used to verify input into Slurm data models -# without needing every method to contain an `if isinstance(...)` block. -def assert_type(*typed_args, **typed_kwargs): - """Check the type of args and kwargs passed to a function/method.""" - - def decorator(func: Callable): - sig = inspect.signature(func) - bound_types = sig.bind_partial(*typed_args, **typed_kwargs).arguments - - @functools.wraps(func) - def wrapper(*args, **kwargs): - bound_values = sig.bind(*args, **kwargs).arguments - for name in bound_types.keys() & bound_values.keys(): - if not isinstance(bound_values[name], bound_types[name]): - raise TypeError(f"{bound_values[name]} is not {bound_types[name]}.") - - return func(*args, **kwargs) - - return wrapper - - return decorator - - -# Generate descriptors for Slurm configuration knobs. -# These descriptors are used for retrieving configuration values but -# also preserve the integrity of Slurm's loose pascal casing. -# The descriptors will use an internal _register dictionary to -# manage the parsed configuration knobs. -def base_descriptors(knob: str): - """Generate descriptors for accessing configuration knob values. - - Args: - knob: Configuration knob to generate descriptors for. - """ - - def getter(self): - return self._register.get(knob, None) - - def setter(self, value): - self._register[knob] = value - - def deleter(self): - try: - del self._register[knob] - except KeyError: - pass - - return getter, setter, deleter - - -# Nodes, FrontendNodes, DownNodes, NodeSets, and Partitions are represented -# as a Python dictionary with a primary key and nested dictionary when -# parsed in from the slurm.conf configuration file: -# -# {"node_1": {"NodeHostname": ..., "NodeAddr": ..., "CPUs", ...}} -# -# Since these models are parsed in this way, they need special descriptors -# for accessing the primary key (e.g. the NodeName), and sub values in the -# nested dictionary. -def primary_key_descriptors(): - """Generate descriptors for accessing a configuration knob key.""" - - def getter(self): - # There will only be a single key in _register, - # so it's okay to return the first index. If the - # primary key doesn't exist, return None. - try: - return list(self._register.keys())[0] - except IndexError: - return None - - def setter(self, value): - old_primary = list(self._register.keys())[0] - if old_primary: - self._register[value] = self._register.pop(old_primary, {}) - else: - self._register[value] = {} - - def deleter(self): - try: - primary_key = list(self._register.keys())[0] - del self._register[primary_key] - except IndexError: - pass - - return getter, setter, deleter - - -def nested_descriptors(knob: str, knob_key_alias: str): - """Generate descriptors for accessing a nested configuration knob. - - Args: - knob: Nested configuration knob to generate descriptors for. - knob_key_alias: Alias of knob key that needs to pbe defined in - register before accessing nested configuration knobs. - """ - - def getter(self): - try: - primary_key = list(self._register.keys())[0] - return self._register[primary_key].get(knob, None) - except IndexError: - raise KeyError(f"{knob_key_alias} must be defined before {knob} can be accessed.") - - def setter(self, value): - try: - primary_key = list(self._register.keys())[0] - self._register[primary_key][knob] = value - except IndexError: - raise KeyError(f"{knob_key_alias} must be defined before {knob} can be accessed.") - - def deleter(self): - try: - primary_key = list(self._register.keys())[0] - del self._register[primary_key][knob] - except IndexError: - raise KeyError(f"{knob_key_alias} must be defined before {knob} can be accessed.") - except KeyError: - pass - - return getter, setter, deleter - - -# Callbacks are used during parsing and marshalling for performing -# extra processing on specific configuration knob values. They contain callables -# that accept a single argument. Makes it easy to convert Python objects to Slurm -# configuration values and vice versa. -class Callback(NamedTuple): - """Object for invoking callables on Slurm configuration knobs during parsing/marshalling. - - Possible callables: - parse: Invoked when value is being parsed in from configuration file. - marshal: Invoked when value is being marshalled into configuration file. - """ - - parse: Optional[Callable[[Any], Any]] = None - marshal: Optional[Callable[[Any], Any]] = None - - -# Common parsing/marshalling callbacks for Slurm configuration values. -# Arrays are denoted using comma/colon separators. Maps are denoted as -# key1=value,key2=value,bool. Booleans are mapped by the inclusion of -# the keyword in maps. So key1=value,key2 would equate to: -# -# { -# "key1": "value", -# "key2": True, -# } -@functools.singledispatch -def _slurm_dict(v): - raise TypeError(f"Expected str or dict, not {type(v)}") - - -@_slurm_dict.register -def _(v: str): - """Convert Slurm dictionary to Python dictionary.""" - result = {} - for val in v.split(","): - if "=" in val: - sub_opt, sub_val = val.split("=", 1) - result.update({sub_opt: sub_val}) - else: - result.update({val: True}) - - return result - - -@_slurm_dict.register -def _(v: dict): - """Convert Python dictionary to Slurm dictionary.""" - result = [] - for sub_opt, sub_val in v.items(): - if not isinstance(sub_val, bool): - result.append(f"{sub_opt}={sub_val}") - elif sub_val: - result.append(sub_opt) - - return ",".join(result) - - -CommaSeparatorCallback = Callback(lambda v: v.split(","), lambda v: ",".join(v)) -ColonSeparatorCallback = Callback(lambda v: v.split(":"), lambda v: ":".join(v)) -SlurmDictCallback = Callback(_slurm_dict, _slurm_dict) -ReasonCallback = Callback(None, lambda v: f'"{v}"') - - -# All Slurm data models should inherit from this abstract parent class. -# The class provides method definitions for common operations and -# requires models to specify callbacks so that models can be treated -# generically when parsing and marshalling rather than having an infinite if-else tree. -class BaseModel(ABC): - """Abstract base class for Slurm-related data models.""" - - def __init__(self, **kwargs): - self._register = kwargs - - def __repr__(self): - output = self._register - if self.primary_key: - key = list(self._register.keys())[0] - output = {self.primary_key: key, **self._register[key]} - - return f"{self.__class__.__name__}({', '.join(f'{k}={v}' for k, v in output.items())})" - - @property - @abstractmethod - def primary_key(self) -> Optional[str]: - """Primary key for data model. - - A primary key is required for data models that have a unique identifier - to preserve the integrity of the Slurm configuration syntax. For example, - for compute nodes, the primary key would be the node name `NodeName`. Node - name can be used nicely for identifying nodes in maps, but it is difficult to - carry along the NodeName key inside the internal register of the class. - - _primary_key is used to track what the Slurm configuration key should be for - unique identifiers. Without this "protected" attribute, we would likely need - to write a custom parser for each data model. The generic model marshaller can - detect this attribute and marshal the model accordingly. - """ - pass - - @property - @abstractmethod - def callbacks(self) -> MappingProxyType: - """Store callbacks. - - This map will be queried during parsing and marshalling to determine if - a configuration value needs any further processing. Each model class will - need to define the callbacks specific to its configuration knobs. Every model - class should declare whether it has callbacks or not. - - Callbacks should be MappingProxyType (read-only dict) to prevent any accidental - mutation of callbacks used during parsing and marshalling. - """ - pass - - def dict(self) -> Dict: - """Get model in dictionary form. - - Returns a deep copy of model's internal register. The deep copy is needed - because assigned variables all point to the same dictionary in memory. Without the - deep copy, operations performed on the returned dictionary could cause unintended - mutations in the internal register. - """ - return copy.deepcopy(self._register) - - def json(self) -> str: - """Get model as JSON object.""" - return json.dumps(self._register) diff --git a/slurmutils/models/model.py b/slurmutils/models/model.py new file mode 100644 index 0000000..3a3c591 --- /dev/null +++ b/slurmutils/models/model.py @@ -0,0 +1,116 @@ +# Copyright 2024 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +"""Base classes and methods for composing Slurm data models.""" + +__all__ = ["BaseModel", "LineInterface", "format_key", "generate_descriptors"] + +import copy +import json +import re +from abc import ABC, abstractmethod +from typing import Any, Dict + +from ..exceptions import ModelError + +_acronym = re.compile(r"(?<=[A-Z])(?=[A-Z][a-z])") +_camelize = re.compile(r"(?<=[a-z0-9])(?=[A-Z])") + + +def format_key(key: str) -> str: + """Format Slurm configuration keys from SlurmCASe to camelCase. + + Args: + key: Configuration key to format into camel case. + + Notes: + Slurm configuration syntax does not follow proper PascalCasing + format, so we cannot put keys directly through a kebab case converter + to get the desired format. Some additional processing is needed for + certain keys before the key can properly camelized. + + For example, without additional preprocessing, the key `CPUs` will + become `cp-us` if put through a caramelize with being preformatted to `Cpus`. + """ + if "CPUs" in key: + key = key.replace("CPUs", "Cpus") + key = _acronym.sub(r"_", key) + return _camelize.sub(r"_", key).lower() + + +def generate_descriptors(opt: str): + """Generate descriptors for retrieving and mutating configuration options. + + Args: + opt: Configuration option to generate descriptors for. + """ + + def getter(self): + return self.data.get(opt, None) + + def setter(self, value): + self.data[opt] = value + + def deleter(self): + del self.data[opt] + + return getter, setter, deleter + + +class LineInterface: + """Interface for data models that can be constructed from a configuration line.""" + + @classmethod + @abstractmethod + def from_str(cls, line: str): + """Construct data model from configuration line.""" + + @abstractmethod + def __str__(self) -> str: + """Return model as configuration line.""" + + +class BaseModel(ABC): + """Base model for Slurm data models.""" + + def __init__(self, validator=None, /, **kwargs) -> None: + for k, v in kwargs.items(): + if not hasattr(validator, k): + raise ModelError( + ( + f"unrecognized argument {k}. " + + f"valid arguments are {[opt.name for opt in validator]}" + ) + ) + + self.data = kwargs + + @classmethod + def from_dict(cls, data: Dict[str, Any]): + """Construct new model from dictionary.""" + return cls(**data) + + @classmethod + def from_json(cls, obj: str): + """Construct new model from JSON object.""" + data = json.loads(obj) + return cls.from_dict(data) + + def dict(self) -> Dict[str, Any]: + """Return model as dictionary.""" + return copy.deepcopy(self.data) + + def json(self) -> str: + """Return model as json object.""" + return json.dumps(self.dict()) diff --git a/slurmutils/models/option.py b/slurmutils/models/option.py new file mode 100644 index 0000000..a591f2b --- /dev/null +++ b/slurmutils/models/option.py @@ -0,0 +1,449 @@ +# Copyright 2024 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +"""Configuration options for Slurm data models.""" + +__all__ = [ + "SlurmdbdConfigOptions", + "SlurmConfigOptions", + "NodeOptions", + "DownNodeOptions", + "FrontendNodeOptions", + "NodeSetOptions", + "PartitionOptions", +] + +from dataclasses import dataclass, fields +from typing import Iterable + +from .callback import ( + Callback, + ColonSeparatorCallback, + CommaSeparatorCallback, + ReasonCallback, + SlurmDictCallback, +) + + +@dataclass(frozen=True) +class _Option: + """Base for configuration option dataclasses.""" + + @classmethod + def keys(cls) -> Iterable[str]: + """Yield iterable list of configuration option names.""" + for field in fields(cls): + yield field.name + + +@dataclass(frozen=True) +class SlurmdbdConfigOptions(_Option): + """`slurmdbd.conf` configuration options.""" + + AllowNoDefAcct: Callback = Callback() + AllResourcesAbsolute: Callback = Callback() + ArchiveDir: Callback = Callback() + ArchiveEvents: Callback = Callback() + ArchiveJobs: Callback = Callback() + ArchiveResvs: Callback = Callback() + ArchiveScript: Callback = Callback() + ArchiveSteps: Callback = Callback() + ArchiveSuspend: Callback = Callback() + ArchiveTXN: Callback = Callback() + ArchiveUsage: Callback = Callback() + AuthAltTypes: Callback = CommaSeparatorCallback + AuthAltParameters: Callback = SlurmDictCallback + AuthInfo: Callback = Callback() + AuthType: Callback = Callback() + CommitDelay: Callback = Callback() + CommunicationParameters: Callback = SlurmDictCallback + DbdAddr: Callback = Callback() + DbdBackupHost: Callback = Callback() + DbdHost: Callback = Callback() + DbdPort: Callback = Callback() + DebugFlags: Callback = CommaSeparatorCallback + DebugLevel: Callback = Callback() + DebugLevelSyslog: Callback = Callback() + DefaultQOS: Callback = Callback() + LogFile: Callback = Callback() + LogTimeFormat: Callback = Callback() + MaxQueryTimeRange: Callback = Callback() + MessageTimeout: Callback = Callback() + Parameters: Callback = CommaSeparatorCallback + PidFile: Callback = Callback() + PluginDir: Callback = ColonSeparatorCallback + PrivateData: Callback = CommaSeparatorCallback + PurgeEventAfter: Callback = Callback() + PurgeJobAfter: Callback = Callback() + PurgeResvAfter: Callback = Callback() + PurgeStepAfter: Callback = Callback() + PurgeSuspendAfter: Callback = Callback() + PurgeTXNAfter: Callback = Callback() + PurgeUsageAfter: Callback = Callback() + SlurmUser: Callback = Callback() + StorageBackupHost: Callback = Callback() + StorageHost: Callback = Callback() + StorageLoc: Callback = Callback() + StorageParameters: Callback = SlurmDictCallback + StoragePass: Callback = Callback() + StoragePort: Callback = Callback() + StorageType: Callback = Callback() + StorageUser: Callback = Callback() + TCPTimeout: Callback = Callback() + TrackSlurmctldDown: Callback = Callback() + TrackWCKey: Callback = Callback() + + +@dataclass(frozen=True) +class SlurmConfigOptions(_Option): + """`slurm.conf` configuration options.""" + + AccountingStorageBackupHost: Callback = CommaSeparatorCallback + AccountingStorageEnforce: Callback = Callback() + AccountingStorageExternalHost: Callback = Callback() + AccountingStorageHost: Callback = Callback() + AccountingStorageParameters: Callback = SlurmDictCallback + AccountingStoragePass: Callback = Callback() + AccountingStoragePort: Callback = Callback() + AccountingStorageTRES: Callback = CommaSeparatorCallback + AccountingStorageType: Callback = Callback() + AccountingStorageUser: Callback = Callback() + AccountingStoreFlags: Callback = CommaSeparatorCallback + AcctGatherNodeFreq: Callback = Callback() + AcctGatherEnergyType: Callback = Callback() + AcctGatherInterconnectType: Callback = Callback() + AcctGatherFilesystemType: Callback = Callback() + AcctGatherProfileType: Callback = Callback() + AllowSpecResourcesUsage: Callback = Callback() + AuthAltTypes: Callback = CommaSeparatorCallback + AuthAltParameters: Callback = Callback() + AuthInfo: Callback = SlurmDictCallback + AuthType: Callback = Callback() + BatchStartTimeout: Callback = Callback() + BcastExclude: Callback = CommaSeparatorCallback + BcastParameters: Callback = SlurmDictCallback + BurstBufferType: Callback = Callback() + CliFilterPlugins: Callback = CommaSeparatorCallback + ClusterName: Callback = Callback() + CommunicationParameters: Callback = SlurmDictCallback + CheckGhalQuiesce: Callback = Callback() + DisableIPv4: Callback = Callback() + EnableIPv6: Callback = Callback() + NoCtldInAddrAny: Callback = Callback() + NoInAddrAny: Callback = Callback() + CompleteWait: Callback = Callback() + CoreSpecPlugin: Callback = Callback() + CpuFreqDef: Callback = CommaSeparatorCallback + CpuFreqGovernors: Callback = CommaSeparatorCallback + CredType: Callback = Callback() + DebugFlags: Callback = CommaSeparatorCallback + BurstBuffer: Callback = Callback() + DefCpuPerGPU: Callback = Callback() + DefMemPerCPU: Callback = Callback() + DefMemPerGPU: Callback = Callback() + DefMemPerNode: Callback = Callback() + DependencyParameters: Callback = SlurmDictCallback + DisableRootJobs: Callback = Callback() + EioTimeout: Callback = Callback() + EnforcePartLimits: Callback = Callback() + Epilog: Callback = Callback() + EpilogMsgTime: Callback = Callback() + EpilogSlurmctld: Callback = Callback() + FairShareDampeningFactor: Callback = Callback() + FederationParameters: Callback = CommaSeparatorCallback + FirstJobId: Callback = Callback() + GetEnvTimeout: Callback = Callback() + GresTypes: Callback = Callback() + GroupUpdateForce: Callback = Callback() + GroupUpdateTime: Callback = Callback() + GpuFreqDef: Callback = Callback() + HealthCheckInterval: Callback = Callback() + HealthCheckNodeState: Callback = CommaSeparatorCallback + HealthCheckProgram: Callback = Callback() + InactiveLimit: Callback = Callback() + InteractiveStepOptions: Callback = Callback() + JobAcctGatherType: Callback = Callback() + JobAcctGatherFrequency: Callback = SlurmDictCallback + JobAcctGatherParams: Callback = Callback() + NoShared: Callback = Callback() + UsePss: Callback = Callback() + OverMemoryKill: Callback = Callback() + DisableGPUAcct: Callback = Callback() + JobCompHost: Callback = Callback() + JobCompLoc: Callback = Callback() + JobCompParams: Callback = SlurmDictCallback + JobCompPass: Callback = Callback() + JobCompPort: Callback = Callback() + JobCompType: Callback = Callback() + JobCompUser: Callback = Callback() + JobContainerType: Callback = Callback() + JobFileAppend: Callback = Callback() + JobRequeue: Callback = Callback() + JobSubmitPlugins: Callback = CommaSeparatorCallback + KillOnBadExit: Callback = Callback() + KillWait: Callback = Callback() + MaxBatchRequeue: Callback = Callback() + NodeFeaturesPlugins: Callback = Callback() + LaunchParameters: Callback = SlurmDictCallback + Licenses: Callback = CommaSeparatorCallback + LogTimeFormat: Callback = Callback() + MailDomain: Callback = Callback() + MailProg: Callback = Callback() + MaxArraySize: Callback = Callback() + MaxDBDMsgs: Callback = Callback() + MaxJobCount: Callback = Callback() + MaxJobId: Callback = Callback() + MaxMemPerCPU: Callback = Callback() + MaxMemPerNode: Callback = Callback() + MaxNodeCount: Callback = Callback() + MaxStepCount: Callback = Callback() + MaxTasksPerNode: Callback = Callback() + MCSParameters: Callback = Callback() + MCSPlugin: Callback = Callback() + MessageTimeout: Callback = Callback() + MinJobAge: Callback = Callback() + MpiDefault: Callback = Callback() + MpiParams: Callback = Callback() + OverTimeLimit: Callback = Callback() + PluginDir: Callback = ColonSeparatorCallback + PlugStackConfig: Callback = Callback() + PowerParameters: Callback = SlurmDictCallback + PowerPlugin: Callback = Callback() + PreemptMode: Callback = CommaSeparatorCallback + PreemptParameters: Callback = SlurmDictCallback + PreemptType: Callback = Callback() + PreemptExemptTime: Callback = Callback() + PrEpParameters: Callback = Callback() + PrEpPlugins: Callback = CommaSeparatorCallback + PriorityCalcPeriod: Callback = Callback() + PriorityDecayHalfLife: Callback = Callback() + PriorityFavorSmall: Callback = Callback() + PriorityFlags: Callback = Callback() + PriorityMaxAge: Callback = Callback() + PriorityParameters: Callback = Callback() + PrioritySiteFactorParameters: Callback = Callback() + PrioritySiteFactorPlugin: Callback = Callback() + PriorityType: Callback = Callback() + PriorityUsageResetPeriod: Callback = Callback() + PriorityWeightAge: Callback = Callback() + PriorityWeightAssoc: Callback = Callback() + PriorityWeightFairshare: Callback = Callback() + PriorityWeightJobSize: Callback = Callback() + PriorityWeightPartition: Callback = Callback() + PriorityWeightQOS: Callback = Callback() + PriorityWeightTRES: Callback = SlurmDictCallback + PrivateData: Callback = CommaSeparatorCallback + ProctrackType: Callback = Callback() + Prolog: Callback = Callback() + PrologEpilogTimeout: Callback = Callback() + PrologFlags: Callback = CommaSeparatorCallback + PrologSlurmctld: Callback = Callback() + PropagatePrioProcess: Callback = Callback() + PropagateResourceLimits: Callback = CommaSeparatorCallback + PropagateResourceLimitsExcept: Callback = CommaSeparatorCallback + RebootProgram: Callback = Callback() + ReconfigFlags: Callback = Callback() + KeepPartInfo: Callback = Callback() + KeepPartState: Callback = Callback() + KeepPowerSaveSettings: Callback = Callback() + RequeueExit: Callback = Callback() + RequeueExitHold: Callback = Callback() + ResumeFailProgram: Callback = Callback() + ResumeProgram: Callback = Callback() + ResumeRate: Callback = Callback() + ResumeTimeout: Callback = Callback() + ResvEpilog: Callback = Callback() + ResvOverRun: Callback = Callback() + ResvProlog: Callback = Callback() + ReturnToService: Callback = Callback() + SchedulerParameters: Callback = SlurmDictCallback + SchedulerTimeSlice: Callback = Callback() + SchedulerType: Callback = Callback() + ScronParameters: Callback = CommaSeparatorCallback + SelectType: Callback = Callback() + SelectTypeParameters: Callback = Callback() + SlurmctldAddr: Callback = Callback() + SlurmctldDebug: Callback = Callback() + SlurmctldHost: Callback = Callback() + SlurmctldLogFile: Callback = Callback() + SlurmctldParameters: Callback = SlurmDictCallback + SlurmctldPidFile: Callback = Callback() + SlurmctldPort: Callback = Callback() + SlurmctldPrimaryOffProg: Callback = Callback() + SlurmctldPrimaryOnProg: Callback = Callback() + SlurmctldSyslogDebug: Callback = Callback() + SlurmctldTimeout: Callback = Callback() + SlurmdDebug: Callback = Callback() + SlurmdLogFile: Callback = Callback() + SlurmdParameters: Callback = CommaSeparatorCallback + SlurmdPidFile: Callback = Callback() + SlurmdPort: Callback = Callback() + SlurmdSpoolDir: Callback = Callback() + SlurmdSyslogDebug: Callback = Callback() + SlurmdTimeout: Callback = Callback() + SlurmdUser: Callback = Callback() + SlurmSchedLogFile: Callback = Callback() + SlurmSchedLogLevel: Callback = Callback() + SlurmUser: Callback = Callback() + SrunEpilog: Callback = Callback() + SrunPortRange: Callback = Callback() + SrunProlog: Callback = Callback() + StateSaveLocation: Callback = Callback() + SuspendExcNodes: Callback = Callback() + SuspendExcParts: Callback = Callback() + SuspendExcStates: Callback = Callback() + SuspendProgram: Callback = Callback() + SuspendRate: Callback = Callback() + SuspendTime: Callback = Callback() + SuspendTimeout: Callback = Callback() + SwitchParameters: Callback = SlurmDictCallback + SwitchType: Callback = Callback() + TaskEpilog: Callback = Callback() + TaskPlugin: Callback = CommaSeparatorCallback + TaskPluginParam: Callback = SlurmDictCallback + Cores: Callback = Callback() + Sockets: Callback = Callback() + Threads: Callback = Callback() + SlurmdOffSpec: Callback = Callback() + Verbose: Callback = Callback() + Autobind: Callback = Callback() + TaskProlog: Callback = Callback() + TCPTimeout: Callback = Callback() + TmpFS: Callback = Callback() + TopologyParam: Callback = CommaSeparatorCallback + Dragonfly: Callback = Callback() + RoutePart: Callback = Callback() + SwitchAsNodeRank: Callback = Callback() + RouteTree: Callback = Callback() + TopoOptional: Callback = Callback() + TopologyPlugin: Callback = Callback() + TrackWCKey: Callback = Callback() + TreeWidth: Callback = Callback() + UnkillableStepProgram: Callback = Callback() + UnkillableStepTimeout: Callback = Callback() + UsePAM: Callback = Callback() + VSizeFactor: Callback = Callback() + WaitTime: Callback = Callback() + X11Parameters: Callback = Callback() + + +@dataclass(frozen=True) +class NodeOptions(_Option): + """`slurm.conf` node configuration options.""" + + NodeName: Callback = Callback() + NodeHostname: Callback = Callback() + NodeAddr: Callback = Callback() + BcastAddr: Callback = Callback() + Boards: Callback = Callback() + CoreSpecCount: Callback = Callback() + CoresPerSocket: Callback = Callback() + CpuBind: Callback = Callback() + CPUs: Callback = Callback() + CpuSpecList: Callback = CommaSeparatorCallback + Features: Callback = CommaSeparatorCallback + Gres: Callback = CommaSeparatorCallback + MemSpecLimit: Callback = Callback() + Port: Callback = Callback() + Procs: Callback = Callback() + RealMemory: Callback = Callback() + Reason: Callback = ReasonCallback + Sockets: Callback = Callback() + SocketsPerBoard: Callback = Callback() + State: Callback = Callback() + ThreadsPerCore: Callback = Callback() + TmpDisk: Callback = Callback() + Weight: Callback = Callback() + + +@dataclass(frozen=True) +class DownNodeOptions(_Option): + """`slurm.conf` down node configuration options.""" + + DownNodes: Callback = CommaSeparatorCallback + Reason: Callback = ReasonCallback + State: Callback = Callback() + + +@dataclass(frozen=True) +class FrontendNodeOptions(_Option): + """`slurm.conf` frontend node configuration options.""" + + FrontendName: Callback = Callback() + FrontendAddr: Callback = Callback() + AllowGroups: Callback = CommaSeparatorCallback + AllowUsers: Callback = CommaSeparatorCallback + DenyGroups: Callback = CommaSeparatorCallback + DenyUsers: Callback = CommaSeparatorCallback + Port: Callback = Callback() + Reason: Callback = ReasonCallback + State: Callback = Callback() + + +@dataclass(frozen=True) +class NodeSetOptions(_Option): + """`slurm.conf` node set configuration options.""" + + NodeSet: Callback = Callback() + Feature: Callback = Callback() + Nodes: Callback = CommaSeparatorCallback + + +@dataclass(frozen=True) +class PartitionOptions(_Option): + """`slurm.conf` partition configuration options.""" + + PartitionName: Callback = Callback() + AllocNodes: Callback = CommaSeparatorCallback + AllowAccounts: Callback = CommaSeparatorCallback + AllowGroups: Callback = CommaSeparatorCallback + AllowQos: Callback = CommaSeparatorCallback + Alternate: Callback = Callback() + CpuBind: Callback = Callback() + Default: Callback = Callback() + DefaultTime: Callback = Callback() + DefCpuPerGPU: Callback = Callback() + DefMemPerCPU: Callback = Callback() + DefMemPerGPU: Callback = Callback() + DefMemPerNode: Callback = Callback() + DenyAccounts: Callback = CommaSeparatorCallback + DenyQos: Callback = CommaSeparatorCallback + DisableRootJobs: Callback = Callback() + ExclusiveUser: Callback = Callback() + GraceTime: Callback = Callback() + Hidden: Callback = Callback() + LLN: Callback = Callback() + MaxCPUsPerNode: Callback = Callback() + MaxCPUsPerSocket: Callback = Callback() + MaxMemPerCPU: Callback = Callback() + MaxMemPerNode: Callback = Callback() + MaxNodes: Callback = Callback() + MaxTime: Callback = Callback() + MinNodes: Callback = Callback() + Nodes: Callback = CommaSeparatorCallback + OverSubscribe: Callback = Callback() + OverTimeLimit: Callback = Callback() + PowerDownOnIdle: Callback = Callback() + PreemptMode: Callback = Callback() + PriorityJobFactor: Callback = Callback() + PriorityTier: Callback = Callback() + QOS: Callback = Callback() + ReqResv: Callback = Callback() + ResumeTimeout: Callback = Callback() + RootOnly: Callback = Callback() + SelectTypeParameters: Callback = Callback() + State: Callback = Callback() + SuspendTime: Callback = Callback() + SuspendTimeout: Callback = Callback() + TRESBillingWeights: Callback = SlurmDictCallback diff --git a/slurmutils/models/slurm.py b/slurmutils/models/slurm.py index c49c4fd..920c4e3 100644 --- a/slurmutils/models/slurm.py +++ b/slurmutils/models/slurm.py @@ -12,809 +12,305 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . -"""Generic data models for the Slurm workload manager.""" - -import functools -from collections import UserList -from collections.abc import MutableMapping -from types import MappingProxyType -from typing import Any, Dict, Optional - -from ._model import ( - BaseModel, - ColonSeparatorCallback, - CommaSeparatorCallback, - ReasonCallback, - SlurmDictCallback, - assert_type, - base_descriptors, - nested_descriptors, - primary_key_descriptors, +"""Data models for `slurm.conf` configuration file.""" + +import copy +from typing import Any, Dict, List + +from ..editors.editor import marshall_content, parse_line +from .model import BaseModel, LineInterface, format_key, generate_descriptors +from .option import ( + DownNodeOptions, + FrontendNodeOptions, + NodeOptions, + NodeSetOptions, + PartitionOptions, + SlurmConfigOptions, ) -_node_descriptors = functools.partial(nested_descriptors, knob_key_alias="NodeName") -_frontend_descriptors = functools.partial(nested_descriptors, knob_key_alias="FrontendName") -_nodeset_descriptors = functools.partial(nested_descriptors, knob_key_alias="NodeSet") -_partition_descriptors = functools.partial(nested_descriptors, knob_key_alias="PartitionName") +class Node(BaseModel, LineInterface): + """`Node` data model.""" -class Node(BaseModel): - """Object representing Node(s) definition in slurm.conf. + def __init__(self, *, NodeName: str, **kwargs) -> None: # noqa N803 + self.__node_name = NodeName + super().__init__(NodeOptions, **kwargs) - Node definition and data validators sourced from - the slurm.conf manpage. `man slurm.conf.5` - """ + @property + def node_name(self) -> str: + """Get node name.""" + return self.__node_name - def __init__(self, **kwargs): - super().__init__() - self._register.update({kwargs.pop("NodeName"): {**kwargs}}) - - primary_key = "NodeName" - callbacks = MappingProxyType( - { - "cpu_spec_list": CommaSeparatorCallback, - "features": CommaSeparatorCallback, - "gres": CommaSeparatorCallback, - "reason": ReasonCallback, - } - ) - - node_name = property(*primary_key_descriptors()) - node_hostname = property(*_node_descriptors("NodeHostname")) - node_addr = property(*_node_descriptors("NodeAddr")) - bcast_addr = property(*_node_descriptors("BcastAddr")) - boards = property(*_node_descriptors("Boards")) - core_spec_count = property(*_node_descriptors("CoreSpecCount")) - cores_per_socket = property(*_node_descriptors("CoresPerSocket")) - cpu_bind = property(*_node_descriptors("CpuBind")) - cpus = property(*_node_descriptors("CPUs")) - cpu_spec_list = property(*_node_descriptors("CpuSpecList")) - features = property(*_node_descriptors("Features")) - gres = property(*_node_descriptors("Gres")) - mem_spec_limit = property(*_node_descriptors("MemSpecLimit")) - port = property(*_node_descriptors("Port")) - procs = property(*_node_descriptors("Procs")) - real_memory = property(*_node_descriptors("RealMemory")) - reason = property(*_node_descriptors("Reason")) - sockets = property(*_node_descriptors("Sockets")) - sockets_per_board = property(*_node_descriptors("SocketsPerBoard")) - state = property(*_node_descriptors("State")) - threads_per_core = property(*_node_descriptors("ThreadsPerCore")) - tmp_disk = property(*_node_descriptors("TmpDisk")) - weight = property(*_node_descriptors("Weight")) - - -class DownNodes(BaseModel): - """Object representing DownNodes definition in slurm.conf. - - DownNodes definition and data validators sourced from - the slurm.conf manpage. `man slurm.conf.5` - """ - - primary_key = None - callbacks = MappingProxyType( - { - "down_nodes": CommaSeparatorCallback, - "reason": ReasonCallback, - } - ) - - down_nodes = property(*base_descriptors("DownNodes")) - reason = property(*base_descriptors("Reason")) - state = property(*base_descriptors("State")) - - -class FrontendNode(BaseModel): - """FrontendNode data model. - - FrontendNode definition and data validators sourced from - the slurm.conf manpage. `man slurm.conf.5` - """ + @node_name.setter + def node_name(self, name: str) -> None: + """Set new node name.""" + self.__node_name = name - def __init__(self, **kwargs): - super().__init__() - self._register.update({kwargs.pop("FrontendName"): {**kwargs}}) - - primary_key = "FrontendName" - callbacks = MappingProxyType( - { - "allow_groups": CommaSeparatorCallback, - "allow_users": CommaSeparatorCallback, - "deny_groups": CommaSeparatorCallback, - "deny_users": CommaSeparatorCallback, - "reason": ReasonCallback, - } - ) - - frontend_name = property(*primary_key_descriptors()) - frontend_addr = property(*_frontend_descriptors("FrontendAddr")) - allow_groups = property(*_frontend_descriptors("AllowGroups")) - allow_users = property(*_frontend_descriptors("AllowUsers")) - deny_groups = property(*_frontend_descriptors("DenyGroups")) - deny_users = property(*_frontend_descriptors("DenyUsers")) - port = property(*_frontend_descriptors("Port")) - reason = property(*_frontend_descriptors("Reason")) - state = property(*_frontend_descriptors("State")) - - -class NodeSet(BaseModel): - """Object representing NodeSet definition in slurm.conf. - - NodeSet definition and data validators sourced from - the slurm.conf manpage. `man slurm.conf.5` - """ + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "Node": + """Construct model from dictionary.""" + name = list(data.keys())[0] + return cls(NodeName=name, **data[name]) + + @classmethod + def from_str(cls, line: str) -> "Node": + """Construct model from configuration line.""" + data = parse_line(NodeOptions, line) + return cls(**data) + + def dict(self) -> Dict[str, Any]: + """Return model as dictionary.""" + return copy.deepcopy({self.__node_name: self.data}) + + def __str__(self) -> str: + """Return model as configuration line.""" + line = [f"NodeName={self.__node_name}"] + line.extend(marshall_content(NodeOptions, self.data)) + return " ".join(line) + + +class DownNodes(BaseModel, LineInterface): + """`DownNodes` data model.""" def __init__(self, **kwargs): - super().__init__() - self._register.update({kwargs.pop("NodeSet"): {**kwargs}}) + super().__init__(DownNodeOptions, **kwargs) - primary_key = "NodeSet" - callbacks = MappingProxyType({"nodes": CommaSeparatorCallback}) + @classmethod + def from_str(cls, line: str) -> "DownNodes": + """Construct model from configuration line.""" + data = parse_line(DownNodeOptions, line) + return cls(**data) - node_set = property(*primary_key_descriptors()) - feature = property(*_nodeset_descriptors("Feature")) - nodes = property(*_nodeset_descriptors("Nodes")) + def __str__(self) -> str: + """Return model as configuration line.""" + return " ".join(marshall_content(DownNodeOptions, self.data)) -class Partition(BaseModel): - """Object representing Partition definition in slurm.conf. +class FrontendNode(BaseModel, LineInterface): + """`FrontendNode` data model.""" - Partition definition and data validators sourced from - the slurm.conf manpage. `man slurm.conf.5` - """ + def __init__(self, *, FrontendName: str, **kwargs) -> None: # noqa N803 + self.__frontend_name = FrontendName + super().__init__(FrontendNodeOptions, **kwargs) - def __init__(self, **kwargs): - super().__init__() - self._register.update({kwargs.pop("PartitionName"): {**kwargs}}) - - primary_key = "PartitionName" - callbacks = MappingProxyType( - { - "alloc_nodes": CommaSeparatorCallback, - "allow_accounts": CommaSeparatorCallback, - "allow_groups": CommaSeparatorCallback, - "allow_qos": CommaSeparatorCallback, - "deny_accounts": CommaSeparatorCallback, - "deny_qos": CommaSeparatorCallback, - "nodes": CommaSeparatorCallback, - "tres_billing_weights": SlurmDictCallback, - } - ) - - partition_name = property(*primary_key_descriptors()) - alloc_nodes = property(*_partition_descriptors("AllocNodes")) - allow_accounts = property(*_partition_descriptors("AllowAccounts")) - allow_groups = property(*_partition_descriptors("AllowGroups")) - allow_qos = property(*_partition_descriptors("AllowQos")) - alternate = property(*_partition_descriptors("Alternate")) - cpu_bind = property(*_partition_descriptors("CpuBind")) - default = property(*_partition_descriptors("Default")) - default_time = property(*_partition_descriptors("DefaultTime")) - def_cpu_per_gpu = property(*_partition_descriptors("DefCpuPerGPU")) - def_mem_per_cpu = property(*_partition_descriptors("DefMemPerCPU")) - def_mem_per_gpu = property(*_partition_descriptors("DefMemPerGPU")) - def_mem_per_node = property(*_partition_descriptors("DefMemPerNode")) - deny_accounts = property(*_partition_descriptors("DenyAccounts")) - deny_qos = property(*_partition_descriptors("DenyQos")) - disable_root_jobs = property(*_partition_descriptors("DisableRootJobs")) - exclusive_user = property(*_partition_descriptors("ExclusiveUser")) - grace_time = property(*_partition_descriptors("GraceTime")) - hidden = property(*_partition_descriptors("Hidden")) - lln = property(*_partition_descriptors("LLN")) - max_cpus_per_node = property(*_partition_descriptors("MaxCPUsPerNode")) - max_cpus_per_socket = property(*_partition_descriptors("MaxCPUsPerSocket")) - max_mem_per_cpu = property(*_partition_descriptors("MaxMemPerCPU")) - max_mem_per_node = property(*_partition_descriptors("MaxMemPerNode")) - max_nodes = property(*_partition_descriptors("MaxNodes")) - max_time = property(*_partition_descriptors("MaxTime")) - min_nodes = property(*_partition_descriptors("MinNodes")) - nodes = property(*_partition_descriptors("Nodes")) - over_subscribe = property(*_partition_descriptors("OverSubscribe")) - over_time_limit = property(*_partition_descriptors("OverTimeLimit")) - power_down_on_idle = property(*_partition_descriptors("PowerDownOnIdle")) - preempt_mode = property(*_partition_descriptors("PreemptMode")) - priority_job_factor = property(*_partition_descriptors("PriorityJobFactor")) - priority_tier = property(*_partition_descriptors("PriorityTier")) - qos = property(*_partition_descriptors("QOS")) - req_resv = property(*_partition_descriptors("ReqResv")) - resume_timeout = property(*_partition_descriptors("ResumeTimeout")) - root_only = property(*_partition_descriptors("RootOnly")) - select_type_parameters = property(*_partition_descriptors("SelectTypeParameters")) - state = property(*_partition_descriptors("State")) - suspend_time = property(*_partition_descriptors("SuspendTime")) - suspend_timeout = property(*_partition_descriptors("SuspendTimeout")) - tres_billing_weights = property(*_partition_descriptors("TRESBillingWeights")) - - -class NodeMap(MutableMapping): - """Map of Node names to dictionaries for composing `Node` objects.""" - - def __init__(self, data: Optional[Dict[str, Dict[str, Any]]] = None): - if data is None: - data = {} - - self.data = data - - @assert_type(value=Node) - def __setitem__(self, key: str, value: Node) -> None: - if key != value.node_name: - raise ValueError(f"{key} and {value.node_name} are not equal.") - else: - self.data.update(value.dict()) - - def __delitem__(self, key: str) -> None: - del self.data[key] - - def __getitem__(self, key: str) -> Node: - try: - node = self.data.get(key) - return Node(NodeName=key, **node) - except KeyError: - raise KeyError(f"Node {key} is not defined.") - - def __len__(self): - return len(self.data) - - def __iter__(self): - return iter([Node(NodeName=k, **self.data[k]) for k in self.data.keys()]) - - -class FrontendNodeMap(MutableMapping): - """Map of FrontendNode names to dictionaries for composing `FrontendNode` objects.""" - - def __init__(self, data: Optional[Dict[str, Dict[str, Any]]] = None): - if data is None: - data = {} - - self.data = data - - @assert_type(value=FrontendNode) - def __setitem__(self, key: str, value: FrontendNode) -> None: - if key != value.frontend_name: - raise ValueError(f"{key} and {value.frontend_name} are not equal.") - else: - self.data.update(value.dict()) - - def __delitem__(self, key: str) -> None: - del self.data[key] - - def __getitem__(self, key: str) -> FrontendNode: - try: - frontend_node = self.data.get(key) - return FrontendNode(FrontendName=key, **frontend_node) - except KeyError: - raise KeyError(f"FrontendNode {key} is not defined.") - - def __len__(self): - return len(self.data) - - def __iter__(self): - return iter([FrontendNode(FrontendName=k, **self.data[k]) for k in self.data.keys()]) - - -class DownNodesList(UserList): - """List of dictionaries for composing `DownNodes` objects.""" - - def __getitem__(self, i): - if isinstance(i, slice): - return self.__class__(self.data[i]) - else: - return DownNodes(**self.data[i]) - - @assert_type(value=DownNodes) - def __setitem__(self, i: int, value: DownNodes): - super().__setitem__(i, value.dict()) - - @assert_type(value=DownNodes) - def __contains__(self, value): - return value.dict() in self.data - - def __iter__(self): - return iter([DownNodes(**data) for data in self.data]) - - def __add__(self, other): - if isinstance(other, DownNodesList): - return self.__class__(self.data + other.data) - elif isinstance(other, type(self.data)): - # Cannot use `assert_type` here because isinstance does - # not support using subscripted generics for runtime validation. - for down_node in other: - if not isinstance(down_node, DownNodes): - raise TypeError(f"{down_node} is not {type(DownNodes)}.") - - return self.__class__(self.data + other) - - return self.__class__(self.data + list(other)) - - def __radd__(self, other): - if isinstance(other, DownNodesList): - return self.__class__(other.data + self.data) - elif isinstance(other, type(self.data)): - for down_node in other: - if not isinstance(down_node, DownNodes): - raise TypeError(f"{down_node} is not {type(DownNodes)}.") - - return self.__class__(other + self.data) - - return self.__class__(list(other) + self.data) - - def __iadd__(self, other): - if isinstance(other, DownNodesList): - self.data += other.data - elif isinstance(other, type(self.data)): - for down_node in other: - if not isinstance(down_node, DownNodes): - raise TypeError(f"{down_node} is not {type(DownNodes)}.") - - self.data += other - else: - if not isinstance(other, DownNodes): - raise TypeError(f"{other} is not {type(DownNodes)}.") - - self.data += other - return self - - @assert_type(value=DownNodes) - def append(self, value: DownNodes): - """Add DownNodes object to list of DownNodes.""" - super().append(value.dict()) - - @assert_type(value=DownNodes) - def insert(self, i, value): - """Insert DownNodes object into list of DownNodes at the given index.""" - super().insert(i, value.dict()) - - @assert_type(value=DownNodes) - def remove(self, value): - """Remove DownNodes object from list of DownNodes.""" - self.data.remove(value.dict()) - - @assert_type(value=DownNodes) - def count(self, value): - """Count the numbers of occurrences for the given DownNodes object. - - Warnings: - Each DownNodes object should only occur once (1). If the object - occurs more than once, this can create BIG problems when trying to - restart the Slurm daemons. - """ - return self.data.count(value.dict()) - - @assert_type(value=DownNodes) - def index(self, value, *args): - """Get the index of the give DownNodes object.""" - return self.data.index(value.dict(), *args) - - def extend(self, other): - """Extend DownNodes list by appending DownNodes objects from the iterable.""" - if isinstance(other, DownNodesList): - self.data.extend(other.data) - else: - for down_node in other: - if not isinstance(down_node, DownNodes): - raise TypeError(f"{down_node} is not {type(DownNodes)}.") - - self.data.extend([v.dict() for v in other]) - - -class NodeSetMap(MutableMapping): - """Map of NodeSet names to dictionaries for composing `NodeSet` objects.""" - - def __init__(self, data: Optional[Dict[str, Dict[str, Any]]] = None): - if data is None: - data = {} - self.data = data - - @assert_type(value=NodeSet) - def __setitem__(self, key: str, value: NodeSet) -> None: - if key != value.node_set: - raise ValueError(f"{key} and {value.node_set} are not equal.") - else: - self.data.update(value.dict()) - - def __delitem__(self, key: str) -> None: - del self.data[key] - - def __getitem__(self, key: str) -> NodeSet: - try: - node_set = self.data.get(key) - return NodeSet(NodeSet=key, **node_set) - except KeyError: - raise KeyError(f"NodeSet {key} is not defined.") + @property + def frontend_name(self) -> str: + """Get frontend node name.""" + return self.__frontend_name - def __len__(self): - return len(self.data) - - def __iter__(self): - return iter([NodeSet(NodeSet=k, **self.data[k]) for k in self.data.keys()]) - - -class PartitionMap(MutableMapping): - """Map of Partition names to dictionaries for composing `Partition` objects.""" - - def __init__(self, data: Optional[Dict[str, Dict[str, Any]]] = None): - if data is None: - data = {} - self.data = data - - def __contains__(self, key: str): - return key in self.data - - def __len__(self): - return len(self.data) - - def __iter__(self): - return iter([Partition(PartitionName=k, **self.data[k]) for k in self.data.keys()]) + @frontend_name.setter + def frontend_name(self, name: str) -> None: + """Set new frontend node name.""" + self.__frontend_name = name - def __getitem__(self, key: str) -> Partition: - try: - partition = self.data.get(key) - return Partition(PartitionName=key, **partition) - except KeyError: - raise KeyError(f"Partition {key} is not defined.") - - @assert_type(value=Partition) - def __setitem__(self, key: str, value: Partition) -> None: - if key != value.partition_name: - raise ValueError(f"{key} and {value.partition_name} are not equal.") - else: - self.data.update(value.dict()) - - def __delitem__(self, key: str) -> None: - del self.data[key] + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "FrontendNode": + """Construct model from dictionary.""" + name = list(data.keys())[0] + return cls(FrontendName=name, **data[name]) + @classmethod + def from_str(cls, line: str) -> "FrontendNode": + """Construct model from configuration line.""" + data = parse_line(FrontendNodeOptions, line) + return cls(**data) -class SlurmConfig(BaseModel): - """Object representing the slurm.conf configuration file. - - Top-level configuration definition and data validators sourced from - the slurm.conf manpage. `man slurm.conf.5` - """ - - primary_key = None - callbacks = MappingProxyType( - { - "acct_storage_external_host": CommaSeparatorCallback, - "acct_storage_param": SlurmDictCallback, - "acct_storage_tres": CommaSeparatorCallback, - "acct_store_flags": CommaSeparatorCallback, - "auth_alt_types": CommaSeparatorCallback, - "auth_alt_param": SlurmDictCallback, - "auth_info": SlurmDictCallback, - "bcast_exclude": CommaSeparatorCallback, - "bcast_param": SlurmDictCallback, - "cli_filter_plugins": CommaSeparatorCallback, - "communication_params": SlurmDictCallback, - "cpu_freq_def": CommaSeparatorCallback, - "cpu_freq_governors": CommaSeparatorCallback, - "debug_flags": CommaSeparatorCallback, - "dependency_param": SlurmDictCallback, - "federation_param": CommaSeparatorCallback, - "health_check_node_state": CommaSeparatorCallback, - "job_acct_gather_frequency": SlurmDictCallback, - "job_comp_params": SlurmDictCallback, - "job_submit_plugins": CommaSeparatorCallback, - "launch_parameters": SlurmDictCallback, - "licenses": CommaSeparatorCallback, - "plugin_dir": ColonSeparatorCallback, - "power_parameters": SlurmDictCallback, - "preempt_mode": CommaSeparatorCallback, - "preempt_param": SlurmDictCallback, - "prep_plugins": CommaSeparatorCallback, - "priority_weight_tres": SlurmDictCallback, - "private_data": CommaSeparatorCallback, - "prolog_flags": CommaSeparatorCallback, - "propagate_resource_limits": CommaSeparatorCallback, - "propagate_resource_limits_except": CommaSeparatorCallback, - "scheduler_param": SlurmDictCallback, - "scron_param": CommaSeparatorCallback, - "slurmctld_param": SlurmDictCallback, - "slurmd_param": CommaSeparatorCallback, - "switch_param": SlurmDictCallback, - "task_plugin": CommaSeparatorCallback, - "task_plugin_param": SlurmDictCallback, - "topology_param": CommaSeparatorCallback, - } - ) - - include = property(*base_descriptors("Include")) - accounting_storage_backup_host = property(*base_descriptors("AccountingStorageBackupHost")) - accounting_storage_enforce = property(*base_descriptors("AccountingStorageEnforce")) - account_storage_external_host = property(*base_descriptors("AccountStorageExternalHost")) - accounting_storage_host = property(*base_descriptors("AccountingStorageHost")) - accounting_storage_parameters = property(*base_descriptors("AccountingStorageParameters")) - accounting_storage_pass = property(*base_descriptors("AccountingStoragePass")) - accounting_storage_port = property(*base_descriptors("AccountingStoragePort")) - accounting_storage_tres = property(*base_descriptors("AccountingStorageTRES")) - accounting_storage_type = property(*base_descriptors("AccountingStorageType")) - accounting_storage_user = property(*base_descriptors("AccountingStorageUser")) - accounting_store_flags = property(*base_descriptors("AccountingStoreFlags")) - acct_gather_node_freq = property(*base_descriptors("AcctGatherNodeFreq")) - acct_gather_energy_type = property(*base_descriptors("AcctGatherEnergyType")) - acct_gather_interconnect_type = property(*base_descriptors("AcctGatherInterconnectType")) - acct_gather_filesystem_type = property(*base_descriptors("AcctGatherFilesystemType")) - acct_gather_profile_type = property(*base_descriptors("AcctGatherProfileType")) - allow_spec_resources_usage = property(*base_descriptors("AllowSpecResourcesUsage")) - auth_alt_types = property(*base_descriptors("AuthAltTypes")) - auth_alt_parameters = property(*base_descriptors("AuthAltParameters")) - auth_info = property(*base_descriptors("AuthInfo")) - auth_type = property(*base_descriptors("AuthType")) - batch_start_timeout = property(*base_descriptors("BatchStartTimeout")) - bcast_exclude = property(*base_descriptors("BcastExclude")) - bcast_parameters = property(*base_descriptors("BcastParameters")) - burst_buffer_type = property(*base_descriptors("BurstBufferType")) - cli_filter_plugins = property(*base_descriptors("CliFilterPlugins")) - cluster_name = property(*base_descriptors("ClusterName")) - communication_parameters = property(*base_descriptors("CommunicationParameters")) - complete_wait = property(*base_descriptors("CompleteWait")) - core_spec_plugin = property(*base_descriptors("CoreSpecPlugin")) - cpu_freq_def = property(*base_descriptors("CpuFreqDef")) - cpu_freq_governors = property(*base_descriptors("CpuFreqGovernors")) - cred_type = property(*base_descriptors("CredType")) - debug_flags = property(*base_descriptors("DebugFlags")) - def_cpu_per_gpu = property(*base_descriptors("DefCpuPerGPU")) - def_mem_per_cpu = property(*base_descriptors("DefMemPerCPU")) - def_mem_per_gpu = property(*base_descriptors("DefMemPerGPU")) - def_mem_per_node = property(*base_descriptors("DefMemPerNode")) - dependency_parameters = property(*base_descriptors("DependencyParameters")) - disable_root_jobs = property(*base_descriptors("DisableRootJobs")) - eio_timeout = property(*base_descriptors("EioTimeout")) - enforce_part_limits = property(*base_descriptors("EnforcePartLimits")) - epilog = property(*base_descriptors("Epilog")) - epilog_msg_time = property(*base_descriptors("EpilogMsgTime")) - epilog_slurmctld = property(*base_descriptors("EpilogSlurmctld")) - ext_sensors_freq = property(*base_descriptors("ExtSensorsFreq")) - ext_sensors_type = property(*base_descriptors("ExtSensorsType")) - fair_share_dampening_factor = property(*base_descriptors("FairShareDampeningFactor")) - federation_parameters = property(*base_descriptors("FederationParameters")) - first_job_id = property(*base_descriptors("FirstJobId")) - get_env_timeout = property(*base_descriptors("GetEnvTimeout")) - gres_types = property(*base_descriptors("GresTypes")) - group_update_force = property(*base_descriptors("GroupUpdateForce")) - group_update_time = property(*base_descriptors("GroupUpdateTime")) - gpu_freq_def = property(*base_descriptors("GpuFreqDef")) - health_check_interval = property(*base_descriptors("HealthCheckInterval")) - health_check_node_state = property(*base_descriptors("HealthCheckNodeState")) - health_check_program = property(*base_descriptors("HealthCheckProgram")) - inactive_limit = property(*base_descriptors("InactiveLimit")) - interactive_step_options = property(*base_descriptors("InteractiveStepOptions")) - job_acct_gather_type = property(*base_descriptors("JobAcctGatherType")) - job_acct_gather_frequency = property(*base_descriptors("JobAcctGatherFrequency")) - job_acct_gather_params = property(*base_descriptors("JobAcctGatherParams")) - job_comp_host = property(*base_descriptors("JobCompHost")) - job_comp_loc = property(*base_descriptors("JobCompLoc")) - job_comp_params = property(*base_descriptors("JobCompParams")) - job_comp_pass = property(*base_descriptors("JobCompPass")) - job_comp_port = property(*base_descriptors("JobCompPort")) - job_comp_type = property(*base_descriptors("JobCompType")) - job_comp_user = property(*base_descriptors("JobCompUser")) - job_container_type = property(*base_descriptors("JobContainerType")) - job_file_append = property(*base_descriptors("JobFileAppend")) - job_requeue = property(*base_descriptors("JobRequeue")) - job_submit_plugins = property(*base_descriptors("JobSubmitPlugins")) - kill_on_bad_exit = property(*base_descriptors("KillOnBadExit")) - kill_wait = property(*base_descriptors("KillWait")) - max_batch_requeue = property(*base_descriptors("MaxBatchRequeue")) - node_features_plugins = property(*base_descriptors("NodeFeaturesPlugins")) - launch_parameters = property(*base_descriptors("LaunchParameters")) - licenses = property(*base_descriptors("Licenses")) - log_time_format = property(*base_descriptors("LogTimeFormat")) - mail_domain = property(*base_descriptors("MailDomain")) - mail_prog = property(*base_descriptors("MailProg")) - max_array_size = property(*base_descriptors("MaxArraySize")) - max_job_count = property(*base_descriptors("MaxJobCount")) - max_job_id = property(*base_descriptors("MaxJobId")) - max_mem_per_cpu = property(*base_descriptors("MaxMemPerCPU")) - max_mem_per_node = property(*base_descriptors("MaxMemPerNode")) - max_node_count = property(*base_descriptors("MaxNodeCount")) - max_step_count = property(*base_descriptors("MaxStepCount")) - max_tasks_per_node = property(*base_descriptors("MaxTasksPerNode")) - mcs_parameters = property(*base_descriptors("MCSParameters")) - mcs_plugin = property(*base_descriptors("MCSPlugin")) - message_timeout = property(*base_descriptors("MessageTimeout")) - min_job_age = property(*base_descriptors("MinJobAge")) - mpi_default = property(*base_descriptors("MpiDefault")) - mpi_params = property(*base_descriptors("MpiParams")) - over_time_limit = property(*base_descriptors("OverTimeLimit")) - plugin_dir = property(*base_descriptors("PluginDir")) - plug_stack_config = property(*base_descriptors("PlugStackConfig")) - power_parameters = property(*base_descriptors("PowerParameters")) - power_plugin = property(*base_descriptors("PowerPlugin")) - preempt_mode = property(*base_descriptors("PreemptMode")) - preempt_parameters = property(*base_descriptors("PreemptParameters")) - preempt_type = property(*base_descriptors("PreemptType")) - preempt_exempt_time = property(*base_descriptors("PreemptExemptTime")) - prep_parameters = property(*base_descriptors("PrEpParameters")) - prep_plugins = property(*base_descriptors("PrEpPlugins")) - priority_calcp_period = property(*base_descriptors("PriorityCalcpPeriod")) - priority_decay_half_life = property(*base_descriptors("PriorityDecayHalfLife")) - priority_favor_small = property(*base_descriptors("PriorityFavorSmall")) - priority_flags = property(*base_descriptors("PriorityFlags")) - priority_max_age = property(*base_descriptors("PriorityMaxAge")) - priority_parameters = property(*base_descriptors("PriorityParameters")) - priority_site_factor_parameters = property(*base_descriptors("PrioritySiteFactorParameters")) - priority_site_factor_plugin = property(*base_descriptors("PrioritySiteFactorPlugin")) - priority_type = property(*base_descriptors("PriorityType")) - priority_usage_reset_period = property(*base_descriptors("PriorityUsageResetPeriod")) - priority_weight_age = property(*base_descriptors("PriorityWeightAge")) - priority_weight_assoc = property(*base_descriptors("PriorityWeightAssoc")) - priority_weight_fair_share = property(*base_descriptors("PriorityWeightFairShare")) - priority_weight_job_size = property(*base_descriptors("PriorityWeightJobSize")) - priority_weight_partition = property(*base_descriptors("PriorityWeightPartition")) - priority_weight_qos = property(*base_descriptors("PriorityWeightQOS")) - priority_weight_tres = property(*base_descriptors("PriorityWeightTRES")) - private_data = property(*base_descriptors("PrivateData")) - proctrack_type = property(*base_descriptors("ProctrackType")) - prolog = property(*base_descriptors("Prolog")) - prolog_epilog_timeout = property(*base_descriptors("PrologEpilogTimeout")) - prolog_flags = property(*base_descriptors("PrologFlags")) - prolog_slurmctld = property(*base_descriptors("PrologSlurmctld")) - propagate_prio_process = property(*base_descriptors("PropagatePrioProcess")) - propagate_resource_limits = property(*base_descriptors("PropagateResourceLimits")) - propagate_resource_limits_except = property(*base_descriptors("PropagateResourceLimitsExcept")) - reboot_program = property(*base_descriptors("RebootProgram")) - reconfig_flags = property(*base_descriptors("ReconfigFlags")) - requeue_exit = property(*base_descriptors("RequeueExit")) - requeue_exit_hold = property(*base_descriptors("RequeueExitHold")) - resume_fail_program = property(*base_descriptors("ResumeFailProgram")) - resume_program = property(*base_descriptors("ResumeProgram")) - resume_rate = property(*base_descriptors("ResumeRate")) - resume_timeout = property(*base_descriptors("ResumeTimeout")) - resv_epilog = property(*base_descriptors("ResvEpilog")) - resv_over_run = property(*base_descriptors("ResvOverRun")) - resv_prolog = property(*base_descriptors("ResvProlog")) - return_to_service = property(*base_descriptors("ReturnToService")) - route_plugin = property(*base_descriptors("RoutePlugin")) - scheduler_parameters = property(*base_descriptors("SchedulerParameters")) - scheduler_time_slice = property(*base_descriptors("SchedulerTimeSlice")) - scheduler_type = property(*base_descriptors("SchedulerType")) - scron_parameters = property(*base_descriptors("ScronParameters")) - select_type = property(*base_descriptors("SelectType")) - select_type_parameters = property(*base_descriptors("SelectTypeParameters")) - slurmctld_addr = property(*base_descriptors("SlurmctldAddr")) - slurmctld_debug = property(*base_descriptors("SlurmctldDebug")) - slurmctld_host = property(*base_descriptors("SlurmctldHost")) - slurmctld_log_file = property(*base_descriptors("SlurmctldLogFile")) - slurmctld_parameters = property(*base_descriptors("SlurmctldParameters")) - slurmctld_pid_file = property(*base_descriptors("SlurmctldPidFile")) - slurmctld_port = property(*base_descriptors("SlurmctldPort")) - slurmctld_primary_off_prog = property(*base_descriptors("SlurmctldPrimaryOffProg")) - slurmctld_primary_on_prog = property(*base_descriptors("SlurmctldPrimaryOnProg")) - slurmctld_syslog_debug = property(*base_descriptors("SlurmctldSyslogDebug")) - slurmctld_timeout = property(*base_descriptors("SlurmctldTimeout")) - slurmd_debug = property(*base_descriptors("SlurmdDebug")) - slurmd_log_file = property(*base_descriptors("SlurmdLogFile")) - slurmd_parameters = property(*base_descriptors("SlurmdParameters")) - slurmd_pid_file = property(*base_descriptors("SlurmdPidFile")) - slurmd_port = property(*base_descriptors("SlurmdPort")) - slurmd_spool_dir = property(*base_descriptors("SlurmdSpoolDir")) - slurmd_syslog_debug = property(*base_descriptors("SlurmdSyslogDebug")) - slurmd_timeout = property(*base_descriptors("SlurmdTimeout")) - slurmd_user = property(*base_descriptors("SlurmdUser")) - slurm_sched_log_file = property(*base_descriptors("SlurmSchedLogFile")) - slurm_sched_log_level = property(*base_descriptors("SlurmSchedLogLevel")) - slurm_user = property(*base_descriptors("SlurmUser")) - srun_epilog = property(*base_descriptors("SrunEpilog")) - srun_port_range = property(*base_descriptors("SrunPortRange")) - srun_prolog = property(*base_descriptors("SrunProlog")) - state_save_location = property(*base_descriptors("StateSaveLocation")) - suspend_exc_nodes = property(*base_descriptors("SuspendExcNodes")) - suspend_exc_parts = property(*base_descriptors("SuspendExcParts")) - suspend_exc_states = property(*base_descriptors("SuspendExcStates")) - suspend_program = property(*base_descriptors("SuspendProgram")) - suspend_rate = property(*base_descriptors("SuspendRate")) - suspend_time = property(*base_descriptors("SuspendTime")) - suspend_timeout = property(*base_descriptors("SuspendTimeout")) - switch_parameters = property(*base_descriptors("SwitchParameters")) - switch_type = property(*base_descriptors("SwitchType")) - task_epilog = property(*base_descriptors("TaskEpilog")) - task_plugin = property(*base_descriptors("TaskPlugin")) - task_plugin_param = property(*base_descriptors("TaskPluginParam")) - task_prolog = property(*base_descriptors("TaskProlog")) - tcp_timeout = property(*base_descriptors("TCPTimeout")) - tmp_fs = property(*base_descriptors("TmpFS")) - topology_param = property(*base_descriptors("TopologyParam")) - topology_plugin = property(*base_descriptors("TopologyPlugin")) - track_wc_key = property(*base_descriptors("TrackWCKey")) - tree_width = property(*base_descriptors("TreeWidth")) - unkillable_step_program = property(*base_descriptors("UnkillableStepProgram")) - unkillable_step_timeout = property(*base_descriptors("UnkillableStepTimeout")) - use_pam = property(*base_descriptors("UsePAM")) - vsize_factor = property(*base_descriptors("VSizeFactor")) - wait_time = property(*base_descriptors("WaitTime")) - x11_parameters = property(*base_descriptors("X11Parameters")) + def dict(self) -> Dict[str, Any]: + """Return model as dictionary.""" + return copy.deepcopy({self.__frontend_name: self.data}) + + def __str__(self) -> str: + """Return model as configuration line.""" + line = [f"FrontendName={self.__frontend_name}"] + line.extend(marshall_content(FrontendNodeOptions, self.data)) + return " ".join(line) + + +class NodeSet(BaseModel, LineInterface): + """`NodeSet` data model.""" + + def __init__(self, *, NodeSet: str, **kwargs) -> None: # noqa N803 + self.__node_set = NodeSet + super().__init__(NodeSetOptions, **kwargs) @property - def nodes(self) -> NodeMap: - """Get all nodes in the current Slurm configuration.""" - return NodeMap(self._register["nodes"]) + def node_set(self) -> str: + """Get node set name.""" + return self.__node_set - @nodes.setter - @assert_type(value=NodeMap) - def nodes(self, value: NodeMap) -> None: - """Set new nodes in the current Slurm configuration. + @node_set.setter + def node_set(self, name: str) -> None: + """Set new node set name.""" + self.__node_set = name - Will overwrite any pre-existing nodes in the current configuration. - """ - self._register["nodes"] = value.data + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "NodeSet": + """Construct model from dictionary.""" + name = list(data.keys())[0] + return cls(NodeSet=name, **data[name]) + + @classmethod + def from_str(cls, line: str) -> "NodeSet": + """Construct model from configuration line.""" + data = parse_line(NodeSetOptions, line) + return cls(**data) + + def dict(self) -> Dict[str, Any]: + """Return model as dictionary.""" + return copy.deepcopy({self.__node_set: self.data}) + + def __str__(self) -> str: + """Return model as configuration line.""" + line = [f"NodeSet={self.__node_set}"] + line.extend(marshall_content(NodeSetOptions, self.data)) + return " ".join(line) - @nodes.deleter - def nodes(self) -> None: - """Delete all nodes from the current Slurm configuration.""" - self._register["nodes"] = {} + +class Partition(BaseModel, LineInterface): + """`Partition` data model.""" + + def __init__(self, *, PartitionName: str, **kwargs): # noqa N803 + self.__partition_name = PartitionName + super().__init__(PartitionOptions, **kwargs) @property - def frontend_nodes(self) -> FrontendNodeMap: - """Get all frontend nodes in the current Slurm configuration.""" - return FrontendNodeMap(self._register["frontend_nodes"]) + def partition_name(self) -> str: + """Get partition name.""" + return self.__partition_name + + @partition_name.setter + def partition_name(self, name: str) -> None: + """Set new partition name.""" + self.__partition_name = name + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "Partition": + """Construct model from dictionary.""" + name = list(data.keys())[0] + return cls(PartitionName=name, **data[name]) + + @classmethod + def from_str(cls, line: str) -> "Partition": + """Construct model from configuration line.""" + data = parse_line(PartitionOptions, line) + return cls(**data) + + def dict(self) -> Dict[str, Any]: + """Return model as dictionary.""" + return copy.deepcopy({self.__partition_name: self.data}) + + def __str__(self) -> str: + """Return model as configuration line.""" + line = [f"PartitionName={self.__partition_name}"] + line.extend(marshall_content(PartitionOptions, self.data)) + return " ".join(line) - @frontend_nodes.setter - @assert_type(value=FrontendNodeMap) - def frontend_nodes(self, value: FrontendNodeMap) -> None: - """Set new frontend nodes in the current Slurm configuration. - Will overwrite any pre-existing frontend nodes in the current configuration. - """ - self._register["frontend_nodes"] = value.data +class SlurmConfig(BaseModel): + """`slurm.conf` data model.""" + + def __init__( + self, + *, + Nodes: Dict[str, Any] = None, # noqa N803 + DownNodes: List[Dict[str, Any]] = None, # noqa N803 + FrontendNodes: Dict[str, Any] = None, # noqa N803 + NodeSets: Dict[str, Any] = None, # noqa N803 + Partitions: Dict[str, Any] = None, # noqa N803 + **kwargs, + ) -> None: + super().__init__(SlurmConfigOptions, **kwargs) + self.data["Nodes"] = Nodes or {} + self.data["DownNodes"] = DownNodes or [] + self.data["FrontendNodes"] = FrontendNodes or {} + self.data["NodeSets"] = NodeSets or {} + self.data["Partitions"] = Partitions or {} - @frontend_nodes.deleter - def frontend_nodes(self) -> None: - """Delete all frontend nodes from the current Slurm configuration.""" - self._register["frontend_nodes"] = {} + @property + def nodes(self): + """Get map of all nodes in the Slurm configuration.""" + return self.data["Nodes"] + + @nodes.setter + def nodes(self, value): + """Set new node mapping for the Slurm configuration.""" + self.data["Nodes"] = value + + @nodes.deleter + def nodes(self): + """Delete entire node mapping in the Slurm configuration.""" + self.data["Nodes"] = {} @property - def down_nodes(self) -> DownNodesList: - """Get all down nodes in the current Slurm configuration.""" - return DownNodesList(self._register["down_nodes"]) + def down_nodes(self): + """Get list of all down nodes in the Slurm configuration.""" + return self.data["DownNodes"] @down_nodes.setter - @assert_type(value=DownNodesList) - def down_nodes(self, value: DownNodesList) -> None: - """Set new down nodes in the current Slurm configuration. - - Will overwrite any pre-existing down nodes in the current configuration. - """ - self._register["down_nodes"] = value.data + def down_nodes(self, value): + """Set new down node list for the Slurm configuration.""" + self.data["DownNodes"] = value @down_nodes.deleter - def down_nodes(self) -> None: - """Delete all down nodes from the current Slurm configuration.""" - self._register["down_nodes"] = [] + def down_nodes(self): + """Delete entire down node list in the Slurm configuration.""" + self.data["DownNodes"] = [] @property - def node_sets(self) -> NodeSetMap: - """Get all node sets in the current Slurm configuration.""" - return NodeSetMap(self._register["node_sets"]) + def frontend_nodes(self): + """Get map of all frontend nodes in the Slurm configuration.""" + return self.data["FrontendNodes"] - @node_sets.setter - @assert_type(value=NodeSetMap) - def node_sets(self, value: NodeSetMap) -> None: - """Set new node sets in the current Slurm configuration. + @frontend_nodes.setter + def frontend_nodes(self, value): + """Set new frontend node mapping for the Slurm configuration.""" + self.data["FrontendNodes"] = value - Will overwrite any pre-existing node sets in the current configuration. - """ - self._register["node_sets"] = value.data + @frontend_nodes.deleter + def frontend_nodes(self): + """Delete entire frontend node mapping in the Slurm configuration.""" + self.data["FrontendNodes"] = {} + + @property + def node_sets(self): + """Get map of all node sets in the Slurm configuration.""" + return self.data["NodeSets"] + + @node_sets.setter + def node_sets(self, value): + """Set new node set mapping for the Slurm configuration.""" + self.data["NodeSets"] = value @node_sets.deleter - def node_sets(self) -> None: - """Delete all node sets from the current Slurm configuration.""" - self._register["node_sets"] = {} + def node_sets(self): + """Delete entire node set mapping in the Slurm configuration.""" + self.data["NodeSets"] = {} @property - def partitions(self) -> PartitionMap: - """Get all partitions in the current Slurm configuration.""" - return PartitionMap(self._register["partitions"]) + def partitions(self): + """Get map of all partitions in the Slurm configuration.""" + return self.data["Partitions"] @partitions.setter - @assert_type(value=PartitionMap) - def partitions(self, value: PartitionMap) -> None: - """Set new partitions in the current Slurm configuration. - - Will overwrite any pre-existing partitions in the current configuration. - """ - self._register["partitions"] = value.data + def partitions(self, value): + """Set partition mapping for the Slurm configuration.""" + self.data["Partitions"] = value @partitions.deleter - def partitions(self) -> None: - """Delete all partitions from the current Slurm configuration.""" - self._register["partitions"] = {} + def partitions(self): + """Delete entire partition mapping in the Slurm configuration.""" + self.data["Partitions"] = {} + + +for opt in NodeOptions.keys(): + setattr(Node, format_key(opt), property(*generate_descriptors(opt))) +for opt in DownNodeOptions.keys(): + setattr(DownNodes, format_key(opt), property(*generate_descriptors(opt))) +for opt in FrontendNodeOptions.keys(): + setattr(FrontendNode, format_key(opt), property(*generate_descriptors(opt))) +for opt in NodeSetOptions.keys(): + setattr(NodeSet, format_key(opt), property(*generate_descriptors(opt))) +for opt in PartitionOptions.keys(): + setattr(Partition, format_key(opt), property(*generate_descriptors(opt))) +for opt in SlurmConfigOptions.keys(): + setattr(SlurmConfig, format_key(opt), property(*generate_descriptors(opt))) diff --git a/slurmutils/models/slurmdbd.py b/slurmutils/models/slurmdbd.py index 4473578..92ed064 100644 --- a/slurmutils/models/slurmdbd.py +++ b/slurmutils/models/slurmdbd.py @@ -12,87 +12,18 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . -"""Data models for the slurmdbd daemon.""" +"""Data models for `slurmdbd.conf` configuration file.""" -from types import MappingProxyType - -from ._model import ( - BaseModel, - ColonSeparatorCallback, - CommaSeparatorCallback, - SlurmDictCallback, - base_descriptors, -) +from .model import BaseModel, format_key, generate_descriptors +from .option import SlurmdbdConfigOptions class SlurmdbdConfig(BaseModel): - """Object representing the slurmdbd.conf configuration file. + """`slurmdbd.conf` data model.""" - Top-level configuration definition and data validators sourced from - the slurmdbd.conf manpage. `man slurmdbd.conf.5` - """ + def __init__(self, **kwargs): + super().__init__(SlurmdbdConfigOptions, **kwargs) - primary_key = None - callbacks = MappingProxyType( - { - "auth_alt_types": CommaSeparatorCallback, - "auth_alt_parameters": SlurmDictCallback, - "communication_parameters": SlurmDictCallback, - "debug_flags": CommaSeparatorCallback, - "parameters": CommaSeparatorCallback, - "plugin_dir": ColonSeparatorCallback, - "private_data": CommaSeparatorCallback, - "storage_parameters": SlurmDictCallback, - } - ) - archive_dir = property(*base_descriptors("ArchiveDir")) - archive_events = property(*base_descriptors("ArchiveEvents")) - archive_jobs = property(*base_descriptors("ArchiveJobs")) - archive_resvs = property(*base_descriptors("ArchiveResvs")) - archive_script = property(*base_descriptors("ArchiveScript")) - archive_steps = property(*base_descriptors("ArchiveSteps")) - archive_suspend = property(*base_descriptors("ArchiveSuspend")) - archive_txn = property(*base_descriptors("ArchiveTXN")) - archive_usage = property(*base_descriptors("ArchiveUsage")) - auth_info = property(*base_descriptors("AuthInfo")) - auth_alt_types = property(*base_descriptors("AuthAltTypes")) - auth_alt_parameters = property(*base_descriptors("AuthAltParameters")) - auth_type = property(*base_descriptors("AuthType")) - commit_delay = property(*base_descriptors("CommitDelay")) - communication_parameters = property(*base_descriptors("CommunicationParameters")) - dbd_backup_host = property(*base_descriptors("DbdBackupHost")) - dbd_addr = property(*base_descriptors("DbdAddr")) - dbd_host = property(*base_descriptors("DbdHost")) - dbd_port = property(*base_descriptors("DbdPort")) - debug_flags = property(*base_descriptors("DebugFlags")) - debug_level = property(*base_descriptors("DebugLevel")) - debug_level_syslog = property(*base_descriptors("DebugLevelSyslog")) - default_qos = property(*base_descriptors("DefaultQOS")) - log_file = property(*base_descriptors("LogFile")) - log_time_format = property(*base_descriptors("LogTimeFormat")) - max_query_time_range = property(*base_descriptors("MaxQueryTimeRange")) - message_timeout = property(*base_descriptors("MessageTimeout")) - parameters = property(*base_descriptors("Parameters")) - pid_file = property(*base_descriptors("PidFile")) - plugin_dir = property(*base_descriptors("PluginDir")) - private_data = property(*base_descriptors("PrivateData")) - purge_event_after = property(*base_descriptors("PurgeEventAfter")) - purge_job_after = property(*base_descriptors("PurgeJobAfter")) - purge_resv_after = property(*base_descriptors("PurgeResvAfter")) - purge_step_after = property(*base_descriptors("PurgeStepAfter")) - purge_suspend_after = property(*base_descriptors("PurgeSuspendAfter")) - purge_txn_after = property(*base_descriptors("PurgeTXNAfter")) - purge_usage_after = property(*base_descriptors("PurgeUsageAfter")) - slurm_user = property(*base_descriptors("SlurmUser")) - storage_host = property(*base_descriptors("StorageHost")) - storage_backup_host = property(*base_descriptors("StorageBackupHost")) - storage_loc = property(*base_descriptors("StorageLoc")) - storage_parameters = property(*base_descriptors("StorageParameters")) - storage_pass = property(*base_descriptors("StoragePass")) - storage_port = property(*base_descriptors("StoragePort")) - storage_type = property(*base_descriptors("StorageType")) - storage_user = property(*base_descriptors("StorageUser")) - tcp_timeout = property(*base_descriptors("TCPTimeout")) - track_slurmctld_down = property(*base_descriptors("TrackSlurmctldDown")) - track_wc_key = property(*base_descriptors("TrackWCKey")) +for opt in SlurmdbdConfigOptions.keys(): + setattr(SlurmdbdConfig, format_key(opt), property(*generate_descriptors(opt))) From 43992cae4426f0a4741f4bea5bbd6539a82351fc Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Mon, 26 Aug 2024 10:57:11 -0400 Subject: [PATCH 07/19] feat: add `BaseError` and `ModelError` for more granular exceptions Signed-off-by: Jason C. Nucciarone --- slurmutils/exceptions.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/slurmutils/exceptions.py b/slurmutils/exceptions.py index 253c669..18022ee 100644 --- a/slurmutils/exceptions.py +++ b/slurmutils/exceptions.py @@ -15,5 +15,18 @@ """Exceptions raised by Slurm utilities in this package.""" -class EditorError(Exception): +class BaseError(Exception): + """Base exception for errors in `slurmutils` module.""" + + @property + def message(self) -> str: + """Return message passed as argument to exception.""" + return self.args[0] + + +class EditorError(BaseError): """Raise when a Slurm configuration editor encounters an error.""" + + +class ModelError(BaseError): + """Raise when a Slurm configuration model encounters an error.""" From a28785f1555542ab0b2cb24acee575e0c89aeffe Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Mon, 26 Aug 2024 10:58:08 -0400 Subject: [PATCH 08/19] tests: update unit tests for slurmconfig editor * Use from_dict method to create data models. Signed-off-by: Jason C. Nucciarone --- tests/unit/editors/test_slurmconfig.py | 175 +++++++++++++------------ 1 file changed, 94 insertions(+), 81 deletions(-) diff --git a/tests/unit/editors/test_slurmconfig.py b/tests/unit/editors/test_slurmconfig.py index 45fc6c0..af89008 100644 --- a/tests/unit/editors/test_slurmconfig.py +++ b/tests/unit/editors/test_slurmconfig.py @@ -19,7 +19,7 @@ from pathlib import Path from slurmutils.editors import slurmconfig -from slurmutils.models import DownNodes, DownNodesList, Node, NodeMap, Partition, PartitionMap +from slurmutils.models import DownNodes, Node, Partition example_slurm_conf = """# # `slurm.conf` file generated at 2024-01-30 17:18:36.171652 by slurmutils. @@ -87,32 +87,30 @@ def test_loads(self) -> None: self.assertEqual(config.slurmd_spool_dir, "/var/spool/slurmd.spool") self.assertEqual(config.scheduler_type, "sched/backfill") - nodes = config.nodes - for node in nodes: - self.assertIn( - node.node_name, + for name, params in config.nodes.items(): + self.assertIn( # codespell:ignore + name, {"juju-c9fc6f-2", "juju-c9fc6f-3", "juju-c9fc6f-4", "juju-c9fc6f-5"}, ) - self.assertIn( - node.node_addr, {"10.152.28.48", "10.152.28.49", "10.152.28.50", "10.152.28.51"} + self.assertIn( # codespell:ignore + params["NodeAddr"], + {"10.152.28.48", "10.152.28.49", "10.152.28.50", "10.152.28.51"}, ) - self.assertEqual(node.cpus, "1") - self.assertEqual(node.real_memory, "1000") - self.assertEqual(node.tmp_disk, "10000") + self.assertEqual(params["CPUs"], "1") + self.assertEqual(params["RealMemory"], "1000") + self.assertEqual(params["TmpDisk"], "10000") - down_nodes = config.down_nodes - for entry in down_nodes: - self.assertEqual(entry.down_nodes[0], "juju-c9fc6f-5") - self.assertEqual(entry.state, "DOWN") - self.assertEqual(entry.reason, "Maintenance Mode") + for entry in config.down_nodes: + self.assertEqual(entry["DownNodes"][0], "juju-c9fc6f-5") + self.assertEqual(entry["State"], "DOWN") + self.assertEqual(entry["Reason"], "Maintenance Mode") - partitions = config.partitions - for part in partitions: - self.assertIn(part.partition_name, {"DEFAULT", "batch"}) + for partition in config.partitions: + self.assertIn(partition, {"DEFAULT", "batch"}) # codespell:ignore - batch = partitions["batch"] + batch = config.partitions["batch"] self.assertListEqual( - batch.nodes, ["juju-c9fc6f-2", "juju-c9fc6f-3", "juju-c9fc6f-4", "juju-c9fc6f-5"] + batch["Nodes"], ["juju-c9fc6f-2", "juju-c9fc6f-3", "juju-c9fc6f-4", "juju-c9fc6f-5"] ) def test_dumps(self) -> None: @@ -130,10 +128,9 @@ def test_edit(self) -> None: config.max_job_count = 20000 config.proctrack_type = "proctrack/linuxproc" config.plugin_dir.append("/snap/slurm/current/plugins") - node = config.nodes["juju-c9fc6f-2"] + new_node = Node(NodeName="batch-0", **config.nodes["juju-c9fc6f-2"]) del config.nodes["juju-c9fc6f-2"] - node.node_name = "batch-0" - config.nodes[node.node_name] = node + config.nodes.update(new_node.dict()) config = slurmconfig.load("slurm.conf") self.assertIsNone(config.inactive_limit) @@ -143,9 +140,8 @@ def test_edit(self) -> None: config.plugin_dir, ["/usr/local/lib", "/usr/local/slurm/lib", "/snap/slurm/current/plugins"], ) - self.assertEqual(config.nodes["batch-0"].node_addr, "10.152.28.48") + self.assertEqual(config.nodes["batch-0"]["NodeAddr"], "10.152.28.48") - # Test pocket (`nodes`, `frontend_nodes`, ...) descriptors. with slurmconfig.edit("slurm.conf") as config: del config.nodes del config.frontend_nodes @@ -154,74 +150,91 @@ def test_edit(self) -> None: del config.partitions config = slurmconfig.load("slurm.conf") - self.assertDictEqual(config.nodes.data, {}) - self.assertDictEqual(config.frontend_nodes.data, {}) - self.assertListEqual(config.down_nodes.data, []) - self.assertDictEqual(config.node_sets.data, {}) - self.assertDictEqual(config.partitions.data, {}) - - node_list = [ - Node( - NodeName="juju-c9fc6f-2", - NodeAddr="10.152.28.48", - CPUs="1", - RealMemory="1000", - TmpDisk="10000", + self.assertDictEqual(config.nodes, {}) + self.assertDictEqual(config.frontend_nodes, {}) + self.assertListEqual(config.down_nodes, []) + self.assertDictEqual(config.node_sets, {}) + self.assertDictEqual(config.partitions, {}) + + new_nodes = [ + Node.from_dict( + { + "juju-c9fc6f-2": { + "NodeAddr": "10.152.28.48", + "CPUs": "1", + "RealMemory": "1000", + "TmpDisk": "10000", + } + } ), - Node( - NodeName="juju-c9fc6f-3", - NodeAddr="10.152.28.49", - CPUs="1", - RealMemory="1000", - TmpDisk="10000", + Node.from_dict( + { + "juju-c9fc6f-3": { + "NodeAddr": "10.152.28.49", + "CPUs": "1", + "RealMemory": "1000", + "TmpDisk": "10000", + } + } ), - Node( - NodeName="juju-c9fc6f-4", - NodeAddr="10.152.28.50", - CPUs="1", - RealMemory="1000", - TmpDisk="10000", + Node.from_dict( + { + "juju-c9fc6f-4": { + "NodeAddr": "10.152.28.50", + "CPUs": "1", + "RealMemory": "1000", + "TmpDisk": "10000", + } + } ), - Node( - NodeName="juju-c9fc6f-5", - NodeAddr="10.152.28.51", - CPUs="1", - RealMemory="1000", - TmpDisk="10000", + Node.from_dict( + { + "juju-c9fc6f-5": { + "NodeAddr": "10.152.28.51", + "CPUs": "1", + "RealMemory": "1000", + "TmpDisk": "10000", + } + } ), ] - down_nodes = [ - DownNodes(DownNodes=["juju-c9fc6f-5"], State="DOWN", Reason="Maintenance Mode") + new_down_nodes = [ + DownNodes.from_dict( + { + "DownNodes": ["juju-c9fc6f-5"], + "State": "DOWN", + "Reason": "Maintenance Mode", + } + ) ] - partition_list = [ - Partition(PartitionName="DEFAULT", MaxTime="30", MaxNodes="10", State="UP"), - Partition( - PartitionName="batch", - Nodes=["juju-c9fc6f-2", "juju-c9fc6f-3", "juju-c9fc6f-4", "juju-c9fc6f-5"], - MinNodes="4", - MaxTime="120", - AllowGroups=["admin"], + new_partitions = [ + Partition.from_dict({"DEFAULT": {"MaxTime": "30", "MaxNodes": "10", "State": "UP"}}), + Partition.from_dict( + { + "batch": { + "Nodes": [ + "juju-c9fc6f-2", + "juju-c9fc6f-3", + "juju-c9fc6f-4", + "juju-c9fc6f-5", + ], + "MinNodes": "4", + "MaxTime": "120", + "AllowGroups": ["admin"], + } + } ), ] with slurmconfig.edit("slurm.conf") as config: - node_map = NodeMap() - for node in node_list: - node_map[node.node_name] = node - config.nodes = node_map - - down_nodes_list = DownNodesList() - down_nodes_list.extend(down_nodes) - config.down_nodes = down_nodes_list + for node in new_nodes: + config.nodes.update(node.dict()) - partition_map = PartitionMap() - for part in partition_list: - partition_map[part.partition_name] = part - config.partitions = partition_map + for down_node in new_down_nodes: + config.down_nodes.append(down_node.dict()) - config = slurmconfig.load("slurm.conf") - with self.assertRaises(TypeError): - config.frontend_nodes = "yowzah" + for partition in new_partitions: + config.partitions.update(partition.dict()) def tearDown(self): Path("slurm.conf").unlink() From 4afd84343f02dd2bc491afccb6c139b72ca8c39b Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Mon, 26 Aug 2024 10:59:29 -0400 Subject: [PATCH 09/19] chore(deps): remove renovate * Renovate is no longer used on repository to update dependencies. Dependabot is used instead as it does not require an open issue on the issue tracker to work. Signed-off-by: Jason C. Nucciarone --- renovate.json | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 renovate.json diff --git a/renovate.json b/renovate.json deleted file mode 100644 index 39a2b6e..0000000 --- a/renovate.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": [ - "config:base" - ] -} From 6df3071cd9541f0deb3d72db98f9992cc7197498 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Mon, 26 Aug 2024 11:37:33 -0400 Subject: [PATCH 10/19] chore(lint): use explicit re-exports in module __init__.py Signed-off-by: Jason C. Nucciarone --- slurmutils/editors/__init__.py | 4 ++-- slurmutils/models/__init__.py | 16 +++++++--------- slurmutils/models/slurm.py | 9 +++++++++ 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/slurmutils/editors/__init__.py b/slurmutils/editors/__init__.py index 3deb773..4bf472e 100644 --- a/slurmutils/editors/__init__.py +++ b/slurmutils/editors/__init__.py @@ -14,5 +14,5 @@ """Editors for Slurm workload manager configuration files.""" -from . import slurmconfig -from . import slurmdbdconfig +from . import slurmconfig as slurmconfig +from . import slurmdbdconfig as slurmdbdconfig diff --git a/slurmutils/models/__init__.py b/slurmutils/models/__init__.py index 65a8922..3ff5c44 100644 --- a/slurmutils/models/__init__.py +++ b/slurmutils/models/__init__.py @@ -14,12 +14,10 @@ """Data models for common Slurm objects.""" -from .slurm import ( - DownNodes, - FrontendNode, - Node, - NodeSet, - Partition, - SlurmConfig, -) -from .slurmdbd import SlurmdbdConfig +from .slurm import DownNodes as DownNodes +from .slurm import FrontendNode as FrontendNode +from .slurm import Node as Node +from .slurm import NodeSet as NodeSet +from .slurm import Partition as Partition +from .slurm import SlurmConfig as SlurmConfig +from .slurmdbd import SlurmdbdConfig as SlurmdbdConfig diff --git a/slurmutils/models/slurm.py b/slurmutils/models/slurm.py index 920c4e3..606345d 100644 --- a/slurmutils/models/slurm.py +++ b/slurmutils/models/slurm.py @@ -14,6 +14,15 @@ """Data models for `slurm.conf` configuration file.""" +__all__ = [ + "Node", + "DownNodes", + "FrontendNode", + "NodeSet", + "Partition", + "SlurmConfig", +] + import copy from typing import Any, Dict, List From 72797ff4ad9761e04f16f6e56257467a6a46c4d7 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Mon, 26 Aug 2024 12:56:32 -0400 Subject: [PATCH 11/19] docs: slurm.conf -> slurmdbd.conf in function docstrings Signed-off-by: Jason C. Nucciarone --- slurmutils/editors/slurmdbdconfig.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/slurmutils/editors/slurmdbdconfig.py b/slurmutils/editors/slurmdbdconfig.py index 6be5989..9ee7208 100644 --- a/slurmutils/editors/slurmdbdconfig.py +++ b/slurmutils/editors/slurmdbdconfig.py @@ -38,32 +38,32 @@ @loader def load(file: Union[str, os.PathLike]) -> SlurmdbdConfig: - """Load `slurm.conf` data model from slurm.conf file.""" + """Load `slurmdbd.conf` data model from slurmdbd.conf file.""" return loads(Path(file).read_text()) def loads(content: str) -> SlurmdbdConfig: - """Load `slurm.conf` data model from string.""" + """Load `slurmdbd.conf` data model from string.""" return _parse(content) @dumper def dump(config: SlurmdbdConfig, file: Union[str, os.PathLike]) -> None: - """Dump `slurm.conf` data model into slurm.conf file.""" + """Dump `slurmdbd.conf` data model into slurmdbd.conf file.""" Path(file).write_text(dumps(config)) def dumps(config: SlurmdbdConfig) -> str: - """Dump `slurm.conf` data model into a string.""" + """Dump `slurmdbd.conf` data model into a string.""" return _marshall(config) @contextmanager def edit(file: Union[str, os.PathLike]) -> SlurmdbdConfig: - """Edit a slurm.conf file. + """Edit a slurmdbd.conf file. Args: - file: Path to slurm.conf file to edit. If slurm.conf does + file: Path to slurmdbd.conf file to edit. If slurmdbd.conf does not exist at the specified file path, it will be created. """ if not os.path.exists(file): From 8b6f6b583e1817ceaa04cfc70b88f8d9e01c5da4 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Mon, 26 Aug 2024 12:57:44 -0400 Subject: [PATCH 12/19] refactor: use debug logging level instead of warning for ignored comment lines Signed-off-by: Jason C. Nucciarone --- slurmutils/editors/slurmconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slurmutils/editors/slurmconfig.py b/slurmutils/editors/slurmconfig.py index 7aeac91..06ee299 100644 --- a/slurmutils/editors/slurmconfig.py +++ b/slurmutils/editors/slurmconfig.py @@ -86,7 +86,7 @@ def _parse(content: str) -> SlurmConfig: for index, line in enumerate(lines): config, ignore = clean(line) if ignore: - _logger.warning("ignoring line %s at index %s in slurm.conf", line, index) + _logger.debug("ignoring line %s at index %s in slurm.conf", line, index) continue if config.startswith("Include"): From 1eb6d94de6871fd9d920727061ef9d5da325fd0d Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Mon, 26 Aug 2024 13:30:12 -0400 Subject: [PATCH 13/19] docs: update README to match other Charmed HPC projects Signed-off-by: Jason C. Nucciarone --- README.md | 94 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 60 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index c12abf7..ebfca90 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,30 @@ -
- # slurmutils -Utilities and APIs for interfacing with the Slurm workload manager. - -[![Matrix](https://img.shields.io/matrix/ubuntu-hpc%3Amatrix.org?logo=matrix&label=ubuntu-hpc)](https://matrix.to/#/#ubuntu-hpc:matrix.org) +![PyPI - Version](https://img.shields.io/pypi/v/slurmutils) +![PyPI - Downloads](https://img.shields.io/pypi/dm/slurmutils) +![GitHub License](https://img.shields.io/github/license/charmed-hpc/slurmutils) +[![Matrix](https://img.shields.io/matrix/ubuntu-hpc%3Amatrix.org?logo=matrix&label=ubuntu-hpc)](https://matrix.to/#/#hpc:ubuntu.com) -
- -## Features +Utilities and APIs for interfacing with the Slurm workload manager. -`slurmutils` is a collection of various utilities and APIs to make it easier +slurmutils is a collection of various utilities that make it easier for you and your friends to interface with the Slurm workload manager, especially if you are orchestrating deployments of new and current Slurm clusters. Gone are the days of -seething over incomplete Jinja2 templates. Current utilities and APIs shipped in the -`slurmutils` package include: +seething over incomplete Jinja2 templates. Current utilities shipped in the +slurmutils package include: #### `from slurmutils.editors import ...` -* `slurmconfig`: An editor _slurm.conf_ and _Include_ files. -* `slurmdbdconfig`: An editor for _slurmdbd.conf_ files. +* `slurmconfig`: An editor for _slurm.conf_ configuration files. +* `slurmdbdconfig`: An editor for _slurmdbd.conf_ configuration files. -## Installation +For more information on how to use or contribute to slurmutils, +check out the [Getting Started](#-getting-started) and [Development](#-development) +sections below 👇 + +## ✨ Getting Started + +### Installation #### Option 1: Install from PyPI @@ -32,7 +35,7 @@ $ python3 -m pip install slurmutils #### Option 2: Install from source We use the [Poetry](https://python-poetry.org) packaging and dependency manager to -manage this project. It must be installed on your system if installing `slurmutils` +manage this project. It must be installed on your system if installing slurmutils from source. ```shell @@ -41,17 +44,17 @@ $ cd slurmutils $ poetry install ``` -## Usage +### Usage -### Editors +#### Editors -#### `slurmconfig` +##### `slurmconfig` This module provides an API for editing both _slurm.conf_ and _Include_ files, and can create new configuration files if they do not exist. Here's some common Slurm lifecycle management operators you can perform using this editor: -##### Edit a pre-existing _slurm.conf_ configuration file +###### Edit a pre-existing _slurm.conf_ configuration file ```python from slurmutils.editors import slurmconfig @@ -63,7 +66,7 @@ with slurmconfig.edit("/etc/slurm/slurm.conf") as config: config.proctrack_type = "proctrack/linuxproc" ``` -##### Add a new node to the _slurm.conf_ file +###### Add a new node to the _slurm.conf_ file ```python from slurmutils.editors import slurmconfig @@ -77,16 +80,16 @@ with slurmconfig.edit("/etc/slurm/slurm.conf") as config: RealMemory=1000, TmpDisk=10000, ) - config.nodes[node.node_name] = node + config.nodes.update(node.dict()) ``` -#### `slurmdbdconfig` +##### `slurmdbdconfig` This module provides and API for editing _slurmdbd.conf_ files, and can create new _slurmdbd.conf_ files if they do not exist. Here's some operations you can perform on the _slurmdbd.conf_ file using this editor: -##### Edit a pre-existing _slurmdbd.conf_ configuration file +###### Edit a pre-existing _slurmdbd.conf_ configuration file ```python from slurmutils.editors import slurmdbdconfig @@ -99,19 +102,42 @@ with slurmdbdconfig.edit("/etc/slurm/slurmdbd.conf") as config: del config.auth_alt_parameters ``` -## Project & Community +## 🤔 What's next? + +If you want to learn more about all the things you can do with slurmutils, +here are some further resources for you to explore: + +* [Open an issue](https://github.com/charmed-hpc/slurmutils/issues/new?title=ISSUE+TITLE&body=*Please+describe+your+issue*) +* [Ask a question on Github](https://github.com/orgs/charmed-hpc/discussions/categories/q-a) + +## 🛠️ Development + +This project uses [tox](https://tox.wiki) as its command runner, which provides +some useful commands that will help you while hacking on slurmutils: + +```shell +tox run -e fmt # Apply formatting standards to code. +tox run -e lint # Check code against coding style standards. +tox run -e unit # Run unit tests. +``` + +If you're interested in contributing your work to slurmutils, +take a look at our [contributing guidelines](./CONTRIBUTING.md) for further details. + +## 🤝 Project and community + +slurmutils is a project of the [Ubuntu High-Performance Computing community](https://ubuntu.com/community/governance/teams/hpc). +Interested in contributing bug fixes, new editors, documentation, or feedback? Want to join the Ubuntu HPC community? You’ve come to the right place 🤩 -The `slurmutils` package is a project of the -[Ubuntu HPC](https://discourse.ubuntu.com/t/high-performance-computing-team/35988) community. -It is an open-source project that is welcome to community involvement, contributions, suggestions, fixes, -and constructive feedback. Interested in being involved with the development of `slurmutils`? -Check out these links below: +Here’s some links to help you get started with joining the community: -* [Join our online chat](https://matrix.to/#/#ubuntu-hpc:matrix.org) -* [Code of Conduct](https://ubuntu.com/community/code-of-conduct) +* [Ubuntu Code of Conduct](https://ubuntu.com/community/ethos/code-of-conduct) * [Contributing guidelines](./CONTRIBUTING.md) +* [Join the conversation on Matrix](https://matrix.to/#/#hpc:ubuntu.com) +* [Get the latest news on Discourse](https://discourse.ubuntu.com/c/hpc/151) +* [Ask and answer questions on GitHub](https://github.com/orgs/charmed-hpc/discussions/categories/q-a) -## License +## 📋 License -The `slurmutils` package is free software, distributed under the GNU Lesser General Public License, v3.0. -See the [LICENSE](./LICENSE) file for more information. +slurmutils is free software, distributed under the GNU Lesser General Public License, v3.0. +See the [LGPL-3.0 LICENSE](./LICENSE) file for further details. From 82d9ed49cd3b34f79f2e4ee6f4fe875569615e9d Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Mon, 26 Aug 2024 13:42:44 -0400 Subject: [PATCH 14/19] fix: use f-string to generate error message if config file is not found Signed-off-by: Jason C. Nucciarone --- slurmutils/editors/editor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slurmutils/editors/editor.py b/slurmutils/editors/editor.py index e91a525..9deb4b6 100644 --- a/slurmutils/editors/editor.py +++ b/slurmutils/editors/editor.py @@ -108,7 +108,7 @@ def loader(func): def wrapper(*args, **kwargs): fin = args[0] if not path.exists(fin): - raise FileNotFoundError("could not locate %s", fin) + raise FileNotFoundError(f"could not locate {fin}") _logger.debug("reading contents of %s", fin) return func(*args, **kwargs) From b19f9deeb8cd4bef81af13f7a5cca01949856d35 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Mon, 26 Aug 2024 13:54:58 -0400 Subject: [PATCH 15/19] feat: version bump and pypi metadata update Signed-off-by: Jason C. Nucciarone --- pyproject.toml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 099b3bf..7b6c760 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,11 +18,14 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "slurmutils" -version = "0.4.0" +version = "0.5.0" description = "Utilities and APIs for interfacing with the Slurm workload manager." -repository = "https://github.com/canonical/slurmutils" -authors = ["Jason C. Nucciarone "] -maintainers = ["Jason C. Nucciarone "] +repository = "https://github.com/charmed-hpc/slurmutils" +authors = ["Jason C. Nucciarone "] +maintainers = [ + "Jason C. Nucciarone ", + "Ubuntu High-Performance Computing " +] license = "LGPL-3.0-only" readme = "README.md" keywords = ["HPC", "administration", "orchestration", "utility"] @@ -41,7 +44,7 @@ classifiers=[ python = ">=3.8" [tool.poetry.urls] -"Bug Tracker" = "https://github.com/canonical/slurmutils/issues" +"Bug Tracker" = "https://github.com/charmed-hpc/slurmutils/issues" # Testing tools configuration [tool.coverage.run] From 87e6fb2c690a86e6fccdd662acdf5806da72648c Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Mon, 26 Aug 2024 15:56:10 -0400 Subject: [PATCH 16/19] refactor: make `clean` less Go-ish * Have `clean(...)` return None if line is a comment rather than a separate boolean value. Signed-off-by: Jason C. Nucciarone --- slurmutils/editors/editor.py | 27 ++++----------------------- slurmutils/editors/slurmconfig.py | 4 ++-- slurmutils/editors/slurmdbdconfig.py | 4 ++-- 3 files changed, 8 insertions(+), 27 deletions(-) diff --git a/slurmutils/editors/editor.py b/slurmutils/editors/editor.py index 9deb4b6..c416fbb 100644 --- a/slurmutils/editors/editor.py +++ b/slurmutils/editors/editor.py @@ -18,39 +18,20 @@ import shlex from functools import wraps from os import path -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, List, Optional from ..exceptions import EditorError _logger = logging.getLogger("slurmutils") -def _is_comment(line: str) -> bool: - """Check if line is a comment.""" - return line.startswith("#") - - -def _contains_comment(line: str) -> bool: - """Check if line contains an inline comment.""" - return "#" in line - - -def _slice_comment(line: str) -> str: - """Slice inline comment off of line.""" - return line.split("#", maxsplit=1)[0] - - -def clean(line: str) -> Tuple[str, bool]: +def clean(line: str) -> Optional[str]: """Clean line before further processing. Returns: - Returns the cleaned line and False if it should be ignored. - If True, then the processors should ignore the line. + Line with inline comments removed. `None` if line is a comment. """ - if _is_comment(line): - return "", True - - return (_slice_comment(line) if _contains_comment(line) else line).strip(), False + return cleaned if (cleaned := line.split("#", maxsplit=1)[0]) != "" else None def parse_line(options, line: str) -> Dict[str, Any]: diff --git a/slurmutils/editors/slurmconfig.py b/slurmutils/editors/slurmconfig.py index 06ee299..64759db 100644 --- a/slurmutils/editors/slurmconfig.py +++ b/slurmutils/editors/slurmconfig.py @@ -84,8 +84,8 @@ def _parse(content: str) -> SlurmConfig: data = {} lines = content.splitlines() for index, line in enumerate(lines): - config, ignore = clean(line) - if ignore: + config = clean(line) + if config is None: _logger.debug("ignoring line %s at index %s in slurm.conf", line, index) continue diff --git a/slurmutils/editors/slurmdbdconfig.py b/slurmutils/editors/slurmdbdconfig.py index 9ee7208..dbed9cf 100644 --- a/slurmutils/editors/slurmdbdconfig.py +++ b/slurmutils/editors/slurmdbdconfig.py @@ -85,8 +85,8 @@ def _parse(content: str) -> SlurmdbdConfig: data = {} lines = content.splitlines() for index, line in enumerate(lines): - config, ignore = clean(line) - if ignore: + config = clean(line) + if config is None: _logger.debug("ignoring line %s at index %s in slurmdbd.conf", line, index) continue From 703303ff30e6e3631c9c0320376670d33ddcf257 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Mon, 26 Aug 2024 16:00:24 -0400 Subject: [PATCH 17/19] refactor: use more clear naming for configuration option sets Signed-off-by: Jason C. Nucciarone --- slurmutils/editors/slurmconfig.py | 6 +-- slurmutils/editors/slurmdbdconfig.py | 6 +-- slurmutils/models/option.py | 30 +++++++-------- slurmutils/models/slurm.py | 56 ++++++++++++++-------------- slurmutils/models/slurmdbd.py | 6 +-- 5 files changed, 52 insertions(+), 52 deletions(-) diff --git a/slurmutils/editors/slurmconfig.py b/slurmutils/editors/slurmconfig.py index 64759db..072f72c 100644 --- a/slurmutils/editors/slurmconfig.py +++ b/slurmutils/editors/slurmconfig.py @@ -23,7 +23,7 @@ from typing import Union from ..models import DownNodes, FrontendNode, Node, NodeSet, Partition, SlurmConfig -from ..models.option import SlurmConfigOptions +from ..models.option import SlurmConfigOptionSet from .editor import ( clean, dumper, @@ -114,7 +114,7 @@ def _parse(content: str) -> SlurmConfig: partitions.update(Partition.from_str(config).dict()) data["Partitions"] = partitions else: - data.update(parse_line(SlurmConfigOptions, config)) + data.update(parse_line(SlurmConfigOptionSet, config)) return SlurmConfig.from_dict(data) @@ -141,7 +141,7 @@ def _marshall(config: SlurmConfig) -> str: if slurmctld_host: result.extend([f"SlurmctldHost={host}" for host in slurmctld_host]) - result.extend(marshall_content(SlurmConfigOptions, data)) + result.extend(marshall_content(SlurmConfigOptionSet, data)) if nodes: for k, v in nodes.items(): diff --git a/slurmutils/editors/slurmdbdconfig.py b/slurmutils/editors/slurmdbdconfig.py index dbed9cf..7aab0cb 100644 --- a/slurmutils/editors/slurmdbdconfig.py +++ b/slurmutils/editors/slurmdbdconfig.py @@ -24,7 +24,7 @@ from slurmutils.models import SlurmdbdConfig -from ..models.option import SlurmdbdConfigOptions +from ..models.option import SlurmdbdConfigOptionSet from .editor import ( clean, dumper, @@ -90,7 +90,7 @@ def _parse(content: str) -> SlurmdbdConfig: _logger.debug("ignoring line %s at index %s in slurmdbd.conf", line, index) continue - data.update(parse_line(SlurmdbdConfigOptions, config)) + data.update(parse_line(SlurmdbdConfigOptionSet, config)) return SlurmdbdConfig.from_dict(data) @@ -102,5 +102,5 @@ def _marshall(config: SlurmdbdConfig) -> str: config: `slurmdbd.conf` data model to marshall. """ result = [] - result.extend(marshall_content(SlurmdbdConfigOptions, config.dict())) + result.extend(marshall_content(SlurmdbdConfigOptionSet, config.dict())) return "\n".join(result) diff --git a/slurmutils/models/option.py b/slurmutils/models/option.py index a591f2b..a07ae5e 100644 --- a/slurmutils/models/option.py +++ b/slurmutils/models/option.py @@ -15,13 +15,13 @@ """Configuration options for Slurm data models.""" __all__ = [ - "SlurmdbdConfigOptions", - "SlurmConfigOptions", - "NodeOptions", - "DownNodeOptions", - "FrontendNodeOptions", - "NodeSetOptions", - "PartitionOptions", + "SlurmdbdConfigOptionSet", + "SlurmConfigOptionSet", + "NodeOptionSet", + "DownNodeOptionSet", + "FrontendNodeOptionSet", + "NodeSetOptionSet", + "PartitionOptionSet", ] from dataclasses import dataclass, fields @@ -37,7 +37,7 @@ @dataclass(frozen=True) -class _Option: +class _OptionSet: """Base for configuration option dataclasses.""" @classmethod @@ -48,7 +48,7 @@ def keys(cls) -> Iterable[str]: @dataclass(frozen=True) -class SlurmdbdConfigOptions(_Option): +class SlurmdbdConfigOptionSet(_OptionSet): """`slurmdbd.conf` configuration options.""" AllowNoDefAcct: Callback = Callback() @@ -106,7 +106,7 @@ class SlurmdbdConfigOptions(_Option): @dataclass(frozen=True) -class SlurmConfigOptions(_Option): +class SlurmConfigOptionSet(_OptionSet): """`slurm.conf` configuration options.""" AccountingStorageBackupHost: Callback = CommaSeparatorCallback @@ -339,7 +339,7 @@ class SlurmConfigOptions(_Option): @dataclass(frozen=True) -class NodeOptions(_Option): +class NodeOptionSet(_OptionSet): """`slurm.conf` node configuration options.""" NodeName: Callback = Callback() @@ -368,7 +368,7 @@ class NodeOptions(_Option): @dataclass(frozen=True) -class DownNodeOptions(_Option): +class DownNodeOptionSet(_OptionSet): """`slurm.conf` down node configuration options.""" DownNodes: Callback = CommaSeparatorCallback @@ -377,7 +377,7 @@ class DownNodeOptions(_Option): @dataclass(frozen=True) -class FrontendNodeOptions(_Option): +class FrontendNodeOptionSet(_OptionSet): """`slurm.conf` frontend node configuration options.""" FrontendName: Callback = Callback() @@ -392,7 +392,7 @@ class FrontendNodeOptions(_Option): @dataclass(frozen=True) -class NodeSetOptions(_Option): +class NodeSetOptionSet(_OptionSet): """`slurm.conf` node set configuration options.""" NodeSet: Callback = Callback() @@ -401,7 +401,7 @@ class NodeSetOptions(_Option): @dataclass(frozen=True) -class PartitionOptions(_Option): +class PartitionOptionSet(_OptionSet): """`slurm.conf` partition configuration options.""" PartitionName: Callback = Callback() diff --git a/slurmutils/models/slurm.py b/slurmutils/models/slurm.py index 606345d..d04421a 100644 --- a/slurmutils/models/slurm.py +++ b/slurmutils/models/slurm.py @@ -29,12 +29,12 @@ from ..editors.editor import marshall_content, parse_line from .model import BaseModel, LineInterface, format_key, generate_descriptors from .option import ( - DownNodeOptions, - FrontendNodeOptions, - NodeOptions, - NodeSetOptions, - PartitionOptions, - SlurmConfigOptions, + DownNodeOptionSet, + FrontendNodeOptionSet, + NodeOptionSet, + NodeSetOptionSet, + PartitionOptionSet, + SlurmConfigOptionSet, ) @@ -43,7 +43,7 @@ class Node(BaseModel, LineInterface): def __init__(self, *, NodeName: str, **kwargs) -> None: # noqa N803 self.__node_name = NodeName - super().__init__(NodeOptions, **kwargs) + super().__init__(NodeOptionSet, **kwargs) @property def node_name(self) -> str: @@ -64,7 +64,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "Node": @classmethod def from_str(cls, line: str) -> "Node": """Construct model from configuration line.""" - data = parse_line(NodeOptions, line) + data = parse_line(NodeOptionSet, line) return cls(**data) def dict(self) -> Dict[str, Any]: @@ -74,7 +74,7 @@ def dict(self) -> Dict[str, Any]: def __str__(self) -> str: """Return model as configuration line.""" line = [f"NodeName={self.__node_name}"] - line.extend(marshall_content(NodeOptions, self.data)) + line.extend(marshall_content(NodeOptionSet, self.data)) return " ".join(line) @@ -82,17 +82,17 @@ class DownNodes(BaseModel, LineInterface): """`DownNodes` data model.""" def __init__(self, **kwargs): - super().__init__(DownNodeOptions, **kwargs) + super().__init__(DownNodeOptionSet, **kwargs) @classmethod def from_str(cls, line: str) -> "DownNodes": """Construct model from configuration line.""" - data = parse_line(DownNodeOptions, line) + data = parse_line(DownNodeOptionSet, line) return cls(**data) def __str__(self) -> str: """Return model as configuration line.""" - return " ".join(marshall_content(DownNodeOptions, self.data)) + return " ".join(marshall_content(DownNodeOptionSet, self.data)) class FrontendNode(BaseModel, LineInterface): @@ -100,7 +100,7 @@ class FrontendNode(BaseModel, LineInterface): def __init__(self, *, FrontendName: str, **kwargs) -> None: # noqa N803 self.__frontend_name = FrontendName - super().__init__(FrontendNodeOptions, **kwargs) + super().__init__(FrontendNodeOptionSet, **kwargs) @property def frontend_name(self) -> str: @@ -121,7 +121,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "FrontendNode": @classmethod def from_str(cls, line: str) -> "FrontendNode": """Construct model from configuration line.""" - data = parse_line(FrontendNodeOptions, line) + data = parse_line(FrontendNodeOptionSet, line) return cls(**data) def dict(self) -> Dict[str, Any]: @@ -131,7 +131,7 @@ def dict(self) -> Dict[str, Any]: def __str__(self) -> str: """Return model as configuration line.""" line = [f"FrontendName={self.__frontend_name}"] - line.extend(marshall_content(FrontendNodeOptions, self.data)) + line.extend(marshall_content(FrontendNodeOptionSet, self.data)) return " ".join(line) @@ -140,7 +140,7 @@ class NodeSet(BaseModel, LineInterface): def __init__(self, *, NodeSet: str, **kwargs) -> None: # noqa N803 self.__node_set = NodeSet - super().__init__(NodeSetOptions, **kwargs) + super().__init__(NodeSetOptionSet, **kwargs) @property def node_set(self) -> str: @@ -161,7 +161,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "NodeSet": @classmethod def from_str(cls, line: str) -> "NodeSet": """Construct model from configuration line.""" - data = parse_line(NodeSetOptions, line) + data = parse_line(NodeSetOptionSet, line) return cls(**data) def dict(self) -> Dict[str, Any]: @@ -171,7 +171,7 @@ def dict(self) -> Dict[str, Any]: def __str__(self) -> str: """Return model as configuration line.""" line = [f"NodeSet={self.__node_set}"] - line.extend(marshall_content(NodeSetOptions, self.data)) + line.extend(marshall_content(NodeSetOptionSet, self.data)) return " ".join(line) @@ -180,7 +180,7 @@ class Partition(BaseModel, LineInterface): def __init__(self, *, PartitionName: str, **kwargs): # noqa N803 self.__partition_name = PartitionName - super().__init__(PartitionOptions, **kwargs) + super().__init__(PartitionOptionSet, **kwargs) @property def partition_name(self) -> str: @@ -201,7 +201,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "Partition": @classmethod def from_str(cls, line: str) -> "Partition": """Construct model from configuration line.""" - data = parse_line(PartitionOptions, line) + data = parse_line(PartitionOptionSet, line) return cls(**data) def dict(self) -> Dict[str, Any]: @@ -211,7 +211,7 @@ def dict(self) -> Dict[str, Any]: def __str__(self) -> str: """Return model as configuration line.""" line = [f"PartitionName={self.__partition_name}"] - line.extend(marshall_content(PartitionOptions, self.data)) + line.extend(marshall_content(PartitionOptionSet, self.data)) return " ".join(line) @@ -228,7 +228,7 @@ def __init__( Partitions: Dict[str, Any] = None, # noqa N803 **kwargs, ) -> None: - super().__init__(SlurmConfigOptions, **kwargs) + super().__init__(SlurmConfigOptionSet, **kwargs) self.data["Nodes"] = Nodes or {} self.data["DownNodes"] = DownNodes or [] self.data["FrontendNodes"] = FrontendNodes or {} @@ -311,15 +311,15 @@ def partitions(self): self.data["Partitions"] = {} -for opt in NodeOptions.keys(): +for opt in NodeOptionSet.keys(): setattr(Node, format_key(opt), property(*generate_descriptors(opt))) -for opt in DownNodeOptions.keys(): +for opt in DownNodeOptionSet.keys(): setattr(DownNodes, format_key(opt), property(*generate_descriptors(opt))) -for opt in FrontendNodeOptions.keys(): +for opt in FrontendNodeOptionSet.keys(): setattr(FrontendNode, format_key(opt), property(*generate_descriptors(opt))) -for opt in NodeSetOptions.keys(): +for opt in NodeSetOptionSet.keys(): setattr(NodeSet, format_key(opt), property(*generate_descriptors(opt))) -for opt in PartitionOptions.keys(): +for opt in PartitionOptionSet.keys(): setattr(Partition, format_key(opt), property(*generate_descriptors(opt))) -for opt in SlurmConfigOptions.keys(): +for opt in SlurmConfigOptionSet.keys(): setattr(SlurmConfig, format_key(opt), property(*generate_descriptors(opt))) diff --git a/slurmutils/models/slurmdbd.py b/slurmutils/models/slurmdbd.py index 92ed064..c8cb360 100644 --- a/slurmutils/models/slurmdbd.py +++ b/slurmutils/models/slurmdbd.py @@ -15,15 +15,15 @@ """Data models for `slurmdbd.conf` configuration file.""" from .model import BaseModel, format_key, generate_descriptors -from .option import SlurmdbdConfigOptions +from .option import SlurmdbdConfigOptionSet class SlurmdbdConfig(BaseModel): """`slurmdbd.conf` data model.""" def __init__(self, **kwargs): - super().__init__(SlurmdbdConfigOptions, **kwargs) + super().__init__(SlurmdbdConfigOptionSet, **kwargs) -for opt in SlurmdbdConfigOptions.keys(): +for opt in SlurmdbdConfigOptionSet.keys(): setattr(SlurmdbdConfig, format_key(opt), property(*generate_descriptors(opt))) From 67a1bdc9be8fb1b5ea42cba586883cdb6465a183 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Mon, 26 Aug 2024 16:13:57 -0400 Subject: [PATCH 18/19] refactor: add type annotation to `generate_descriptors(...)` func Signed-off-by: Jason C. Nucciarone --- slurmutils/models/model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/slurmutils/models/model.py b/slurmutils/models/model.py index 3a3c591..709075d 100644 --- a/slurmutils/models/model.py +++ b/slurmutils/models/model.py @@ -20,7 +20,7 @@ import json import re from abc import ABC, abstractmethod -from typing import Any, Dict +from typing import Any, Callable, Dict, Tuple from ..exceptions import ModelError @@ -49,7 +49,7 @@ def format_key(key: str) -> str: return _camelize.sub(r"_", key).lower() -def generate_descriptors(opt: str): +def generate_descriptors(opt: str) -> Tuple[Callable, Callable, Callable]: """Generate descriptors for retrieving and mutating configuration options. Args: From 7a09a69db712a9675e20cce98afed4314a7fa1a1 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Mon, 26 Aug 2024 16:14:39 -0400 Subject: [PATCH 19/19] fix: use correct type annotations for SlurmConfig constructor Signed-off-by: Jason C. Nucciarone --- slurmutils/models/slurm.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/slurmutils/models/slurm.py b/slurmutils/models/slurm.py index d04421a..62fe312 100644 --- a/slurmutils/models/slurm.py +++ b/slurmutils/models/slurm.py @@ -24,7 +24,7 @@ ] import copy -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from ..editors.editor import marshall_content, parse_line from .model import BaseModel, LineInterface, format_key, generate_descriptors @@ -221,11 +221,11 @@ class SlurmConfig(BaseModel): def __init__( self, *, - Nodes: Dict[str, Any] = None, # noqa N803 - DownNodes: List[Dict[str, Any]] = None, # noqa N803 - FrontendNodes: Dict[str, Any] = None, # noqa N803 - NodeSets: Dict[str, Any] = None, # noqa N803 - Partitions: Dict[str, Any] = None, # noqa N803 + Nodes: Optional[Dict[str, Any]] = None, # noqa N803 + DownNodes: Optional[List[Dict[str, Any]]] = None, # noqa N803 + FrontendNodes: Optional[Dict[str, Any]] = None, # noqa N803 + NodeSets: Optional[Dict[str, Any]] = None, # noqa N803 + Partitions: Optional[Dict[str, Any]] = None, # noqa N803 **kwargs, ) -> None: super().__init__(SlurmConfigOptionSet, **kwargs)