From 8917314c4463478a6209c0ac457dd07d0ab9729e Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Tue, 23 Jan 2024 10:32:48 -0500 Subject: [PATCH 01/19] feat!: Drop old slurmconf API BREAKING CHANGE: Drops the old API for parsing and marshalling the slurm.conf file. This will allow common parsing logic to be shared among further editors, and will enable better code suggestions to programmers working with Slurm configuration files. Signed-off-by: Jason C. Nucciarone --- slurmutils/__init__.py | 15 ++ slurmutils/slurmconf/__init__.py | 17 -- slurmutils/slurmconf/api.py | 357 --------------------------- slurmutils/slurmconf/callback.py | 176 -------------- slurmutils/slurmconf/token.py | 405 ------------------------------- 5 files changed, 15 insertions(+), 955 deletions(-) create mode 100644 slurmutils/__init__.py delete mode 100644 slurmutils/slurmconf/__init__.py delete mode 100644 slurmutils/slurmconf/api.py delete mode 100644 slurmutils/slurmconf/callback.py delete mode 100644 slurmutils/slurmconf/token.py diff --git a/slurmutils/__init__.py b/slurmutils/__init__.py new file mode 100644 index 0000000..15ebf75 --- /dev/null +++ b/slurmutils/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2024 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +"""Utilities and APIs for interfacing with the Slurm workload manager.""" diff --git a/slurmutils/slurmconf/__init__.py b/slurmutils/slurmconf/__init__.py deleted file mode 100644 index 1f80dbc..0000000 --- a/slurmutils/slurmconf/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2023 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""API for performing CRUD operations on SLURM configuration.""" - -from .api import * diff --git a/slurmutils/slurmconf/api.py b/slurmutils/slurmconf/api.py deleted file mode 100644 index 687cf39..0000000 --- a/slurmutils/slurmconf/api.py +++ /dev/null @@ -1,357 +0,0 @@ -# Copyright 2023 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Parse and render SLURM configuration data.""" - -__all__ = ["SlurmConf", "Node", "DownNode", "FrontendNode", "NodeSet", "Partition"] - -import logging -import os -import pathlib -import re -import shlex -from dataclasses import make_dataclass -from itertools import chain -from typing import Dict, List, Optional, Union - -from .token import ( - DownNodeConfOpts, - FrontendNodeConfOpts, - NodeConfOpts, - NodeSetConfOpts, - PartitionConfOpts, - SlurmConfOpts, -) - -SLURM_CONF_FILE = "/etc/slurm/slurm.conf" -_pre_prop_name = re.compile(r"(.)([A-Z][a-z]+)") -_prop_name = re.compile(r"([a-z0-9])([A-Z])") -_logger = logging.getLogger(__name__) - - -def _snakecase(opt): - """Convert SLURM's loose PascalCase to snake_case. - - Args: - opt: Configuration option in loose PascalCase to convert to snake_case. - """ - pre_prop_name = _pre_prop_name.sub(r"\1_\2", opt) - return _prop_name.sub(r"\1_\2", pre_prop_name).lower() - - -def _gen_descriptors(opt): - """Generate descriptors for accessing SLURM configuration options. - - Args: - opt: Option to generate descriptors for. - """ - - def new_getter(self): - return self._data.get(opt, None) - - def new_setter(self, value: str): - self._data[opt] = value - - def new_deleter(self): - del self._data[opt] - - return new_getter, new_setter, new_deleter - - -# Methods to attach to SLURM data structs. -def _init(self, **kwargs): - """__init__ method for SLURM data structs.""" - self._data = kwargs - - -def _repr(self): - """__repr method for SLURM data structs.""" - return f"{self.__class__.__name__}({', '.join(f'{k}={v}' for k, v in self._data.items())})" - - -def _gen_from_line(conf_opts): - """Generate from_line parser for SLURM data structs.""" - - def from_line(cls, line: str): - data = {} - for token in shlex.split(line): # Use shlex.split(...) to preserve quotation blocks. - opt, value = token.split("=", 1) - if hasattr(conf_opts, opt): - if parse_callback := getattr(conf_opts, opt).parse: - value = parse_callback(value) - data.update({opt: value}) - else: - _logger.warning(f"Unrecognized configuration option: {token}") - - return cls(**data) - - return classmethod(from_line) - - -def _gen_to_line(conf_opts): - """Generate to_line renderer for SLURM data structs.""" - - def to_line(self): - tokens = [] - for opt, value in self._data.items(): - if hasattr(conf_opts, opt): - if render_callback := getattr(conf_opts, opt).render: - value = render_callback(value) - tokens.append(f"{opt}={value}") - else: - _logger.warning(f"Unrecognized configuration option: {opt}") - - return " ".join(tokens) - - return to_line - - -# Generate descriptors for accessing SLURM configuration options -_node_desc = {_snakecase(f): property(*_gen_descriptors(f)) for f in NodeConfOpts._fields} -_dnode_desc = {_snakecase(f): property(*_gen_descriptors(f)) for f in DownNodeConfOpts._fields} -_fnode_desc = {_snakecase(f): property(*_gen_descriptors(f)) for f in FrontendNodeConfOpts._fields} -_nodeset_desc = {_snakecase(f): property(*_gen_descriptors(f)) for f in NodeSetConfOpts._fields} -_part_desc = {_snakecase(f): property(*_gen_descriptors(f)) for f in PartitionConfOpts._fields} - -# Generate SLURM configuration data structs. -Comment = make_dataclass("Comment", ["content", "index", "inline"]) -Node = type( - "Node", - (object,), - { - "__init__": _init, - "__repr__": _repr, - "from_line": _gen_from_line(NodeConfOpts), - "to_line": _gen_to_line(NodeConfOpts), - **_node_desc, - }, -) -DownNode = type( - "DownNode", - (object,), - { - "__init__": _init, - "__repr__": _repr, - "from_line": _gen_from_line(DownNodeConfOpts), - "to_line": _gen_to_line(DownNodeConfOpts), - **_dnode_desc, - }, -) -FrontendNode = type( - "FrontendNode", - (object,), - { - "__init__": _init, - "__repr__": _repr, - "from_line": _gen_from_line(FrontendNodeConfOpts), - "to_line": _gen_to_line(FrontendNodeConfOpts), - **_fnode_desc, - }, -) -NodeSet = type( - "NodeSet", - (object,), - { - "__init__": _init, - "__repr__": _repr, - "from_line": _gen_from_line(NodeSetConfOpts), - "to_line": _gen_to_line(NodeSetConfOpts), - **_nodeset_desc, - }, -) -Partition = type( - "Partition", - (object,), - { - "__init__": _init, - "__repr__": _repr, - "from_line": _gen_from_line(PartitionConfOpts), - "to_line": _gen_to_line(PartitionConfOpts), - **_part_desc, - }, -) - - -def _parse(conf): - """Parse SLURM configuration data. - - Args: - conf: SLURM configuration data in SLURM format. - """ - conf_opts = { - "nodes": {}, - "down_nodes": [], - "frontend_nodes": {}, - "nodesets": {}, - "partitions": {}, - "comments": [], - } - - for index, line in enumerate(conf.splitlines()): - if "#" in line: - if line.startswith("#"): - conf_opts["comments"].append(Comment(line, index, inline=False)) - continue - else: - pos = line.index("#") - conf_opts["comments"].append(Comment(line[pos:], index, inline=True)) - line = line[:pos].strip() - - opt, value = line.split("=", 1) - if opt == "NodeName": - node = Node.from_line(line) - conf_opts["nodes"][node.node_name] = node - elif opt == "DownNodes": - conf_opts["down_nodes"].append(DownNode.from_line(line)) - elif opt == "FrontendName": - frontend = FrontendNode.from_line(line) - conf_opts["frontend_nodes"][frontend.frontend_name] = frontend - elif opt == "NodeSet": - nodeset = NodeSet.from_line(line) - conf_opts["nodesets"][nodeset.nodeset] = nodeset - elif opt == "PartitionName": - partition = Partition.from_line(line) - conf_opts["partitions"][partition.partition_name] = partition - elif opt == "SlurmctldHost": - if "SlurmctldHost" not in conf_opts.keys(): - conf_opts["SlurmctldHost"] = [value] - else: - conf_opts["SlurmctldHost"].append(value) - elif hasattr(SlurmConfOpts, opt): - if parse_callback := getattr(SlurmConfOpts, opt).parse: - value = parse_callback(value) - conf_opts.update({opt: value}) - else: - _logger.warning(f"Unable to parse line: {line}. Invalid configuration") - - return conf_opts - - -def _render(conf): - """Render SLURM configuration data into SLURM format. - - Args: - conf: SLURM configuration data in parsed format. - """ - conf_render = [] - nodes = conf.pop("nodes") - down_nodes = conf.pop("down_nodes") - frontend_nodes = conf.pop("frontend_nodes") - nodesets = conf.pop("nodesets") - partitions = conf.pop("partitions") - comments = conf.pop("comments") - - for opt, value in conf.items(): - if opt == "SlurmctldHost": - conf_render.extend([f"SlurmctldHost={host}" for host in value]) - elif hasattr(SlurmConfOpts, opt): - if render_callback := getattr(SlurmConfOpts, opt).render: - value = render_callback(value) - conf_render.append(f"{opt}={value}") - else: - _logger.warning(f"Unrecognized configuration option: {opt}") - - for struct in chain( - nodes.values(), down_nodes, frontend_nodes.values(), nodesets.values(), partitions.values() - ): - conf_render.append(struct.to_line()) - - for comment in comments: - if comment.inline: - conf_render[comment.index] = conf_render[comment.index] + f" {comment.content}" - else: - conf_render.insert(comment.index, comment.content) - - return "\n".join(conf_render) + "\n" - - -class SlurmConf: - """API interface to the slurm.conf file.""" - - def __init__(self, conf_file: Union[str, os.PathLike] = SLURM_CONF_FILE) -> None: - self._data = {} - self._conf_file = conf_file - - def __enter__(self) -> "SlurmConf": - """Load metadata file when entering context.""" - self.load() - return self - - def __exit__(self, exc_type, exc_val, exc_tb) -> None: - """Render and dump metadata file and configuration file when leaving context.""" - self.dump() - - @property - def comments(self) -> Optional[List[Comment]]: - """Get comments in SLURM configuration file.""" - return self._data.get("comments") - - @property - def nodes(self) -> Optional[Dict[str, Node]]: - """Get nodes in SLURM configuration file.""" - return self._data.get("nodes") - - @property - def down_nodes(self) -> Optional[List[DownNode]]: - """Get down nodes in SLURM configuration file.""" - return self._data.get("down_nodes") - - @property - def frontend_nodes(self) -> Optional[Dict[str, FrontendNode]]: - """Get frontend nodes in SLURM configuration file.""" - return self._data.get("frontend_nodes") - - @property - def nodesets(self) -> Optional[Dict[str, NodeSet]]: - """Get nodesets in SLURM configuration file.""" - return self._data.get("nodesets") - - @property - def partitions(self) -> Optional[Dict[str, Partition]]: - """Get partitions in SLURM configuration file.""" - return self._data.get("partitions") - - def load(self) -> None: - """Load slurm.conf configuration file. - - Notes: - This method will create a blank configuration if the slurm.conf - configuration file passed during initialisation does not exist. - """ - if (conf := pathlib.Path(self._conf_file)).exists(): - self._data = _parse(conf.read_text(encoding="ascii")) - else: - _logger.debug(f"{self._conf_file} not found. Creating blank configuration") - - def dump(self, conf_file: Optional[Union[str, os.PathLike]] = None) -> None: - """Render and dump slurm.conf configuration file. - - Args: - conf_file: Location to dump SLURM configuration information. - - Notes: - This method will overwrite any existing slurm.conf file if a - pre-existing file located in the same location as `conf_file`. - """ - conf_file = conf_file if conf_file else self._conf_file - if (conf := pathlib.Path(conf_file)).exists(): - _logger.debug(f"Overwriting pre-existing {conf_file}") - - conf.write_text(_render(self._data.copy()), encoding="ascii") - - -# Generate SLURM configuration API. -for field in SlurmConfOpts._fields: - # Attach descriptors for modifying configuration values. - setattr(SlurmConf, _snakecase(field), property(*_gen_descriptors(field))) diff --git a/slurmutils/slurmconf/callback.py b/slurmutils/slurmconf/callback.py deleted file mode 100644 index 4db56a6..0000000 --- a/slurmutils/slurmconf/callback.py +++ /dev/null @@ -1,176 +0,0 @@ -# Copyright 2023 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Callbacks for processing SLURM configuration data.""" - -from functools import singledispatch -from typing import Callable, NamedTuple, Optional - - -class Callback(NamedTuple): - """Data struct for holding conf option parser and render methods.""" - - parse: Optional[Callable] = None - render: Optional[Callable] = None - - -# Common manipulators for SLURM configuration data. -def _from_slurm_dict(value): - """Convert configuration value from a SLURM dict to Python dict. - - Args: - value: SLURM configuration value to convert to Python dict. - """ - result = {} - for val in value.split(","): - if "=" in val: - sub_opt, sub_val = val.split("=", 1) - result.update({sub_opt: sub_val}) - else: - result.update({val: True}) - - return result - - -@singledispatch -def _to_slurm_dict(value): - """Convert configuration value to SLURM dict. - - Notes: - Value my either be a Python dict or already in SLURM format, - so a dispatch is used to handle both cases. - """ - raise TypeError(f"Expected str or dict, not {type(value)}") - - -@singledispatch -def _to_slurm_comma_sep(value): - """Convert configuration value to SLURM comma-separated list. - - Notes: - Value my either be a Python list or already in SLURM format, - so a dispatch is used to handle both cases. - """ - raise TypeError(f"Expected str or list, not {type(value)}") - - -@singledispatch -def _to_slurm_colon_sep(value): - """Convert configuration value to SLURM colon-separated list. - - Notes: - Value my either be a Python list or already in SLURM format, - so a dispatch is used to handle both cases. - """ - raise TypeError(f"Expected str or list, not {type(value)}") - - -@_to_slurm_comma_sep.register -@_to_slurm_colon_sep.register -@_to_slurm_dict.register -def _(value: str): - return value - - -@_to_slurm_comma_sep.register -def _(value: list): - return ",".join(value) - - -@_to_slurm_colon_sep.register -def _(value: list): - return ":".join(value) - - -@_to_slurm_dict.register -def _(value: dict): - result = [] - for sub_opt, sub_val in value.items(): - if type(sub_val) != bool: - result.append(f"{sub_opt}={sub_val}") - elif sub_val: - result.append(sub_opt) - - return ",".join(result) - - -# Handler macros. -# SLURM configuration values. -acct_storage_external_host = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -acct_storage_param = Callback(_from_slurm_dict, _to_slurm_dict) -acct_storage_tres = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -acct_store_flags = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -auth_alt_types = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -auth_alt_param = Callback(_from_slurm_dict, _to_slurm_dict) -auth_info = Callback(_from_slurm_dict, _to_slurm_dict) -bcast_exclude = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -bcast_param = Callback(_from_slurm_dict, _to_slurm_dict) -cli_filter_plugins = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -communication_params = Callback(_from_slurm_dict, _to_slurm_dict) -cpu_freq_def = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -cpu_freq_governors = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -debug_flags = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -dependency_param = Callback(_from_slurm_dict, _to_slurm_dict) -federation_param = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -health_check_node_state = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -job_acct_gather_frequency = Callback(_from_slurm_dict, _to_slurm_dict) -job_comp_params = Callback(_from_slurm_dict, _to_slurm_dict) -job_submit_plugins = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -launch_parameters = Callback(_from_slurm_dict, _to_slurm_dict) -licenses = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -plugin_dir = Callback(lambda val: val.split(":"), _to_slurm_colon_sep) -power_parameters = Callback(_from_slurm_dict, _to_slurm_dict) -preempt_mode = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -preempt_param = Callback(_from_slurm_dict, _to_slurm_dict) -prep_plugins = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -priority_weight_tres = Callback(_from_slurm_dict, _to_slurm_dict) -private_data = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -prolog_flags = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -propagate_resource_limits = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -propagate_resource_limits_except = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -scheduler_param = Callback(_from_slurm_dict, _to_slurm_dict) -scron_param = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -slurmctld_param = Callback(_from_slurm_dict, _to_slurm_dict) -slurmd_param = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -switch_param = Callback(_from_slurm_dict, _to_slurm_dict) -task_plugin = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -task_plugin_param = Callback(_from_slurm_dict, _to_slurm_dict) -topology_param = Callback(lambda val: val.split(","), _to_slurm_comma_sep) - -# Node configuration values. -node_cpu_spec_list = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -node_features = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -node_gres = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -node_reason = Callback(None, lambda val: f'"{val}"') - -# DownNode configuration values. -down_name = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -down_reason = Callback(None, lambda val: f'"{val}"') - -# FrontendNode configuration values. -frontend_allow_groups = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -frontend_allow_users = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -frontend_deny_groups = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -frontend_deny_users = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -frontend_reason = Callback(None, lambda val: f'"{val}"') - -# Partition configuration values. -partition_alloc_nodes = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -partition_allow_accounts = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -partition_allow_groups = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -partition_allow_qos = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -partition_deny_accounts = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -partition_deny_qos = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -partition_nodes = Callback(lambda val: val.split(","), _to_slurm_comma_sep) -partition_tres_billing_weights = Callback(_from_slurm_dict, _to_slurm_dict) diff --git a/slurmutils/slurmconf/token.py b/slurmutils/slurmconf/token.py deleted file mode 100644 index e249d27..0000000 --- a/slurmutils/slurmconf/token.py +++ /dev/null @@ -1,405 +0,0 @@ -# Copyright 2023 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""SLURM configuration option tokens.""" - -from typing import NamedTuple - -from .callback import ( - Callback, - acct_storage_external_host, - acct_storage_param, - acct_storage_tres, - acct_store_flags, - auth_alt_param, - auth_alt_types, - auth_info, - bcast_exclude, - bcast_param, - cli_filter_plugins, - communication_params, - cpu_freq_def, - cpu_freq_governors, - debug_flags, - dependency_param, - down_name, - down_reason, - federation_param, - frontend_allow_groups, - frontend_allow_users, - frontend_deny_groups, - frontend_deny_users, - frontend_reason, - health_check_node_state, - job_acct_gather_frequency, - job_comp_params, - job_submit_plugins, - launch_parameters, - licenses, - node_cpu_spec_list, - node_features, - node_gres, - node_reason, - partition_alloc_nodes, - partition_allow_accounts, - partition_allow_groups, - partition_allow_qos, - partition_deny_accounts, - partition_deny_qos, - partition_nodes, - partition_tres_billing_weights, - plugin_dir, - power_parameters, - preempt_mode, - preempt_param, - prep_plugins, - priority_weight_tres, - private_data, - prolog_flags, - propagate_resource_limits, - propagate_resource_limits_except, - scheduler_param, - scron_param, - slurmctld_param, - slurmd_param, - switch_param, - task_plugin, - task_plugin_param, - topology_param, -) - - -class _SlurmConfOpts(NamedTuple): - """Top-level SLURM configuration options.""" - - Include: Callback = Callback() - AccountingStorageBackupHost: Callback = Callback() - AccountingStorageEnforce: Callback = Callback() - AccountStorageExternalHost: Callback = acct_storage_external_host - AccountingStorageHost: Callback = Callback() - AccountingStorageParameters: Callback = acct_storage_param - AccountingStoragePass: Callback = Callback() - AccountingStoragePort: Callback = Callback() - AccountingStorageTRES: Callback = acct_storage_tres - AccountingStorageType: Callback = Callback() - AccountingStorageUser: Callback = Callback() - AccountingStoreFlags: Callback = acct_store_flags - AcctGatherNodeFreq: Callback = Callback() - AcctGatherEnergyType: Callback = Callback() - AcctGatherInterconnectType: Callback = Callback() - AcctGatherFilesystemType: Callback = Callback() - AcctGatherProfileType: Callback = Callback() - AllowSpecResourcesUsage: Callback = Callback() - AuthAltTypes: Callback = auth_alt_types - AuthAltParameters: Callback = auth_alt_param - AuthInfo: Callback = auth_info - AuthType: Callback = Callback() - BatchStartTimeout: Callback = Callback() - BcastExclude: Callback = bcast_exclude - BcastParameters: Callback = bcast_param - BurstBufferType: Callback = Callback() - CliFilterPlugins: Callback = cli_filter_plugins - ClusterName: Callback = Callback() - CommunicationParameters: Callback = communication_params - CompleteWait: Callback = Callback() - CoreSpecPlugin: Callback = Callback() - CpuFreqDef: Callback = cpu_freq_def - CpuFreqGovernors: Callback = cpu_freq_governors - CredType: Callback = Callback() - DebugFlags: Callback = debug_flags - DefCpuPerGPU: Callback = Callback() - DefMemPerCPU: Callback = Callback() - DefMemPerGPU: Callback = Callback() - DefMemPerNode: Callback = Callback() - DependencyParameters: Callback = dependency_param - DisableRootJobs: Callback = Callback() - EioTimeout: Callback = Callback() - EnforcePartLimits: Callback = Callback() - Epilog: Callback = Callback() - EpilogMsgTime: Callback = Callback() - EpilogSlurmctld: Callback = Callback() - ExtSensorsFreq: Callback = Callback() - ExtSensorsType: Callback = Callback() - FairShareDampeningFactor: Callback = Callback() - FederationParameters: Callback = federation_param - FirstJobId: Callback = Callback() - GetEnvTimeout: Callback = Callback() - GresTypes: Callback = Callback() - GroupUpdateForce: Callback = Callback() - GroupUpdateTime: Callback = Callback() - GpuFreqDef: Callback = Callback() - HealthCheckInterval: Callback = Callback() - HealthCheckNodeState: Callback = health_check_node_state - HealthCheckProgram: Callback = Callback() - InactiveLimit: Callback = Callback() - InteractiveStepOptions: Callback = Callback() - JobAcctGatherType: Callback = Callback() - JobAcctGatherFrequency: Callback = job_acct_gather_frequency - JobAcctGatherParams: Callback = Callback() - JobCompHost: Callback = Callback() - JobCompLoc: Callback = Callback() - JobCompParams: Callback = job_comp_params - JobCompPass: Callback = Callback() - JobCompPort: Callback = Callback() - JobCompType: Callback = Callback() - JobCompUser: Callback = Callback() - JobContainerType: Callback = Callback() - JobFileAppend: Callback = Callback() - JobRequeue: Callback = Callback() - JobSubmitPlugins: Callback = job_submit_plugins - KillOnBadExit: Callback = Callback() - KillWait: Callback = Callback() - MaxBatchRequeue: Callback = Callback() - NodeFeaturesPlugins: Callback = Callback() - LaunchParameters: Callback = launch_parameters - Licenses: Callback = licenses - LogTimeFormat: Callback = Callback() - MailDomain: Callback = Callback() - MailProg: Callback = Callback() - MaxArraySize: Callback = Callback() - MaxJobCount: Callback = Callback() - MaxJobId: Callback = Callback() - MaxMemPerCPU: Callback = Callback() - MaxMemPerNode: Callback = Callback() - MaxNodeCount: Callback = Callback() - MaxStepCount: Callback = Callback() - MaxTasksPerNode: Callback = Callback() - MCSParameters: Callback = Callback() - MCSPlugin: Callback = Callback() - MessageTimeout: Callback = Callback() - MinJobAge: Callback = Callback() - MpiDefault: Callback = Callback() - MpiParams: Callback = Callback() - OverTimeLimit: Callback = Callback() - PluginDir: Callback = plugin_dir - PlugStackConfig: Callback = Callback() - PowerParameters: Callback = power_parameters - PowerPlugin: Callback = Callback() - PreemptMode: Callback = preempt_mode - PreemptParameters: Callback = preempt_param - PreemptType: Callback = Callback() - PreemptExemptTime: Callback = Callback() - PrEpParameters: Callback = Callback() - PrEpPlugins: Callback = prep_plugins - PriorityCalcpPeriod: Callback = Callback() - PriorityDecayHalfLife: Callback = Callback() - PriorityFavorSmall: Callback = Callback() - PriorityFlags: Callback = Callback() - PriorityMaxAge: Callback = Callback() - PriorityParameters: Callback = Callback() - PrioritySiteFactorParameters: Callback = Callback() - PrioritySiteFactorPlugin: Callback = Callback() - PriorityType: Callback = Callback() - PriorityUsageResetPeriod: Callback = Callback() - PriorityWeightAge: Callback = Callback() - PriorityWeightAssoc: Callback = Callback() - PriorityWeightFairShare: Callback = Callback() - PriorityWeightJobSize: Callback = Callback() - PriorityWeightPartition: Callback = Callback() - PriorityWeightQOS: Callback = Callback() - PriorityWeightTRES: Callback = priority_weight_tres - PrivateData: Callback = private_data - ProctrackType: Callback = Callback() - Prolog: Callback = Callback() - PrologEpilogTimeout: Callback = Callback() - PrologFlags: Callback = prolog_flags - PrologSlurmctld: Callback = Callback() - PropagatePrioProcess: Callback = Callback() - PropagateResourceLimits: Callback = propagate_resource_limits - PropagateResourceLimitsExcept: Callback = propagate_resource_limits_except - RebootProgram: Callback = Callback() - ReconfigFlags: Callback = Callback() - RequeueExit: Callback = Callback() - RequeueExitHold: Callback = Callback() - ResumeFailProgram: Callback = Callback() - ResumeProgram: Callback = Callback() - ResumeRate: Callback = Callback() - ResumeTimeout: Callback = Callback() - ResvEpilog: Callback = Callback() - ResvOverRun: Callback = Callback() - ResvProlog: Callback = Callback() - ReturnToService: Callback = Callback() - RoutePlugin: Callback = Callback() - SchedulerParameters: Callback = scheduler_param - SchedulerTimeSlice: Callback = Callback() - SchedulerType: Callback = Callback() - ScronParameters: Callback = scron_param - SelectType: Callback = Callback() - SelectTypeParameters: Callback = Callback() - SlurmctldAddr: Callback = Callback() - SlurmctldDebug: Callback = Callback() - SlurmctldHost: Callback = Callback() - SlurmctldLogFile: Callback = Callback() - SlurmctldParameters: Callback = slurmctld_param - SlurmctldPidFile: Callback = Callback() - SlurmctldPort: Callback = Callback() - SlurmctldPrimaryOffProg: Callback = Callback() - SlurmctldPrimaryOnProg: Callback = Callback() - SlurmctldSyslogDebug: Callback = Callback() - SlurmctldTimeout: Callback = Callback() - SlurmdDebug: Callback = Callback() - SlurmdLogFile: Callback = Callback() - SlurmdParameters: Callback = slurmd_param - SlurmdPidFile: Callback = Callback() - SlurmdPort: Callback = Callback() - SlurmdSpoolDir: Callback = Callback() - SlurmdSyslogDebug: Callback = Callback() - SlurmdTimeout: Callback = Callback() - SlurmdUser: Callback = Callback() - SlurmSchedLogFile: Callback = Callback() - SlurmSchedLogLevel: Callback = Callback() - SlurmUser: Callback = Callback() - SrunEpilog: Callback = Callback() - SrunPortRange: Callback = Callback() - SrunProlog: Callback = Callback() - StateSaveLocation: Callback = Callback() - SuspendExcNodes: Callback = Callback() - SuspendExcParts: Callback = Callback() - SuspendExcStates: Callback = Callback() - SuspendProgram: Callback = Callback() - SuspendRate: Callback = Callback() - SuspendTime: Callback = Callback() - SuspendTimeout: Callback = Callback() - SwitchParameters: Callback = switch_param - SwitchType: Callback = Callback() - TaskEpilog: Callback = Callback() - TaskPlugin: Callback = task_plugin - TaskPluginParam: Callback = task_plugin_param - TaskProlog: Callback = Callback() - TCPTimeout: Callback = Callback() - TmpFS: Callback = Callback() - TopologyParam: Callback = topology_param - TopologyPlugin: Callback = Callback() - TrackWCKey: Callback = Callback() - TreeWidth: Callback = Callback() - UnkillableStepProgram: Callback = Callback() - UnkillableStepTimeout: Callback = Callback() - UsePAM: Callback = Callback() - VSizeFactor: Callback = Callback() - WaitTime: Callback = Callback() - X11Parameters: Callback = Callback() - - -class _NodeConfOpts(NamedTuple): - """SLURM node configuration options.""" - - NodeName: Callback = Callback() - NodeHostname: Callback = Callback() - NodeAddr: Callback = Callback() - BcastAddr: Callback = Callback() - Boards: Callback = Callback() - CoreSpecCount: Callback = Callback() - CoresPerSocket: Callback = Callback() - CpuBind: Callback = Callback() - CPUs: Callback = Callback() - CpuSpecList: Callback = node_cpu_spec_list - Features: Callback = node_features - Gres: Callback = node_gres - MemSpecLimit: Callback = Callback() - Port: Callback = Callback() - Procs: Callback = Callback() - RealMemory: Callback = Callback() - Reason: Callback = node_reason - Sockets: Callback = Callback() - SocketsPerBoard: Callback = Callback() - State: Callback = Callback() - ThreadsPerCore: Callback = Callback() - TmpDisk: Callback = Callback() - Weight: Callback = Callback() - - -class _DownNodeConfOpts(NamedTuple): - """SLURM down node configuration options.""" - - DownNodes: Callback = down_name - Reason: Callback = down_reason - State: Callback = Callback() - - -class _FrontendNodeConfOpts(NamedTuple): - """SLURM frontend node configuration options.""" - - FrontendName: Callback = Callback() - FrontendAddr: Callback = Callback() - AllowGroups: Callback = frontend_allow_groups - AllowUsers: Callback = frontend_allow_users - DenyGroups: Callback = frontend_deny_groups - DenyUsers: Callback = frontend_deny_users - Port: Callback = Callback() - Reason: Callback = frontend_reason - State: Callback = Callback() - - -class _NodeSetConfOpts(NamedTuple): - """SLURM nodeset configuration options.""" - - NodeSet: Callback = Callback() - Feature: Callback = Callback() - Nodes: Callback = Callback() - - -class _PartitionConfOpts(NamedTuple): - """SLURM partition configuration options.""" - - PartitionName: Callback = Callback() - AllocNodes: Callback = partition_alloc_nodes - AllowAccounts: Callback = partition_allow_accounts - AllowGroups: Callback = partition_allow_groups - AllowQos: Callback = partition_allow_qos - Alternate: Callback = Callback() - CpuBind: Callback = Callback() - Default: Callback = Callback() - DefaultTime: Callback = Callback() - DefCpuPerGPU: Callback = Callback() - DefMemPerCPU: Callback = Callback() - DefMemPerGPU: Callback = Callback() - DefMemPerNode: Callback = Callback() - DenyAccounts: Callback = partition_deny_accounts - DenyQos: Callback = partition_deny_qos - DisableRootJobs: Callback = Callback() - ExclusiveUser: Callback = Callback() - GraceTime: Callback = Callback() - Hidden: Callback = Callback() - LLN: Callback = Callback() - MaxCPUsPerNode: Callback = Callback() - MaxCPUsPerSocket: Callback = Callback() - MaxMemPerCPU: Callback = Callback() - MaxMemPerNode: Callback = Callback() - MaxNodes: Callback = Callback() - MaxTime: Callback = Callback() - MinNodes: Callback = Callback() - Nodes: Callback = partition_nodes - OverSubscribe: Callback = Callback() - OverTimeLimit: Callback = Callback() - PowerDownOnIdle: Callback = Callback() - PreemptMode: Callback = Callback() - PriorityJobFactor: Callback = Callback() - PriorityTier: Callback = Callback() - QOS: Callback = Callback() - ReqResv: Callback = Callback() - ResumeTimeout: Callback = Callback() - RootOnly: Callback = Callback() - SelectTypeParameters: Callback = Callback() - State: Callback = Callback() - SuspendTime: Callback = Callback() - SuspendTimeout: Callback = Callback() - TRESBillingWeights: Callback = partition_tres_billing_weights - - -SlurmConfOpts = _SlurmConfOpts() -NodeConfOpts = _NodeConfOpts() -DownNodeConfOpts = _DownNodeConfOpts() -FrontendNodeConfOpts = _FrontendNodeConfOpts() -NodeSetConfOpts = _NodeSetConfOpts() -PartitionConfOpts = _PartitionConfOpts() From a98ca29e57e2c6f8fd0330e96597382dd178d270 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Wed, 24 Jan 2024 15:43:24 -0500 Subject: [PATCH 02/19] chore(meta): Ignore lint rule D105 Ignore lint error where magic methods do not have a docstring. Magic method behavior is already explained by Python documentation, and the defined methods behave the same as the would if they were not overridden. Signed-off-by: Jason C. Nucciarone --- pyproject.toml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 90e3d1d..34424cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Poetry +[tool.poetry] +name = "slurmutils" +version = "0.1.0" +description = "Utilities and APIs for interfacing with the Slurm workload manager" +authors = ["Jason C. Nucciarone "] +license = "LGPL-3.0" +readme = "README.md" + +[tool.poetry.dependencies] +python = ">=3.8" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + # Testing tools configuration [tool.coverage.run] branch = true @@ -50,7 +66,7 @@ extend-ignore = [ "D409", "D413", ] -ignore = ["E501", "D107"] +ignore = ["E501", "D105", "D107"] extend-exclude = ["__pycache__", "*.egg_info", "__init__.py"] per-file-ignores = {"tests/*" = ["D100","D101","D102","D103","D104"]} From f632bed651e4a543e9ee45b00ffb8104757f29d3 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Wed, 24 Jan 2024 15:55:21 -0500 Subject: [PATCH 03/19] chore(meta): Add poetry.lock file Using poetry for packaging and dependency management since the tool integrates nicely with pyproject.toml and does not require carrying around a requirements.txt file. Signed-off-by: Jason C. Nucciarone --- poetry.lock | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 poetry.lock diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..eb7f21c --- /dev/null +++ b/poetry.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +package = [] + +[metadata] +lock-version = "2.0" +python-versions = ">=3.8" +content-hash = "dedbcc8ad01960ccbef8502c70bda77771c2826a438e1e94ef27a36c71acd91a" From f1546802a78082662401cb3d3fa007db8573eb1b Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Thu, 25 Jan 2024 13:51:04 -0500 Subject: [PATCH 04/19] chore(meta)!: Update license to LGPLv3 BREAKING CHANGE: Switches license away from Apache-2.0. This change is in line with Canonical's policy on how libraries should be licensed. Signed-off-by: Jason C. Nucciarone --- CONTRIBUTING.md | 4 +- LICENSE | 366 ++++++++++++++++------------------- README.md | 2 +- setup.py | 26 +-- tests/unit/test_slurmconf.py | 20 +- 5 files changed, 191 insertions(+), 227 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d7f7bd2..4e49d06 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -170,7 +170,7 @@ included in the project: with a clear title and description against the `main` branch. **IMPORTANT**: By submitting a patch, improvement, or new feature, you agree to allow the maintainers of slurmutils to -license your contributions under the terms of the [Apache 2.0 license](./LICENSE), and you agree to sign +license your contributions under the terms of the [GNU Lesser General Public License, v3.0](./LICENSE), and you agree to sign [Canonical's contributor license agreement](https://ubuntu.com/legal/contributors) ## Discussions @@ -196,5 +196,5 @@ The following guidelines must be adhered to if you are writing code to be merged ## License & CLA By contributing your code to slurmutils, you agree to license your contribution under the -[Apache 2.0 license](https://www.apache.org/licenses/LICENSE-2.0.html), and you agree to +[GNU Lesser General Public License, v3.0](./LICENSE), and you agree to sign [Canonical's contributor license agreement](https://ubuntu.com/legal/contributors). diff --git a/LICENSE b/LICENSE index 261eeb9..0a04128 100644 --- a/LICENSE +++ b/LICENSE @@ -1,201 +1,165 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. diff --git a/README.md b/README.md index d641c95..642fded 100644 --- a/README.md +++ b/README.md @@ -75,5 +75,5 @@ Check out these links below: ## License -The `slurmutils` package is free software, distributed under the Apache Software License, version 2.0. +The `slurmutils` package is free software, distributed under the GNU Lesser General Public License, v3.0. See the [LICENSE](./LICENSE) file for more information. diff --git a/setup.py b/setup.py index 2c5c0f8..83612e1 100644 --- a/setup.py +++ b/setup.py @@ -1,17 +1,17 @@ #!/usr/bin/env python3 -# Copyright 2023 Canonical Ltd. +# Copyright 2024 Canonical Ltd. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. # -# http://www.apache.org/licenses/LICENSE-2.0 +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . """setup.py for slurmutils package.""" @@ -24,7 +24,7 @@ version="0.1.0", author="Jason C. Nucciarone", author_email="jason.nucciarone@canonical.com", - license="Apache-2.0", + license="LGPL-3.0", url="https://github.com/canonical/slurmutils", description="Utilities and APIs for interacting with the SLURM workload manager", long_description=open("README.md").read() if exists("README.md") else "", @@ -34,7 +34,7 @@ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", "Intended Audience :: Science/Research", - "License :: OSI Approved :: Apache Software License", + "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)", "Operating System :: POSIX :: Linux", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", @@ -42,6 +42,6 @@ ], packages=[ "slurmutils", - "slurmutils.slurmconf" + "slurmutils.editors" ] ) diff --git a/tests/unit/test_slurmconf.py b/tests/unit/test_slurmconf.py index 885e71c..6bda475 100644 --- a/tests/unit/test_slurmconf.py +++ b/tests/unit/test_slurmconf.py @@ -1,17 +1,17 @@ #!/usr/bin/env python3 -# Copyright 2023 Canonical Ltd. +# Copyright 2024 Canonical Ltd. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. # -# http://www.apache.org/licenses/LICENSE-2.0 +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . """Test SLURM configuration API.""" From 95c82bdb78f0de3bc743fa084d75f8b632211359 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Thu, 25 Jan 2024 13:51:38 -0500 Subject: [PATCH 05/19] chore(meta): Fix spacing between sections Signed-off-by: Jason C. Nucciarone --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 34424cd..bbde4d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,6 @@ log_cli_level = "INFO" [tool.codespell] skip = "build,lib,venv,icon.svg,.tox,.git,.mypy_cache,.ruff_cache,.vscode,.coverage" - # Formatting tools configuration [tool.black] line-length = 99 From a1b48f255a5b9cb35107abd5e34656fafb7975ea Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Thu, 25 Jan 2024 13:53:18 -0500 Subject: [PATCH 06/19] feat!: Add data models BREAKING CHANGE: No longer use the token lib that was used to classify configuration knobs in slurm.conf. Data models are no longer dynamically generated at runtime. Signed-off-by: Jason C. Nucciarone --- slurmutils/models/__init__.py | 18 + slurmutils/models/_model.py | 255 ++++++++++++ slurmutils/models/slurm.py | 735 ++++++++++++++++++++++++++++++++++ slurmutils/models/slurmdbd.py | 76 ++++ 4 files changed, 1084 insertions(+) create mode 100644 slurmutils/models/__init__.py create mode 100644 slurmutils/models/_model.py create mode 100644 slurmutils/models/slurm.py create mode 100644 slurmutils/models/slurmdbd.py diff --git a/slurmutils/models/__init__.py b/slurmutils/models/__init__.py new file mode 100644 index 0000000..b5b9509 --- /dev/null +++ b/slurmutils/models/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2024 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +"""Data models for common Slurm objects.""" + +from .slurm import DownNodes, FrontendNode, Node, NodeSet, Partition, SlurmConfig +from .slurmdbd import SlurmdbdConfig diff --git a/slurmutils/models/_model.py b/slurmutils/models/_model.py new file mode 100644 index 0000000..dea92ac --- /dev/null +++ b/slurmutils/models/_model.py @@ -0,0 +1,255 @@ +# Copyright 2024 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +"""Macros for Slurm workload manager data models.""" + +import copy +import functools +import inspect +import json +from abc import ABC, abstractmethod +from types import MappingProxyType +from typing import Any, Callable, Dict, NamedTuple, Optional + + +# Simple type checking decorator; used to verify input into Slurm data models +# without needing every method to contain an `if isinstance(...)` block. +def assert_type(*typed_args, **typed_kwargs): + """Check the type of args and kwargs passed to a function/method.""" + + def decorator(func: Callable): + sig = inspect.signature(func) + bound_types = sig.bind_partial(*typed_args, **typed_kwargs).arguments + + @functools.wraps(func) + def wrapper(*args, **kwargs): + bound_values = sig.bind(*args, **kwargs).arguments + for name in bound_types.keys() & bound_values.keys(): + if not isinstance(bound_values[name], bound_types[name]): + raise TypeError(f"{bound_values[name]} is not {bound_types[name]}.") + + return func(*args, **kwargs) + + return wrapper + + return decorator + + +# Generate descriptors for Slurm configuration knobs. +# These descriptors are used for retrieving configuration values but +# also preserve the integrity of Slurm's loose pascal casing. +# The descriptors will use an internal _register dictionary to +# manage the parsed configuration knobs. +def base_descriptors(knob: str): + """Generate descriptors for accessing configuration knob values. + + Args: + knob: Configuration knob to generate descriptors for. + """ + + def getter(self): + return self._register.get(knob, None) + + def setter(self, value): + self._register[knob] = value + + def deleter(self): + try: + del self._register[knob] + except KeyError: + pass + + return getter, setter, deleter + + +# Nodes, FrontendNodes, DownNodes, NodeSets, and Partitions are represented +# as a Python dictionary with a primary key and nested dictionary when +# parsed in from the slurm.conf configuration file: +# +# {"node_1": {"NodeHostname": ..., "NodeAddr": ..., "CPUs", ...}} +# +# Since these models are parsed in this way, they need special descriptors +# for accessing the primary key (e.g. the NodeName), and sub values in the +# nested dictionary. +def primary_key_descriptors(): + """Generate descriptors for accessing a configuration knob key.""" + + def getter(self): + # There will only be a single key in _register, + # so it's okay to return the first index. If the + # primary key doesn't exist, return None. + try: + return list(self._register.keys())[0] + except IndexError: + return None + + def setter(self, value): + old_primary = list(self._register.keys())[0] + if old_primary: + self._register[value] = self._register.pop(old_primary, {}) + else: + self._register[value] = {} + + def deleter(self): + try: + primary_key = list(self._register.keys())[0] + del self._register[primary_key] + except IndexError: + pass + + return getter, setter, deleter + + +def nested_descriptors(knob: str, knob_key_alias: str): + """Generate descriptors for accessing a nested configuration knob. + + Args: + knob: Nested configuration knob to generate descriptors for. + knob_key_alias: Alias of knob key that needs to pbe defined in + register before accessing nested configuration knobs. + """ + + def getter(self): + try: + primary_key = list(self._register.keys())[0] + return self._register[primary_key].get(knob, None) + except IndexError: + raise KeyError(f"{knob_key_alias} must be defined before {knob} can be accessed.") + + def setter(self, value): + try: + primary_key = list(self._register.keys())[0] + self._register[primary_key][knob] = value + except IndexError: + raise KeyError(f"{knob_key_alias} must be defined before {knob} can be accessed.") + + def deleter(self): + try: + primary_key = list(self._register.keys())[0] + del self._register[primary_key][knob] + except IndexError: + raise KeyError(f"{knob_key_alias} must be defined before {knob} can be accessed.") + except KeyError: + pass + + return getter, setter, deleter + + +# Callbacks are used during parsing and marshalling for performing +# extra processing on specific configuration knob values. They contain callables +# that accept a single argument. Makes it easy to convert Python objects to Slurm +# configuration values and vice versa. +class Callback(NamedTuple): + """Object for invoking callables on Slurm configuration knobs during parsing/marshalling. + + Possible callables: + parse: Invoked when value is being parsed in from configuration file. + marshal: Invoked when value is being marshalled into configuration file. + """ + + parse: Optional[Callable[[Any], Any]] = None + marshal: Optional[Callable[[Any], Any]] = None + + +# Common parsing/marshalling callbacks for Slurm configuration values. +# Arrays are denoted using comma/colon separators. Maps are denoted as +# key1=value,key2=value,bool. Booleans are mapped by the inclusion of +# the keyword in maps. So key1=value,key2 would equate to: +# +# { +# "key1": "value", +# "key2": True, +# } +@functools.singledispatch +def _slurm_dict(v): + raise TypeError(f"Expected str or dict, not {type(v)}") + + +@_slurm_dict.register +def _(v: str): + """Convert Slurm dictionary to Python dictionary.""" + result = {} + for val in v.split(","): + if "=" in val: + sub_opt, sub_val = val.split("=", 1) + result.update({sub_opt: sub_val}) + else: + result.update({val: True}) + + return result + + +@_slurm_dict.register +def _(v: dict): + """Convert Python dictionary to Slurm dictionary.""" + result = [] + for sub_opt, sub_val in v.items(): + if not isinstance(sub_val, bool): + result.append(f"{sub_opt}={sub_val}") + elif sub_val: + result.append(sub_opt) + + return ",".join(result) + + +CommaSeparatorCallback = Callback(lambda v: v.split(","), lambda v: ",".join(v)) +ColonSeparatorCallback = Callback(lambda v: v.split(":"), lambda v: ":".join(v)) +SlurmDictCallback = Callback(_slurm_dict, _slurm_dict) +ReasonCallback = Callback(None, lambda v: f'"{v}"') + + +# All Slurm data models should inherit from this abstract parent class. +# The class provides method definitions for common operations and +# requires models to specify callbacks so that models can be treated +# generically when parsing and marshalling rather than having an infinite if-else tree. +class BaseModel(ABC): + """Abstract base class for Slurm-related data models.""" + + def __init__(self, **kwargs): + self._register = kwargs + + def __repr__(self): + return ( + f"{self.__class__.__name__}" + f"({', '.join(f'{k}={v}' for k, v in self._register.items())})" + ) + + @property + @abstractmethod + def _callbacks(self) -> MappingProxyType: + """Store callbacks. + + This map will be queried during parsing and marshalling to determine if + a configuration value needs any further processing. Each model class will + need to define the callbacks specific to its configuration knobs. Every model + class should declare whether it has callbacks or not. + + Callbacks should be MappingProxyType (read-only dict) to prevent any accidental + mutation of callbacks used during parsing and marshalling. + """ + pass + + def dict(self) -> Dict: + """Get model in dictionary form. + + Returns a deep copy of model's internal register. The deep copy is needed + because assigned variables all point to the same dictionary in memory. Without the + deep copy, operations performed on the returned dictionary could cause unintended + mutations in the internal register. + """ + return copy.deepcopy(self._register) + + def json(self) -> str: + """Get model as JSON object.""" + return json.dumps(self._register) diff --git a/slurmutils/models/slurm.py b/slurmutils/models/slurm.py new file mode 100644 index 0000000..66648c3 --- /dev/null +++ b/slurmutils/models/slurm.py @@ -0,0 +1,735 @@ +# Copyright 2024 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +"""Generic data models for the Slurm workload manager.""" + +import functools +from collections import UserList +from collections.abc import MutableMapping +from types import MappingProxyType +from typing import Any, Dict + +from ._model import ( + BaseModel, + ColonSeparatorCallback, + CommaSeparatorCallback, + ReasonCallback, + SlurmDictCallback, + assert_type, + base_descriptors, + nested_descriptors, + primary_key_descriptors, +) + +_node_descriptors = functools.partial(nested_descriptors, knob_key_alias="NodeName") +_frontend_descriptors = functools.partial(nested_descriptors, knob_key_alias="FrontendName") +_nodeset_descriptors = functools.partial(nested_descriptors, knob_key_alias="NodeSet") +_partition_descriptors = functools.partial(nested_descriptors, knob_key_alias="PartitionName") + + +class Node(BaseModel): + """Object representing Node(s) definition in slurm.conf. + + Node definition and data validators sourced from + the slurm.conf manpage. `man slurm.conf.5` + """ + + def __init__(self, **kwargs): + super().__init__() + self._register.update({kwargs.pop("NodeName"): {**kwargs}}) + + _callbacks = MappingProxyType( + { + "cpu_spec_list": CommaSeparatorCallback, + "features": CommaSeparatorCallback, + "gres": CommaSeparatorCallback, + "reason": ReasonCallback, + } + ) + + node_name = property(*primary_key_descriptors()) + node_hostname = property(*_node_descriptors("NodeHostname")) + node_addr = property(*_node_descriptors("NodeAddr")) + bcast_addr = property(*_node_descriptors("BcastAddr")) + boards = property(*_node_descriptors("Boards")) + core_spec_count = property(*_node_descriptors("CoreSpecCount")) + cores_per_socket = property(*_node_descriptors("CoresPerSocket")) + cpu_bind = property(*_node_descriptors("CpuBind")) + cpus = property(*_node_descriptors("CPUs")) + cpu_spec_list = property(*_node_descriptors("CpuSpecList")) + features = property(*_node_descriptors("Features")) + gres = property(*_node_descriptors("Gres")) + mem_spec_limit = property(*_node_descriptors("MemSpecLimit")) + port = property(*_node_descriptors("Port")) + procs = property(*_node_descriptors("Procs")) + real_memory = property(*_node_descriptors("RealMemory")) + reason = property(*_node_descriptors("Reason")) + sockets = property(*_node_descriptors("Sockets")) + sockets_per_board = property(*_node_descriptors("SocketsPerBoard")) + state = property(*_node_descriptors("State")) + threads_per_core = property(*_node_descriptors("ThreadsPerCore")) + tmp_disk = property(*_node_descriptors("TmpDisk")) + weight = property(*_node_descriptors("Weight")) + + +class DownNodes(BaseModel): + """Object representing DownNodes definition in slurm.conf. + + DownNodes definition and data validators sourced from + the slurm.conf manpage. `man slurm.conf.5` + """ + + _callbacks = MappingProxyType( + { + "down_nodes": CommaSeparatorCallback, + "reason": ReasonCallback, + } + ) + + down_nodes = property(*base_descriptors("DownNodes")) + reason = property(*base_descriptors("Reason")) + state = property(*base_descriptors("State")) + + +class FrontendNode(BaseModel): + """FrontendNode data model. + + FrontendNode definition and data validators sourced from + the slurm.conf manpage. `man slurm.conf.5` + """ + + def __init__(self, **kwargs): + super().__init__() + self._register.update({kwargs.pop("FrontendName"): {**kwargs}}) + + _callbacks = MappingProxyType( + { + "allow_groups": CommaSeparatorCallback, + "allow_users": CommaSeparatorCallback, + "deny_groups": CommaSeparatorCallback, + "deny_users": CommaSeparatorCallback, + "reason": ReasonCallback, + } + ) + + frontend_name = property(*primary_key_descriptors()) + frontend_addr = property(*_frontend_descriptors("FrontendAddr")) + allow_groups = property(*_frontend_descriptors("AllowGroups")) + allow_users = property(*_frontend_descriptors("AllowUsers")) + deny_groups = property(*_frontend_descriptors("DenyGroups")) + deny_users = property(*_frontend_descriptors("DenyUsers")) + port = property(*_frontend_descriptors("Port")) + reason = property(*_frontend_descriptors("Reason")) + state = property(*_frontend_descriptors("State")) + + +class NodeSet(BaseModel): + """Object representing NodeSet definition in slurm.conf. + + NodeSet definition and data validators sourced from + the slurm.conf manpage. `man slurm.conf.5` + """ + + def __init__(self, **kwargs): + super().__init__() + self._register.update({kwargs.pop("NodeSet"): {**kwargs}}) + + _callbacks = MappingProxyType({"nodes": CommaSeparatorCallback}) + + node_set = property(*primary_key_descriptors()) + feature = property(*_nodeset_descriptors("Feature")) + nodes = property(*_nodeset_descriptors("Nodes")) + + +class Partition(BaseModel): + """Object representing Partition definition in slurm.conf. + + Partition definition and data validators sourced from + the slurm.conf manpage. `man slurm.conf.5` + """ + + def __init__(self, **kwargs): + super().__init__() + self._register.update({kwargs.pop("PartitionName"): {**kwargs}}) + + _callbacks = MappingProxyType( + { + "alloc_nodes": CommaSeparatorCallback, + "allow_accounts": CommaSeparatorCallback, + "allow_groups": CommaSeparatorCallback, + "allow_qos": CommaSeparatorCallback, + "deny_accounts": CommaSeparatorCallback, + "deny_qos": CommaSeparatorCallback, + "nodes": CommaSeparatorCallback, + "tres_billing_weights": SlurmDictCallback, + } + ) + + partition_name = property(*primary_key_descriptors()) + alloc_nodes = property(*_partition_descriptors("AllocNodes")) + allow_accounts = property(*_partition_descriptors("AllowAccounts")) + allow_groups = property(*_partition_descriptors("AllowGroups")) + allow_qos = property(*_partition_descriptors("AllowQos")) + alternate = property(*_partition_descriptors("Alternate")) + cpu_bind = property(*_partition_descriptors("CpuBind")) + default = property(*_partition_descriptors("Default")) + default_time = property(*_partition_descriptors("DefaultTime")) + def_cpu_per_gpu = property(*_partition_descriptors("DefCpuPerGPU")) + def_mem_per_cpu = property(*_partition_descriptors("DefMemPerCPU")) + def_mem_per_gpu = property(*_partition_descriptors("DefMemPerGPU")) + def_mem_per_node = property(*_partition_descriptors("DefMemPerNode")) + deny_accounts = property(*_partition_descriptors("DenyAccounts")) + deny_qos = property(*_partition_descriptors("DenyQos")) + disable_root_jobs = property(*_partition_descriptors("DisableRootJobs")) + exclusive_user = property(*_partition_descriptors("ExclusiveUser")) + grace_time = property(*_partition_descriptors("GraceTime")) + hidden = property(*_partition_descriptors("Hidden")) + lln = property(*_partition_descriptors("LLN")) + max_cpus_per_node = property(*_partition_descriptors("MaxCPUsPerNode")) + max_cpus_per_socket = property(*_partition_descriptors("MaxCPUsPerSocket")) + max_mem_per_cpu = property(*_partition_descriptors("MaxMemPerCPU")) + max_mem_per_node = property(*_partition_descriptors("MaxMemPerNode")) + max_nodes = property(*_partition_descriptors("MaxNodes")) + max_time = property(*_partition_descriptors("MaxTime")) + min_nodes = property(*_partition_descriptors("MinNodes")) + nodes = property(*_partition_descriptors("Nodes")) + over_subscribe = property(*_partition_descriptors("OverSubscribe")) + over_time_limit = property(*_partition_descriptors("OverTimeLimit")) + power_down_on_idle = property(*_partition_descriptors("PowerDownOnIdle")) + preempt_mode = property(*_partition_descriptors("PreemptMode")) + priority_job_factor = property(*_partition_descriptors("PriorityJobFactor")) + priority_tier = property(*_partition_descriptors("PriorityTier")) + qos = property(*_partition_descriptors("QOS")) + req_resv = property(*_partition_descriptors("ReqResv")) + resume_timeout = property(*_partition_descriptors("ResumeTimeout")) + root_only = property(*_partition_descriptors("RootOnly")) + select_type_parameters = property(*_partition_descriptors("SelectTypeParameters")) + state = property(*_partition_descriptors("State")) + suspend_time = property(*_partition_descriptors("SuspendTime")) + suspend_timeout = property(*_partition_descriptors("SuspendTimeout")) + tres_billing_weights = property(*_partition_descriptors("TRESBillingWeights")) + + +class NodeMap(MutableMapping): + """Map of Node names to dictionaries for composing `Node` objects.""" + + def __init__(self, data: Dict[str, Dict[str, Any]]): + self._register = data + + @assert_type(value=Node) + def __setitem__(self, key: str, value: Node) -> None: + if key != value.node_name: + raise ValueError(f"{key} and {value.node_name} are not equal.") + else: + self._register.update(value.dict()) + + def __delitem__(self, key: str) -> None: + del self._register[key] + + def __getitem__(self, key: str) -> Node: + try: + node = self._register.get(key) + return Node(NodeName=key, **node) + except KeyError: + raise KeyError(f"Node {key} is not defined.") + + def __len__(self): + return len(self._register) + + def __iter__(self): + return iter([Node(NodeName=k, **self._register[k]) for k in self._register.keys()]) + + +class FrontendNodeMap(MutableMapping): + """Map of FrontendNode names to dictionaries for composing `FrontendNode` objects.""" + + def __init__(self, data: Dict[str, Dict[str, Any]]): + self._register = data + + @assert_type(value=FrontendNode) + def __setitem__(self, key: str, value: FrontendNode) -> None: + if key != value.frontend_name: + raise ValueError(f"{key} and {value.frontend_name} are not equal.") + else: + self._register.update(value.dict()) + + def __delitem__(self, key: str) -> None: + del self._register[key] + + def __getitem__(self, key: str) -> FrontendNode: + try: + frontend_node = self._register.get(key) + return FrontendNode(FrontendName=key, **frontend_node) + except KeyError: + raise KeyError(f"FrontendNode {key} is not defined.") + + def __len__(self): + return len(self._register) + + def __iter__(self): + return iter( + [FrontendNode(FrontendName=k, **self._register[k]) for k in self._register.keys()] + ) + + +class DownNodesList(UserList): + """List of dictionaries for composing `DownNodes` objects.""" + + def __getitem__(self, i): + if isinstance(i, slice): + return self.__class__(self.data[i]) + else: + return DownNodes(**self.data[i]) + + @assert_type(value=DownNodes) + def __setitem__(self, i: int, value: DownNodes): + super().__setitem__(i, value.dict()) + + @assert_type(value=DownNodes) + def __contains__(self, value): + return value.dict() in self.data + + def __add__(self, other): + if isinstance(other, DownNodesList): + return self.__class__(self.data + other.data) + elif isinstance(other, type(self.data)): + # Cannot use `assert_type` here because isinstance does + # not support using subscripted generics for runtime validation. + for down_node in other: + if not isinstance(down_node, DownNodes): + raise TypeError(f"{down_node} is not {type(DownNodes)}.") + + return self.__class__(self.data + other) + + return self.__class__(self.data + list(other)) + + def __radd__(self, other): + if isinstance(other, DownNodesList): + return self.__class__(other.data + self.data) + elif isinstance(other, type(self.data)): + for down_node in other: + if not isinstance(down_node, DownNodes): + raise TypeError(f"{down_node} is not {type(DownNodes)}.") + + return self.__class__(other + self.data) + + return self.__class__(list(other) + self.data) + + def __iadd__(self, other): + if isinstance(other, DownNodesList): + self.data += other.data + elif isinstance(other, type(self.data)): + for down_node in other: + if not isinstance(down_node, DownNodes): + raise TypeError(f"{down_node} is not {type(DownNodes)}.") + + self.data += other + else: + if not isinstance(other, DownNodes): + raise TypeError(f"{other} is not {type(DownNodes)}.") + + self.data += other + return self + + @assert_type(value=DownNodes) + def append(self, value: DownNodes): + """Add DownNodes object to list of DownNodes.""" + super().append(value.dict()) + + @assert_type(value=DownNodes) + def insert(self, i, value): + """Insert DownNodes object into list of DownNodes at the given index.""" + super().insert(i, value.dict()) + + @assert_type(value=DownNodes) + def remove(self, value): + """Remove DownNodes object from list of DownNodes.""" + self.data.remove(value.dict()) + + @assert_type(value=DownNodes) + def count(self, value): + """Count the numbers of occurrences for the given DownNodes object. + + Warnings: + Each DownNodes object should only occur once (1). If the object + occurs more than once, this can create BIG problems when trying to + restart the Slurm daemons. + """ + return self.data.count(value.dict()) + + @assert_type(value=DownNodes) + def index(self, value, *args): + """Get the index of the give DownNodes object.""" + return self.data.index(value.dict(), *args) + + def extend(self, other): + """Extend DownNodes list by appending DownNodes objects from the iterable.""" + if isinstance(other, DownNodesList): + self.data.extend(other.data) + else: + for down_node in other: + if not isinstance(down_node, DownNodes): + raise TypeError(f"{down_node} is not {type(DownNodes)}.") + + self.data.extend(other) + + +class NodeSetMap(MutableMapping): + """Map of NodeSet names to dictionaries for composing `NodeSet` objects.""" + + def __init__(self, data: Dict[str, Dict[str, Any]]): + self._register = data + + @assert_type(value=NodeSet) + def __setitem__(self, key: str, value: NodeSet) -> None: + if key != value.node_set: + raise ValueError(f"{key} and {value.node_set} are not equal.") + else: + self._register.update(value.dict()) + + def __delitem__(self, key: str) -> None: + del self._register[key] + + def __getitem__(self, key: str) -> NodeSet: + try: + node_set = self._register.get(key) + return NodeSet(NodeSet=key, **node_set) + except KeyError: + raise KeyError(f"NodeSet {key} is not defined.") + + def __len__(self): + return len(self._register) + + def __iter__(self): + return iter([NodeSet(NodeSet=k, **self._register[k]) for k in self._register.keys()]) + + +class PartitionMap(MutableMapping): + """Map of Partition names to dictionaries for composing `Partition` objects.""" + + def __init__(self, data: Dict[str, Dict[str, Any]]): + self._register = data + + def __contains__(self, key: str): + return key in self._register + + def __len__(self): + return len(self._register) + + def __iter__(self): + return iter( + [Partition(PartitionName=k, **self._register[k]) for k in self._register.keys()] + ) + + def __getitem__(self, key: str) -> Partition: + try: + partition = self._register.get(key) + return Partition(PartitionName=key, **partition) + except KeyError: + raise KeyError(f"Partition {key} is not defined.") + + @assert_type(value=Partition) + def __setitem__(self, key: str, value: Partition) -> None: + if key != value.partition_name: + raise ValueError(f"{key} and {value.partition_name} are not equal.") + else: + self._register.update(value.dict()) + + def __delitem__(self, key: str) -> None: + del self._register[key] + + +class SlurmConfig(BaseModel): + """Object representing the slurm.conf configuration file. + + Top-level configuration definition and data validators sourced from + the slurm.conf manpage. `man slurm.conf.5` + """ + + _callbacks = MappingProxyType( + { + "acct_storage_external_host": CommaSeparatorCallback, + "acct_storage_param": SlurmDictCallback, + "acct_storage_tres": CommaSeparatorCallback, + "acct_store_flags": CommaSeparatorCallback, + "auth_alt_types": CommaSeparatorCallback, + "auth_alt_param": SlurmDictCallback, + "auth_info": SlurmDictCallback, + "bcast_exclude": CommaSeparatorCallback, + "bcast_param": SlurmDictCallback, + "cli_filter_plugins": CommaSeparatorCallback, + "communication_params": SlurmDictCallback, + "cpu_freq_def": CommaSeparatorCallback, + "cpu_freq_governors": CommaSeparatorCallback, + "debug_flags": CommaSeparatorCallback, + "dependency_param": SlurmDictCallback, + "federation_param": CommaSeparatorCallback, + "health_check_node_state": CommaSeparatorCallback, + "job_acct_gather_frequency": SlurmDictCallback, + "job_comp_params": SlurmDictCallback, + "job_submit_plugins": CommaSeparatorCallback, + "launch_parameters": SlurmDictCallback, + "licenses": CommaSeparatorCallback, + "plugin_dir": ColonSeparatorCallback, + "power_parameters": SlurmDictCallback, + "preempt_mode": CommaSeparatorCallback, + "preempt_param": SlurmDictCallback, + "prep_plugins": CommaSeparatorCallback, + "priority_weight_tres": SlurmDictCallback, + "private_data": CommaSeparatorCallback, + "prolog_flags": CommaSeparatorCallback, + "propagate_resource_limits": CommaSeparatorCallback, + "propagate_resource_limits_except": CommaSeparatorCallback, + "scheduler_param": SlurmDictCallback, + "scron_param": CommaSeparatorCallback, + "slurmctld_param": SlurmDictCallback, + "slurmd_param": CommaSeparatorCallback, + "switch_param": SlurmDictCallback, + "task_plugin": CommaSeparatorCallback, + "task_plugin_param": SlurmDictCallback, + "topology_param": CommaSeparatorCallback, + } + ) + + include = property(*base_descriptors("Include")) + accounting_storage_backup_host = property(*base_descriptors("AccountingStorageBackupHost")) + accounting_storage_enforce = property(*base_descriptors("AccountingStorageEnforce")) + account_storage_external_host = property(*base_descriptors("AccountStorageExternalHost")) + accounting_storage_host = property(*base_descriptors("AccountingStorageHost")) + accounting_storage_parameters = property(*base_descriptors("AccountingStorageParameters")) + accounting_storage_pass = property(*base_descriptors("AccountingStoragePass")) + accounting_storage_port = property(*base_descriptors("AccountingStoragePort")) + accounting_storage_tres = property(*base_descriptors("AccountingStorageTRES")) + accounting_storage_type = property(*base_descriptors("AccountingStorageType")) + accounting_storage_user = property(*base_descriptors("AccountingStorageUser")) + accounting_store_flags = property(*base_descriptors("AccountingStoreFlags")) + acct_gather_node_freq = property(*base_descriptors("AcctGatherNodeFreq")) + acct_gather_energy_type = property(*base_descriptors("AcctGatherEnergyType")) + acct_gather_interconnect_type = property(*base_descriptors("AcctGatherInterconnectType")) + acct_gather_filesystem_type = property(*base_descriptors("AcctGatherFilesystemType")) + acct_gather_profile_type = property(*base_descriptors("AcctGatherProfileType")) + allow_spec_resources_usage = property(*base_descriptors("AllowSpecResourcesUsage")) + auth_alt_types = property(*base_descriptors("AuthAltTypes")) + auth_alt_parameters = property(*base_descriptors("AuthAltParameters")) + auth_info = property(*base_descriptors("AuthInfo")) + auth_type = property(*base_descriptors("AuthType")) + batch_start_timeout = property(*base_descriptors("BatchStartTimeout")) + bcast_exclude = property(*base_descriptors("BcastExclude")) + bcast_parameters = property(*base_descriptors("BcastParameters")) + burst_buffer_type = property(*base_descriptors("BurstBufferType")) + cli_filter_plugins = property(*base_descriptors("CliFilterPlugins")) + cluster_name = property(*base_descriptors("ClusterName")) + communication_parameters = property(*base_descriptors("CommunicationParameters")) + complete_wait = property(*base_descriptors("CompleteWait")) + core_spec_plugin = property(*base_descriptors("CoreSpecPlugin")) + cpu_freq_def = property(*base_descriptors("CpuFreqDef")) + cpu_freq_governors = property(*base_descriptors("CpuFreqGovernors")) + cred_type = property(*base_descriptors("CredType")) + debug_flags = property(*base_descriptors("DebugFlags")) + def_cpu_per_gpu = property(*base_descriptors("DefCpuPerGPU")) + def_mem_per_cpu = property(*base_descriptors("DefMemPerCPU")) + def_mem_per_gpu = property(*base_descriptors("DefMemPerGPU")) + def_mem_per_node = property(*base_descriptors("DefMemPerNode")) + dependency_parameters = property(*base_descriptors("DependencyParameters")) + disable_root_jobs = property(*base_descriptors("DisableRootJobs")) + eio_timeout = property(*base_descriptors("EioTimeout")) + enforce_part_limits = property(*base_descriptors("EnforcePartLimits")) + epilog = property(*base_descriptors("Epilog")) + epilog_msg_time = property(*base_descriptors("EpilogMsgTime")) + epilog_slurmctld = property(*base_descriptors("EpilogSlurmctld")) + ext_sensors_freq = property(*base_descriptors("ExtSensorsFreq")) + ext_sensors_type = property(*base_descriptors("ExtSensorsType")) + fair_share_dampening_factor = property(*base_descriptors("FairShareDampeningFactor")) + federation_parameters = property(*base_descriptors("FederationParameters")) + first_job_id = property(*base_descriptors("FirstJobId")) + get_env_timeout = property(*base_descriptors("GetEnvTimeout")) + gres_types = property(*base_descriptors("GresTypes")) + group_update_force = property(*base_descriptors("GroupUpdateForce")) + group_update_time = property(*base_descriptors("GroupUpdateTime")) + gpu_freq_def = property(*base_descriptors("GpuFreqDef")) + health_check_interval = property(*base_descriptors("HealthCheckInterval")) + health_check_node_state = property(*base_descriptors("HealthCheckNodeState")) + health_check_program = property(*base_descriptors("HealthCheckProgram")) + inactive_limit = property(*base_descriptors("InactiveLimit")) + interactive_step_options = property(*base_descriptors("InteractiveStepOptions")) + job_acct_gather_type = property(*base_descriptors("JobAcctGatherType")) + job_acct_gather_frequency = property(*base_descriptors("JobAcctGatherFrequency")) + job_acct_gather_params = property(*base_descriptors("JobAcctGatherParams")) + job_comp_host = property(*base_descriptors("JobCompHost")) + job_comp_loc = property(*base_descriptors("JobCompLoc")) + job_comp_params = property(*base_descriptors("JobCompParams")) + job_comp_pass = property(*base_descriptors("JobCompPass")) + job_comp_port = property(*base_descriptors("JobCompPort")) + job_comp_type = property(*base_descriptors("JobCompType")) + job_comp_user = property(*base_descriptors("JobCompUser")) + job_container_type = property(*base_descriptors("JobContainerType")) + job_file_append = property(*base_descriptors("JobFileAppend")) + job_requeue = property(*base_descriptors("JobRequeue")) + job_submit_plugins = property(*base_descriptors("JobSubmitPlugins")) + kill_on_bad_exit = property(*base_descriptors("KillOnBadExit")) + kill_wait = property(*base_descriptors("KillWait")) + max_batch_requeue = property(*base_descriptors("MaxBatchRequeue")) + node_features_plugins = property(*base_descriptors("NodeFeaturesPlugins")) + launch_parameters = property(*base_descriptors("LaunchParameters")) + licenses = property(*base_descriptors("Licenses")) + log_time_format = property(*base_descriptors("LogTimeFormat")) + mail_domain = property(*base_descriptors("MailDomain")) + mail_prog = property(*base_descriptors("MailProg")) + max_array_size = property(*base_descriptors("MaxArraySize")) + max_job_count = property(*base_descriptors("MaxJobCount")) + max_job_id = property(*base_descriptors("MaxJobId")) + max_mem_per_cpu = property(*base_descriptors("MaxMemPerCPU")) + max_mem_per_node = property(*base_descriptors("MaxMemPerNode")) + max_node_count = property(*base_descriptors("MaxNodeCount")) + max_step_count = property(*base_descriptors("MaxStepCount")) + max_tasks_per_node = property(*base_descriptors("MaxTasksPerNode")) + mcs_parameters = property(*base_descriptors("MCSParameters")) + mcs_plugin = property(*base_descriptors("MCSPlugin")) + message_timeout = property(*base_descriptors("MessageTimeout")) + min_job_age = property(*base_descriptors("MinJobAge")) + mpi_default = property(*base_descriptors("MpiDefault")) + mpi_params = property(*base_descriptors("MpiParams")) + over_time_limit = property(*base_descriptors("OverTimeLimit")) + plugin_dir = property(*base_descriptors("PluginDir")) + plug_stack_config = property(*base_descriptors("PlugStackConfig")) + power_parameters = property(*base_descriptors("PowerParameters")) + power_plugin = property(*base_descriptors("PowerPlugin")) + preempt_mode = property(*base_descriptors("PreemptMode")) + preempt_parameters = property(*base_descriptors("PreemptParameters")) + preempt_type = property(*base_descriptors("PreemptType")) + preempt_exempt_time = property(*base_descriptors("PreemptExemptTime")) + prep_parameters = property(*base_descriptors("PrEpParameters")) + prep_plugins = property(*base_descriptors("PrEpPlugins")) + priority_calcp_period = property(*base_descriptors("PriorityCalcpPeriod")) + priority_decay_half_life = property(*base_descriptors("PriorityDecayHalfLife")) + priority_favor_small = property(*base_descriptors("PriorityFavorSmall")) + priority_flags = property(*base_descriptors("PriorityFlags")) + priority_max_age = property(*base_descriptors("PriorityMaxAge")) + priority_parameters = property(*base_descriptors("PriorityParameters")) + priority_site_factor_parameters = property(*base_descriptors("PrioritySiteFactorParameters")) + priority_site_factor_plugin = property(*base_descriptors("PrioritySiteFactorPlugin")) + priority_type = property(*base_descriptors("PriorityType")) + priority_usage_reset_period = property(*base_descriptors("PriorityUsageResetPeriod")) + priority_weight_age = property(*base_descriptors("PriorityWeightAge")) + priority_weight_assoc = property(*base_descriptors("PriorityWeightAssoc")) + priority_weight_fair_share = property(*base_descriptors("PriorityWeightFairShare")) + priority_weight_job_size = property(*base_descriptors("PriorityWeightJobSize")) + priority_weight_partition = property(*base_descriptors("PriorityWeightPartition")) + priority_weight_qos = property(*base_descriptors("PriorityWeightQOS")) + priority_weight_tres = property(*base_descriptors("PriorityWeightTRES")) + private_data = property(*base_descriptors("PrivateData")) + proctrack_type = property(*base_descriptors("ProctrackType")) + prolog = property(*base_descriptors("Prolog")) + prolog_epilog_timeout = property(*base_descriptors("PrologEpilogTimeout")) + prolog_flags = property(*base_descriptors("PrologFlags")) + prolog_slurmctld = property(*base_descriptors("PrologSlurmctld")) + propagate_prio_process = property(*base_descriptors("PropagatePrioProcess")) + propagate_resource_limits = property(*base_descriptors("PropagateResourceLimits")) + propagate_resource_limits_except = property(*base_descriptors("PropagateResourceLimitsExcept")) + reboot_program = property(*base_descriptors("RebootProgram")) + reconfig_flags = property(*base_descriptors("ReconfigFlags")) + requeue_exit = property(*base_descriptors("RequeueExit")) + requeue_exit_hold = property(*base_descriptors("RequeueExitHold")) + resume_fail_program = property(*base_descriptors("ResumeFailProgram")) + resume_program = property(*base_descriptors("ResumeProgram")) + resume_rate = property(*base_descriptors("ResumeRate")) + resume_timeout = property(*base_descriptors("ResumeTimeout")) + resv_epilog = property(*base_descriptors("ResvEpilog")) + resv_over_run = property(*base_descriptors("ResvOverRun")) + resv_prolog = property(*base_descriptors("ResvProlog")) + return_to_service = property(*base_descriptors("ReturnToService")) + route_plugin = property(*base_descriptors("RoutePlugin")) + scheduler_parameters = property(*base_descriptors("SchedulerParameters")) + scheduler_time_slice = property(*base_descriptors("SchedulerTimeSlice")) + scheduler_type = property(*base_descriptors("SchedulerType")) + scron_parameters = property(*base_descriptors("ScronParameters")) + select_type = property(*base_descriptors("SelectType")) + select_type_parameters = property(*base_descriptors("SelectTypeParameters")) + slurmctld_addr = property(*base_descriptors("SlurmctldAddr")) + slurmctld_debug = property(*base_descriptors("SlurmctldDebug")) + slurmctld_host = property(*base_descriptors("SlurmctldHost")) + slurmctld_log_file = property(*base_descriptors("SlurmctldLogFile")) + slurmctld_parameters = property(*base_descriptors("SlurmctldParameters")) + slurmctld_pid_file = property(*base_descriptors("SlurmctldPidFile")) + slurmctld_port = property(*base_descriptors("SlurmctldPort")) + slurmctld_primary_off_prog = property(*base_descriptors("SlurmctldPrimaryOffProg")) + slurmctld_primary_on_prog = property(*base_descriptors("SlurmctldPrimaryOnProg")) + slurmctld_syslog_debug = property(*base_descriptors("SlurmctldSyslogDebug")) + slurmctld_timeout = property(*base_descriptors("SlurmctldTimeout")) + slurmd_debug = property(*base_descriptors("SlurmdDebug")) + slurmd_log_file = property(*base_descriptors("SlurmdLogFile")) + slurmd_parameters = property(*base_descriptors("SlurmdParameters")) + slurmd_pid_file = property(*base_descriptors("SlurmdPidFile")) + slurmd_port = property(*base_descriptors("SlurmdPort")) + slurmd_spool_dir = property(*base_descriptors("SlurmdSpoolDir")) + slurmd_syslog_debug = property(*base_descriptors("SlurmdSyslogDebug")) + slurmd_timeout = property(*base_descriptors("SlurmdTimeout")) + slurmd_user = property(*base_descriptors("SlurmdUser")) + slurm_sched_log_file = property(*base_descriptors("SlurmSchedLogFile")) + slurm_sched_log_level = property(*base_descriptors("SlurmSchedLogLevel")) + slurm_user = property(*base_descriptors("SlurmUser")) + srun_epilog = property(*base_descriptors("SrunEpilog")) + srun_port_range = property(*base_descriptors("SrunPortRange")) + srun_prolog = property(*base_descriptors("SrunProlog")) + state_save_location = property(*base_descriptors("StateSaveLocation")) + suspend_exc_nodes = property(*base_descriptors("SuspendExcNodes")) + suspend_exc_parts = property(*base_descriptors("SuspendExcParts")) + suspend_exc_states = property(*base_descriptors("SuspendExcStates")) + suspend_program = property(*base_descriptors("SuspendProgram")) + suspend_rate = property(*base_descriptors("SuspendRate")) + suspend_time = property(*base_descriptors("SuspendTime")) + suspend_timeout = property(*base_descriptors("SuspendTimeout")) + switch_parameters = property(*base_descriptors("SwitchParameters")) + switch_type = property(*base_descriptors("SwitchType")) + task_epilog = property(*base_descriptors("TaskEpilog")) + task_plugin = property(*base_descriptors("TaskPlugin")) + task_plugin_param = property(*base_descriptors("TaskPluginParam")) + task_prolog = property(*base_descriptors("TaskProlog")) + tcp_timeout = property(*base_descriptors("TCPTimeout")) + tmp_fs = property(*base_descriptors("TmpFS")) + topology_param = property(*base_descriptors("TopologyParam")) + topology_plugin = property(*base_descriptors("TopologyPlugin")) + track_wc_key = property(*base_descriptors("TrackWCKey")) + tree_width = property(*base_descriptors("TreeWidth")) + unkillable_step_program = property(*base_descriptors("UnkillableStepProgram")) + unkillable_step_timeout = property(*base_descriptors("UnkillableStepTimeout")) + use_pam = property(*base_descriptors("UsePAM")) + vsize_factor = property(*base_descriptors("VSizeFactor")) + wait_time = property(*base_descriptors("WaitTime")) + x11_parameters = property(*base_descriptors("X11Parameters")) + + @property + def nodes(self) -> NodeMap: + """Get the current nodes in the Slurm configuration.""" + return NodeMap(self._register["nodes"]) + + @property + def frontend_nodes(self) -> FrontendNodeMap: + """Get the current frontend nodes in the Slurm configuration.""" + return FrontendNodeMap(self._register["frontend_nodes"]) + + @property + def down_nodes(self) -> DownNodesList: + """Get the current down nodes in the Slurm configuration.""" + return DownNodesList(self._register["down_nodes"]) + + @property + def node_sets(self) -> NodeSetMap: + """Get the current node sets in the Slurm configuration.""" + return NodeSetMap(self._register["node_sets"]) + + @property + def partitions(self) -> PartitionMap: + """Get the current partitions in the Slurm configuration.""" + return PartitionMap(self._register["partitions"]) diff --git a/slurmutils/models/slurmdbd.py b/slurmutils/models/slurmdbd.py new file mode 100644 index 0000000..d42b962 --- /dev/null +++ b/slurmutils/models/slurmdbd.py @@ -0,0 +1,76 @@ +# Copyright 2024 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +"""Data models for the slurmdbd daemon.""" + +from ._model import BaseModel, base_descriptors + + +class SlurmdbdConfig(BaseModel): + """Object representing the slurmdbd.conf configuration file. + + Top-level configuration definition and data validators sourced from + the slurmdbd.conf manpage. `man slurmdbd.conf.5` + """ + + archive_dir = property(*base_descriptors("ArchiveDir")) + archive_events = property(*base_descriptors("ArchiveEvents")) + archive_jobs = property(*base_descriptors("ArchiveJobs")) + archive_resvs = property(*base_descriptors("ArchiveResvs")) + archive_script = property(*base_descriptors("ArchiveScript")) + archive_steps = property(*base_descriptors("ArchiveSteps")) + archive_suspend = property(*base_descriptors("ArchiveSuspend")) + archive_txn = property(*base_descriptors("ArchiveTXN")) + archive_usage = property(*base_descriptors("ArchiveUsage")) + auth_info = property(*base_descriptors("AuthInfo")) + auth_alt_types = property(*base_descriptors("AuthAltTypes")) + auth_alt_parameters = property(*base_descriptors("AuthAltParameters")) + auth_type = property(*base_descriptors("AuthType")) + commit_delay = property(*base_descriptors("CommitDelay")) + communication_parameters = property(*base_descriptors("CommunicationParameters")) + dbd_backup_host = property(*base_descriptors("DbdBackupHost")) + dbd_addr = property(*base_descriptors("DbdAddr")) + dbd_host = property(*base_descriptors("DbdHost")) + dbd_port = property(*base_descriptors("DbdPort")) + debug_flags = property(*base_descriptors("DebugFlags")) + debug_level = property(*base_descriptors("DebugLevel")) + debug_level_syslog = property(*base_descriptors("DebugLevelSyslog")) + default_qos = property(*base_descriptors("DefaultQOS")) + log_file = property(*base_descriptors("LogFile")) + log_time_format = property(*base_descriptors("LogTimeFormat")) + max_query_time_range = property(*base_descriptors("MaxQueryTimeRange")) + message_timeout = property(*base_descriptors("MessageTimeout")) + parameters = property(*base_descriptors("Parameters")) + pid_file = property(*base_descriptors("PidFile")) + plugin_dir = property(*base_descriptors("PluginDir")) + private_data = property(*base_descriptors("PrivateData")) + purge_event_after = property(*base_descriptors("PurgeEventAfter")) + purge_job_after = property(*base_descriptors("PurgeJobAfter")) + purge_resv_after = property(*base_descriptors("PurgeResvAfter")) + purge_step_after = property(*base_descriptors("PurgeStepAfter")) + purge_suspend_after = property(*base_descriptors("PurgeSuspendAfter")) + purge_txn_after = property(*base_descriptors("PurgeTXNAfter")) + purge_usage_after = property(*base_descriptors("PurgeUsageAfter")) + slurm_user = property(*base_descriptors("SlurmUser")) + storage_host = property(*base_descriptors("StorageHost")) + storage_backup_host = property(*base_descriptors("StorageBackupHost")) + storage_loc = property(*base_descriptors("StorageLoc")) + storage_parameters = property(*base_descriptors("StorageParameters")) + storage_pass = property(*base_descriptors("StoragePass")) + storage_port = property(*base_descriptors("StoragePort")) + storage_type = property(*base_descriptors("StorageType")) + storage_user = property(*base_descriptors("StorageUser")) + tcp_timeout = property(*base_descriptors("TCPTimeout")) + track_slurmctld_down = property(*base_descriptors("TrackSlurmctldDown")) + track_wc_key = property(*base_descriptors("TrackWCKey")) From b8129b145e656e1243560aee66c71b9bd29771a2 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Thu, 25 Jan 2024 13:54:55 -0500 Subject: [PATCH 07/19] feat!: Add editors BREAKING CHANGE: Marshaller is not currently ready for full-scale deployment. Also, a context management class is no longer used for SlurmConf because it was annoying to manage all the internal class attributes but also create secondary constructors. Signed-off-by: Jason C. Nucciarone --- slurmutils/editors/__init__.py | 18 +++ slurmutils/editors/_editor.py | 175 +++++++++++++++++++++++++++ slurmutils/editors/slurmconfig.py | 132 ++++++++++++++++++++ slurmutils/editors/slurmdbdconfig.py | 15 +++ slurmutils/exceptions.py | 19 +++ 5 files changed, 359 insertions(+) create mode 100644 slurmutils/editors/__init__.py create mode 100644 slurmutils/editors/_editor.py create mode 100644 slurmutils/editors/slurmconfig.py create mode 100644 slurmutils/editors/slurmdbdconfig.py create mode 100644 slurmutils/exceptions.py diff --git a/slurmutils/editors/__init__.py b/slurmutils/editors/__init__.py new file mode 100644 index 0000000..3deb773 --- /dev/null +++ b/slurmutils/editors/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2024 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +"""Editors for Slurm workload manager configuration files.""" + +from . import slurmconfig +from . import slurmdbdconfig diff --git a/slurmutils/editors/_editor.py b/slurmutils/editors/_editor.py new file mode 100644 index 0000000..49718eb --- /dev/null +++ b/slurmutils/editors/_editor.py @@ -0,0 +1,175 @@ +# Copyright 2024 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +"""Base methods for Slurm workload manager configuration file editors.""" + +import logging +import re +import shlex +from collections import deque +from os import PathLike +from pathlib import Path +from typing import Deque, Dict, List, Union + +from slurmutils.exceptions import EditorError + +_logger = logging.getLogger(__name__) + + +def dump_base(content, file: Union[str, PathLike], marshaller): + """Dump configuration into file using provided marshalling function. + + Do not use this function directly. + """ + if (loc := Path(file)).exists(): + _logger.warning("Overwriting contents of %s file located at %s.", loc.name, loc) + + _logger.debug("Marshalling configuration into %s file located at %s.", loc.name, loc) + return loc.write_text(marshaller(content), encoding="ascii") + + +def dumps_base(content, marshaller) -> str: + """Dump configuration into Python string using provided marshalling function. + + Do not use this function directly. + """ + return marshaller(content) + + +def load_base(file: Union[str, PathLike], parser): + """Load configuration from file using provided parsing function. + + Do not use this function directly. + """ + if (file := Path(file)).exists(): + _logger.debug("Parsing contents of %s located at %s.", file.name, file) + config = file.read_text(encoding="ascii") + return parser(config) + else: + msg = "Unable to locate file" + _logger.error(msg + " %s.", file) + raise FileNotFoundError(msg + f" {file}") + + +def loads_base(content: str, parser): + """Load configuration from Python String using provided parsing function. + + Do not use this function directly. + """ + return parser(content) + + +# Helper functions for parsing and marshalling Slurm configuration data. + +_loose_pascal_filter = re.compile(r"(.)([A-Z][a-z]+)") +_snakecase_convert = re.compile(r"([a-z0-9])([A-Z])") + + +def _pascal2snake(v: str) -> str: + """Convert string in loose PascalCase to snakecase. + + This private method takes in Slurm configuration knob keys and converts + them to snakecase. The returned snakecase representation is used to + dynamically access Slurm data model attributes and retrieve callbacks. + """ + # The precompiled regex filters do a good job of converting Slurm's + # loose PascalCase to snakecase, however, there are still some tokens + # that slip through such as `CPUs`. This filter identifies those problematic + # tokens and converts them into tokens that can be easily processed by the + # compiled regex expressions. + if "CPUs" in v: + v = v.replace("CPUs", "Cpus") + holder = _loose_pascal_filter.sub(r"\1_\2", v) + return _snakecase_convert.sub(r"\1_\2", holder).lower() + + +def clean(config: Deque[str]) -> Deque[str]: + """Clean loaded configuration file before parsing. + + Cleaning tasks include: + 1. Stripping away comments (#) in configuration. Slurm does not + support octothorpes in strings; only for inline and standalone + comments. **Do not use** octothorpes in Slurm configuration knob + values as Slurm will treat anything proceeding an octothorpe as a comment. + 2. Strip away any extra whitespace at the end of each line. + + Args: + config: Loaded configuration file. Split by newlines. + """ + processed = deque() + while config: + line = config.popleft() + if line.startswith("#"): + # Skip comment lines as they're not necessary for configuration. + continue + elif "#" in line: + # Slice off inline comment and strip away extra whitespace. + processed.append(line[: line.index("#")].strip()) + else: + processed.append(line.strip()) + + return processed + + +def parse_repeating_config(__key, __value, pocket: Dict) -> None: + """Parse `slurm.conf` configuration knobs with keys that can repeat. + + Args: + __key: Configuration knob key that can repeat. + __value: Value of the current configuration knob. + pocket: Dictionary to add parsed configuration knob to. + """ + if __key not in pocket: + pocket[__key] = [__value] + else: + pocket[__key].append(__value) + + +def parse_model(line: str, pocket: Union[Dict, List], model) -> None: + """Parse configuration knobs based on Slurm models. + + Model callbacks will be used for invoking special + parsing if required for the configuration value in line. + + Args: + line: Configuration line to parse. + pocket: Dictionary to add parsed configuration knob to. + model: Slurm data model to use for invoking callbacks and validating knob keys. + """ + holder = {} + for token in shlex.split(line): # Use `shlex.split(...)` to preserve quotation blocks. + # Word in front of the first `=` denotes the parent configuration knob key. + option, value = token.split("=", maxsplit=1) + if hasattr(model, attr := _pascal2snake(option)): + if attr in model._callbacks and (callback := model._callbacks[attr].parse) is not None: + holder.update({option: callback(value)}) + else: + holder.update({option: value}) + else: + raise EditorError( + f"{option} is not a valid configuration option for {model.__class__.__name__}." + ) + + # Use temporary model object to update pocket with a Python dictionary + # in the format that we want the dictionary to be. + if isinstance(pocket, list): + pocket.append(model(**holder).dict()) + else: + pocket.update(model(**holder).dict()) + + +def header(): + """Generate header for marshalled configuration.""" + # TODO: Add a header with the time the configuration + # file was created. diff --git a/slurmutils/editors/slurmconfig.py b/slurmutils/editors/slurmconfig.py new file mode 100644 index 0000000..bf591ab --- /dev/null +++ b/slurmutils/editors/slurmconfig.py @@ -0,0 +1,132 @@ +# Copyright 2024 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +"""Edit slurm.conf files.""" + +__all__ = ["dump", "dumps", "load", "loads", "edit"] + +import functools +import os +from collections import deque +from contextlib import contextmanager +from typing import Union + +from slurmutils.models import DownNodes, FrontendNode, Node, NodeSet, Partition, SlurmConfig + +from ._editor import ( + clean, + dump_base, + dumps_base, + load_base, + loads_base, + parse_model, + parse_repeating_config, +) + + +def _marshaller(config: SlurmConfig) -> str: + """Marshall Python object into slurm.conf configuration file. + + Args: + config: Python object to convert to slurm.conf configuration file. + """ + ... + + +_parse_slurm = functools.partial(parse_model, model=SlurmConfig) +_parse_node = functools.partial(parse_model, model=Node) +_parse_frontend = functools.partial(parse_model, model=FrontendNode) +_parse_down_node = functools.partial(parse_model, model=DownNodes) +_parse_node_set = functools.partial(parse_model, model=NodeSet) +_parse_partition = functools.partial(parse_model, model=Partition) + + +def _parser(config: str) -> SlurmConfig: + """Parse slurm.conf configuration file into Python object. + + Args: + config: Content of slurm.conf configuration file. + """ + slurm_conf = {} + nodes = {} + down_nodes = [] + frontend_nodes = {} + node_sets = {} + partitions = {} + + # import pdb + # pdb.set_trace() + config = clean(deque(config.splitlines())) + while config: + line = config.popleft() + # slurm.conf `Include` is the only configuration knob whose + # separator is whitespace rather than `=`. + if line.startswith("Include"): + option, value = line.split(maxsplit=1) + parse_repeating_config(option, value, pocket=slurm_conf) + + # `SlurmctldHost` is the same as `Include` where it can + # be specified on multiple lines. + elif line.startswith("SlurmctldHost"): + option, value = line.split("=", 1) + parse_repeating_config(option, value, pocket=slurm_conf) + + # Check if option maps to slurm.conf data model. If so, invoke parsing + # rules for that specific data model and enter its parsed information + # into the appropriate pocket. + elif line.startswith("NodeName"): + _parse_node(line, pocket=nodes) + elif line.startswith("DownNodes"): + _parse_down_node(line, pocket=down_nodes) + elif line.startswith("FrontendNode"): + _parse_frontend(line, pocket=frontend_nodes) + elif line.startswith("NodeSet"): + _parse_node_set(line, pocket=node_sets) + elif line.startswith("PartitionName"): + _parse_partition(line, pocket=partitions) + else: + _parse_slurm(line, pocket=slurm_conf) + + return SlurmConfig( + **slurm_conf, + nodes=nodes, + frontend_nodes=frontend_nodes, + down_nodes=down_nodes, + node_sets=node_sets, + partitions=partitions, + ) + + +dump = functools.partial(dump_base, marshaller=_marshaller) +dumps = functools.partial(dumps_base, marshaller=_marshaller) +load = functools.partial(load_base, parser=_parser) +loads = functools.partial(loads_base, parser=_parser) + + +@contextmanager +def edit(file: Union[str, os.PathLike]) -> SlurmConfig: + """Edit a slurm.conf file. + + Args: + file: Path to slurm.conf file to edit. If slurm.conf does + not exist at the specified file path, it will be created. + """ + if os.path.exists(file): + # Create an empty SlurmConfig that can be populated. + config = SlurmConfig() + else: + config = load(file=file) + + yield config + dump(content=config, file=file) diff --git a/slurmutils/editors/slurmdbdconfig.py b/slurmutils/editors/slurmdbdconfig.py new file mode 100644 index 0000000..82e43c4 --- /dev/null +++ b/slurmutils/editors/slurmdbdconfig.py @@ -0,0 +1,15 @@ +# Copyright 2024 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +"""Edit slurmdbd.conf files.""" diff --git a/slurmutils/exceptions.py b/slurmutils/exceptions.py new file mode 100644 index 0000000..253c669 --- /dev/null +++ b/slurmutils/exceptions.py @@ -0,0 +1,19 @@ +# Copyright 2024 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +"""Exceptions raised by Slurm utilities in this package.""" + + +class EditorError(Exception): + """Raise when a Slurm configuration editor encounters an error.""" From b9e6a92901a37f47b6ebf5d297528c1808e07a27 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Tue, 30 Jan 2024 13:34:36 -0500 Subject: [PATCH 08/19] feat: Add marshaller for slurm.conf Signed-off-by: Jason C. Nucciarone --- slurmutils/editors/_editor.py | 66 ++++++++++++++++++++-- slurmutils/editors/slurmconfig.py | 91 ++++++++++++++++++++++++------- slurmutils/models/_model.py | 18 ++++++ slurmutils/models/slurm.py | 6 ++ 4 files changed, 156 insertions(+), 25 deletions(-) diff --git a/slurmutils/editors/_editor.py b/slurmutils/editors/_editor.py index 49718eb..662640c 100644 --- a/slurmutils/editors/_editor.py +++ b/slurmutils/editors/_editor.py @@ -20,7 +20,7 @@ from collections import deque from os import PathLike from pathlib import Path -from typing import Deque, Dict, List, Union +from typing import Deque, Dict, List, Optional, Set, Union from slurmutils.exceptions import EditorError @@ -122,6 +122,15 @@ def clean(config: Deque[str]) -> Deque[str]: return processed +def header(msg: str) -> str: + """Generate header for marshalled configuration file. + + Args: + msg: Message to put into header. + """ + return "#\n" + "".join(f"# {line}\n" for line in msg.splitlines()) + "#\n" + + def parse_repeating_config(__key, __value, pocket: Dict) -> None: """Parse `slurm.conf` configuration knobs with keys that can repeat. @@ -158,7 +167,7 @@ def parse_model(line: str, pocket: Union[Dict, List], model) -> None: holder.update({option: value}) else: raise EditorError( - f"{option} is not a valid configuration option for {model.__class__.__name__}." + f"{option} is not a valid configuration option for {model.__name__}." ) # Use temporary model object to update pocket with a Python dictionary @@ -169,7 +178,52 @@ def parse_model(line: str, pocket: Union[Dict, List], model) -> None: pocket.update(model(**holder).dict()) -def header(): - """Generate header for marshalled configuration.""" - # TODO: Add a header with the time the configuration - # file was created. +def marshal_model( + model, ignore: Optional[Set] = None, inline: bool = False +) -> Union[List[str], str]: + """Marshal a Slurm model back into its Slurm configuration syntax. + + Args: + model: Slurm model object to marshal into Slurm configuration syntax. + ignore: Set of keys to ignore on model object when marshalling. Useful for models that + have child models under certain keys that are directly handled. Default is None. + inline: If True, marshal object into single line rather than multiline. Default is False. + """ + marshalled = [] + if ignore is None: + # Create an empty set if not ignores are specified. Prevents us from needing to + # rely on a mutable default in the function signature. + ignore = set() + + if primary_key := model._primary_key: + attr = _pascal2snake(primary_key) + primary_value = getattr(model, attr) + data = {primary_key: primary_value, **model.dict()[primary_value]} + else: + data = model.dict() + + for option, value in data.items(): + if option not in ignore: + if hasattr(model, attr := _pascal2snake(option)): + if ( + attr in model._callbacks + and (callback := model._callbacks[attr].marshal) is not None + ): + value = callback(value) + + marshalled.append(f"{option}={value}") + else: + raise EditorError( + f"{option} is not a valid configuration option for {model.__class__.__name__}." + ) + else: + _logger.debug("Ignoring option %s. Option is present in ignore set %s", option, ignore) + + if inline: + # Whitespace is the seperator in Slurm configuration syntax. + marshalled = " ".join(marshalled) + "\n" + else: + # Append newline character so that each configuration is on its own line. + marshalled = [line + "\n" for line in marshalled] + + return marshalled diff --git a/slurmutils/editors/slurmconfig.py b/slurmutils/editors/slurmconfig.py index bf591ab..e438080 100644 --- a/slurmutils/editors/slurmconfig.py +++ b/slurmutils/editors/slurmconfig.py @@ -20,6 +20,7 @@ import os from collections import deque from contextlib import contextmanager +from datetime import datetime from typing import Union from slurmutils.models import DownNodes, FrontendNode, Node, NodeSet, Partition, SlurmConfig @@ -28,8 +29,10 @@ clean, dump_base, dumps_base, + header, load_base, loads_base, + marshal_model, parse_model, parse_repeating_config, ) @@ -39,17 +42,69 @@ def _marshaller(config: SlurmConfig) -> str: """Marshall Python object into slurm.conf configuration file. Args: - config: Python object to convert to slurm.conf configuration file. + config: `SlurmConfig` object to convert to slurm.conf configuration file. """ - ... - + marshalled = [header(f"`slurm.conf` file generated at {datetime.now()} by slurmutils.")] + + if config.include: + marshalled.append(header("Included configuration files")) + marshalled.extend([f"Include {i}\n" for i in config.include] + ["\n"]) + if config.slurmctld_host: + marshalled.extend([f"SlurmctldHost={host}\n" for host in config.slurmctld_host] + ["\n"]) + + # Marshal the SlurmConfig object into Slurm configuration format. + # Ignore pockets containing child models as they will be marshalled inline. + marshalled.extend( + marshal_model( + config, + ignore={ + "Includes", + "SlurmctldHost", + "nodes", + "frontend_nodes", + "down_nodes", + "node_sets", + "partitions", + }, + ) + + ["\n"] + ) -_parse_slurm = functools.partial(parse_model, model=SlurmConfig) -_parse_node = functools.partial(parse_model, model=Node) -_parse_frontend = functools.partial(parse_model, model=FrontendNode) -_parse_down_node = functools.partial(parse_model, model=DownNodes) -_parse_node_set = functools.partial(parse_model, model=NodeSet) -_parse_partition = functools.partial(parse_model, model=Partition) + if len(config.nodes) != 0: + marshalled.extend( + [header("Node configurations")] + + [marshal_model(node, inline=True) for node in config.nodes] + + ["\n"] + ) + + if len(config.frontend_nodes) != 0: + marshalled.extend( + [header("Frontend node configurations")] + + [marshal_model(frontend, inline=True) for frontend in config.frontend_nodes] + + ["\n"] + ) + + if len(config.down_nodes) != 0: + marshalled.extend( + [header("Down node configurations")] + + [marshal_model(down_node, inline=True) for down_node in config.down_nodes] + + ["\n"] + ) + + if len(config.node_sets) != 0: + marshalled.extend( + [header("Node set configurations")] + + [marshal_model(node_set, inline=True) for node_set in config.node_sets] + + ["\n"] + ) + + if len(config.partitions) != 0: + marshalled.extend( + [header("Partition configurations")] + + [marshal_model(part, inline=True) for part in config.partitions] + ) + + return "".join(marshalled) def _parser(config: str) -> SlurmConfig: @@ -60,13 +115,11 @@ def _parser(config: str) -> SlurmConfig: """ slurm_conf = {} nodes = {} - down_nodes = [] frontend_nodes = {} + down_nodes = [] node_sets = {} partitions = {} - # import pdb - # pdb.set_trace() config = clean(deque(config.splitlines())) while config: line = config.popleft() @@ -86,17 +139,17 @@ def _parser(config: str) -> SlurmConfig: # rules for that specific data model and enter its parsed information # into the appropriate pocket. elif line.startswith("NodeName"): - _parse_node(line, pocket=nodes) - elif line.startswith("DownNodes"): - _parse_down_node(line, pocket=down_nodes) + parse_model(line, pocket=nodes, model=Node) elif line.startswith("FrontendNode"): - _parse_frontend(line, pocket=frontend_nodes) + parse_model(line, pocket=frontend_nodes, model=FrontendNode) + elif line.startswith("DownNodes"): + parse_model(line, pocket=down_nodes, model=DownNodes) elif line.startswith("NodeSet"): - _parse_node_set(line, pocket=node_sets) + parse_model(line, pocket=node_sets, model=NodeSet) elif line.startswith("PartitionName"): - _parse_partition(line, pocket=partitions) + parse_model(line, pocket=partitions, model=Partition) else: - _parse_slurm(line, pocket=slurm_conf) + parse_model(line, pocket=slurm_conf, model=SlurmConfig) return SlurmConfig( **slurm_conf, diff --git a/slurmutils/models/_model.py b/slurmutils/models/_model.py index dea92ac..9006bad 100644 --- a/slurmutils/models/_model.py +++ b/slurmutils/models/_model.py @@ -225,6 +225,24 @@ def __repr__(self): f"({', '.join(f'{k}={v}' for k, v in self._register.items())})" ) + @property + @abstractmethod + def _primary_key(self) -> Optional[str]: + """Primary key for data model. + + A primary key is required for data models that have a unique identifier + to preserve the integrity of the Slurm configuration syntax. For example, + for compute nodes, the primary key would be the node name `NodeName`. Node + name can be used nicely for identifying nodes in maps, but it is difficult to + carry along the NodeName key inside the internal register of the class. + + _primary_key is used to track what the Slurm configuration key should be for + unique identifiers. Without this "protected" attribute, we would likely need + to write a custom parser for each data model. The generic model marshaller can + detect this attribute and marshal the model accordingly. + """ + pass + @property @abstractmethod def _callbacks(self) -> MappingProxyType: diff --git a/slurmutils/models/slurm.py b/slurmutils/models/slurm.py index 66648c3..8face14 100644 --- a/slurmutils/models/slurm.py +++ b/slurmutils/models/slurm.py @@ -49,6 +49,7 @@ def __init__(self, **kwargs): super().__init__() self._register.update({kwargs.pop("NodeName"): {**kwargs}}) + _primary_key = "NodeName" _callbacks = MappingProxyType( { "cpu_spec_list": CommaSeparatorCallback, @@ -90,6 +91,7 @@ class DownNodes(BaseModel): the slurm.conf manpage. `man slurm.conf.5` """ + _primary_key = None _callbacks = MappingProxyType( { "down_nodes": CommaSeparatorCallback, @@ -113,6 +115,7 @@ def __init__(self, **kwargs): super().__init__() self._register.update({kwargs.pop("FrontendName"): {**kwargs}}) + _primary_key = "FrontendName" _callbacks = MappingProxyType( { "allow_groups": CommaSeparatorCallback, @@ -145,6 +148,7 @@ def __init__(self, **kwargs): super().__init__() self._register.update({kwargs.pop("NodeSet"): {**kwargs}}) + _primary_key = "NodeSet" _callbacks = MappingProxyType({"nodes": CommaSeparatorCallback}) node_set = property(*primary_key_descriptors()) @@ -163,6 +167,7 @@ def __init__(self, **kwargs): super().__init__() self._register.update({kwargs.pop("PartitionName"): {**kwargs}}) + _primary_key = "PartitionName" _callbacks = MappingProxyType( { "alloc_nodes": CommaSeparatorCallback, @@ -457,6 +462,7 @@ class SlurmConfig(BaseModel): the slurm.conf manpage. `man slurm.conf.5` """ + _primary_key = None _callbacks = MappingProxyType( { "acct_storage_external_host": CommaSeparatorCallback, From 3f4ae84711668c3f0ad90629d5f9df4b0072a1b4 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Tue, 30 Jan 2024 14:36:42 -0500 Subject: [PATCH 09/19] feat: Add marshaller for slurmdbd.conf Signed-off-by: Jason C. Nucciarone --- slurmutils/editors/slurmdbdconfig.py | 74 ++++++++++++++++++++++++++++ slurmutils/models/slurmdbd.py | 24 ++++++++- 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/slurmutils/editors/slurmdbdconfig.py b/slurmutils/editors/slurmdbdconfig.py index 82e43c4..2eba5d4 100644 --- a/slurmutils/editors/slurmdbdconfig.py +++ b/slurmutils/editors/slurmdbdconfig.py @@ -13,3 +13,77 @@ # along with this program. If not, see . """Edit slurmdbd.conf files.""" + +__all__ = ["dump", "dumps", "load", "loads", "edit"] + +import functools +import os +from collections import deque +from contextlib import contextmanager +from datetime import datetime +from typing import Union + +from slurmutils.models import SlurmdbdConfig + +from ._editor import ( + clean, + dump_base, + dumps_base, + header, + load_base, + loads_base, + marshal_model, + parse_model, +) + + +def _marshaller(config: SlurmdbdConfig) -> str: + """Marshal Python object into slurmdbd.conf configuration file. + + Args: + config: `SlurmdbdConfig` object to convert to slurmdbd.conf configuration file. + """ + marshalled = [header(f"`slurmdbd.conf` file generated at {datetime.now()} by slurmutils.")] + marshalled.extend(marshal_model(config)) + + return "".join(marshalled) + + +def _parser(config: str) -> SlurmdbdConfig: + """Parse slurmdbd.conf configuration file into Python object. + + Args: + config: Content of slurmdbd.conf configuration file. + """ + slurmdbd_conf = {} + + config = clean(deque(config.splitlines())) + while config: + line = config.popleft() + parse_model(line, pocket=slurmdbd_conf, model=SlurmdbdConfig) + + return SlurmdbdConfig(**slurmdbd_conf) + + +dump = functools.partial(dump_base, marshaller=_marshaller) +dumps = functools.partial(dumps_base, marshaller=_marshaller) +load = functools.partial(load_base, parser=_parser) +loads = functools.partial(loads_base, parser=_parser) + + +@contextmanager +def edit(file: Union[str, os.PathLike]) -> SlurmdbdConfig: + """Edit a slurmdbd.conf file. + + Args: + file: Path to slurmdbd.conf file to edit. If slurmdbd.conf does + not exist at the specified file path, it will be created. + """ + if os.path.exists(file): + # Create an empty SlurmConfig that can be populated. + config = SlurmdbdConfig() + else: + config = load(file=file) + + yield config + dump(content=config, file=file) diff --git a/slurmutils/models/slurmdbd.py b/slurmutils/models/slurmdbd.py index d42b962..316d3ba 100644 --- a/slurmutils/models/slurmdbd.py +++ b/slurmutils/models/slurmdbd.py @@ -14,7 +14,15 @@ """Data models for the slurmdbd daemon.""" -from ._model import BaseModel, base_descriptors +from types import MappingProxyType + +from ._model import ( + BaseModel, + ColonSeparatorCallback, + CommaSeparatorCallback, + SlurmDictCallback, + base_descriptors, +) class SlurmdbdConfig(BaseModel): @@ -24,6 +32,20 @@ class SlurmdbdConfig(BaseModel): the slurmdbd.conf manpage. `man slurmdbd.conf.5` """ + _primary_key = None + _callbacks = MappingProxyType( + { + "auth_alt_types": CommaSeparatorCallback, + "auth_alt_parameters": SlurmDictCallback, + "communication_parameters": CommaSeparatorCallback, + "debug_flags": CommaSeparatorCallback, + "parameters": CommaSeparatorCallback, + "plugin_dir": ColonSeparatorCallback, + "private_data": CommaSeparatorCallback, + "storage_parameters": SlurmDictCallback, + } + ) + archive_dir = property(*base_descriptors("ArchiveDir")) archive_events = property(*base_descriptors("ArchiveEvents")) archive_jobs = property(*base_descriptors("ArchiveJobs")) From bc5e2a56337d0ad88c9ecffe30e200f43e2903db Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Tue, 30 Jan 2024 17:54:08 -0500 Subject: [PATCH 10/19] feat: Add unit tests for slurmconfig module Signed-off-by: Jason C. Nucciarone --- tests/unit/editors/test_slurmconfig.py | 146 +++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 tests/unit/editors/test_slurmconfig.py diff --git a/tests/unit/editors/test_slurmconfig.py b/tests/unit/editors/test_slurmconfig.py new file mode 100644 index 0000000..6c90d58 --- /dev/null +++ b/tests/unit/editors/test_slurmconfig.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +# Copyright 2024 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +"""Unit tests for the slurm.conf editor.""" + +import unittest +from pathlib import Path + +from slurmutils.editors import slurmconfig + +example_slurm_conf = """# +# `slurm.conf` file generated at 2024-01-30 17:18:36.171652 by slurmutils. +# +SlurmctldHost=juju-c9fc6f-0(10.152.28.20) +SlurmctldHost=juju-c9fc6f-1(10.152.28.100) + +ClusterName=charmed-hpc +AuthType=auth/munge +Epilog=/usr/local/slurm/epilog +Prolog=/usr/local/slurm/prolog +FirstJobId=65536 +InactiveLimit=120 +JobCompType=jobcomp/filetxt +JobCompLoc=/var/log/slurm/jobcomp +KillWait=30 +MaxJobCount=10000 +MinJobAge=3600 +PluginDir=/usr/local/lib:/usr/local/slurm/lib +ReturnToService=0 +SchedulerType=sched/backfill +SlurmctldLogFile=/var/log/slurm/slurmctld.log +SlurmdLogFile=/var/log/slurm/slurmd.log +SlurmctldPort=7002 +SlurmdPort=7003 +SlurmdSpoolDir=/var/spool/slurmd.spool +StateSaveLocation=/var/spool/slurm.state +SwitchType=switch/none +TmpFS=/tmp +WaitTime=30 + +# +# Node configurations +# +NodeName=juju-c9fc6f-2 NodeAddr=10.152.28.48 CPUs=1 RealMemory=1000 TmpDisk=10000 +NodeName=juju-c9fc6f-3 NodeAddr=10.152.28.49 CPUs=1 RealMemory=1000 TmpDisk=10000 +NodeName=juju-c9fc6f-4 NodeAddr=10.152.28.50 CPUs=1 RealMemory=1000 TmpDisk=10000 +NodeName=juju-c9fc6f-5 NodeAddr=10.152.28.51 CPUs=1 RealMemory=1000 TmpDisk=10000 + +# +# Down node configurations +# +DownNodes=juju-c9fc6f-5 State=DOWN Reason="Maintenance Mode" + +# +# Partition configurations +# +PartitionName=DEFAULT MaxTime=30 MaxNodes=10 State=UP +PartitionName=batch Nodes=juju-c9fc6f-2,juju-c9fc6f-3,juju-c9fc6f-4,juju-c9fc6f-5 MinNodes=4 MaxTime=120 AllowGroups=admin +""" + + +class TestSlurmConfigEditor(unittest.TestCase): + """Unit tests for slurm.conf file editor.""" + + def setUp(self) -> None: + Path("slurm.conf").write_text(example_slurm_conf) + + def test_loads(self) -> None: + """Test `loads` method of slurmconfig module.""" + config = slurmconfig.loads(example_slurm_conf) + self.assertListEqual( + config.slurmctld_host, ["juju-c9fc6f-0(10.152.28.20)", "juju-c9fc6f-1(10.152.28.100)"] + ) + self.assertEqual(config.slurmd_spool_dir, "/var/spool/slurmd.spool") + self.assertEqual(config.scheduler_type, "sched/backfill") + + nodes = config.nodes + for node in nodes: + self.assertIn( + node.node_name, + {"juju-c9fc6f-2", "juju-c9fc6f-3", "juju-c9fc6f-4", "juju-c9fc6f-5"}, + ) + self.assertIn( + node.node_addr, {"10.152.28.48", "10.152.28.49", "10.152.28.50", "10.152.28.51"} + ) + self.assertEqual(node.cpus, "1") + self.assertEqual(node.real_memory, "1000") + self.assertEqual(node.tmp_disk, "10000") + + down_nodes = config.down_nodes + for entry in down_nodes: + self.assertEqual(entry.down_nodes[0], "juju-c9fc6f-5") + self.assertEqual(entry.state, "DOWN") + self.assertEqual(entry.reason, "Maintenance Mode") + + partitions = config.partitions + for part in partitions: + self.assertIn(part.partition_name, {"DEFAULT", "batch"}) + + batch = partitions["batch"] + self.assertListEqual( + batch.nodes, ["juju-c9fc6f-2", "juju-c9fc6f-3", "juju-c9fc6f-4", "juju-c9fc6f-5"] + ) + + def test_dumps(self) -> None: + """Test `dumps` method of slurmconfig module.""" + config = slurmconfig.loads(example_slurm_conf) + + self.assertNotEqual(slurmconfig.dumps(config), example_slurm_conf) + + def test_edit(self) -> None: + """Test if SlurmConf can successfully edit the example configuration file.""" + with slurmconfig.edit("slurm.conf") as config: + del config.inactive_limit + config.max_job_count = 20000 + config.proctrack_type = "proctrack/linuxproc" + config.plugin_dir.append("/snap/slurm/current/plugins") + node = config.nodes["juju-c9fc6f-2"] + del config.nodes["juju-c9fc6f-2"] + node.node_name = "batch-0" + config.nodes[node.node_name] = node + + config = slurmconfig.load("slurm.conf") + self.assertIsNone(config.inactive_limit) + self.assertEqual(config.max_job_count, "20000") + self.assertEqual(config.proctrack_type, "proctrack/linuxproc") + self.assertListEqual( + config.plugin_dir, + ["/usr/local/lib", "/usr/local/slurm/lib", "/snap/slurm/current/plugins"], + ) + self.assertEqual(config.nodes["batch-0"].node_addr, "10.152.28.48") + + def tearDown(self): + Path("slurm.conf").unlink() From 3b0afaaa2c9199912b6cef53bf12bc2a6d987246 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Tue, 30 Jan 2024 17:55:28 -0500 Subject: [PATCH 11/19] feat!: Define __iter__ for DownNodesList type BREAKING CHANGE: Overrides default __iter__ behavior of UserList. Signed-off-by: Jason C. Nucciarone --- slurmutils/models/slurm.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/slurmutils/models/slurm.py b/slurmutils/models/slurm.py index 8face14..fed7fc8 100644 --- a/slurmutils/models/slurm.py +++ b/slurmutils/models/slurm.py @@ -305,6 +305,9 @@ def __setitem__(self, i: int, value: DownNodes): def __contains__(self, value): return value.dict() in self.data + def __iter__(self): + return iter([DownNodes(**data) for data in self.data]) + def __add__(self, other): if isinstance(other, DownNodesList): return self.__class__(self.data + other.data) From 95af5bd2f8ce7c8d4e865a7a772f6f5effa76991 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Tue, 30 Jan 2024 18:00:32 -0500 Subject: [PATCH 12/19] bugfix: Fix behavior of edit contextmanager The conditional for determining whether to create an empty Config object or to load in a pre-existing configuration was busted. If the pre-existing config existed, the context manager would return an empty Config object which is not the desired behavior. By adding `not` in front of the path check, the conditional now evaluates properly. Loads in the config if the config file exists; returns an empty Config object if it doesn't. Also fixed a grammar error in the slurmconfig `_marshaller` function docstring. Signed-off-by: Jason C. Nucciarone --- slurmutils/editors/slurmconfig.py | 4 ++-- slurmutils/editors/slurmdbdconfig.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/slurmutils/editors/slurmconfig.py b/slurmutils/editors/slurmconfig.py index e438080..a9df6cc 100644 --- a/slurmutils/editors/slurmconfig.py +++ b/slurmutils/editors/slurmconfig.py @@ -39,7 +39,7 @@ def _marshaller(config: SlurmConfig) -> str: - """Marshall Python object into slurm.conf configuration file. + """Marshal Python object into slurm.conf configuration file. Args: config: `SlurmConfig` object to convert to slurm.conf configuration file. @@ -175,7 +175,7 @@ def edit(file: Union[str, os.PathLike]) -> SlurmConfig: file: Path to slurm.conf file to edit. If slurm.conf does not exist at the specified file path, it will be created. """ - if os.path.exists(file): + if not os.path.exists(file): # Create an empty SlurmConfig that can be populated. config = SlurmConfig() else: diff --git a/slurmutils/editors/slurmdbdconfig.py b/slurmutils/editors/slurmdbdconfig.py index 2eba5d4..0c5a9b7 100644 --- a/slurmutils/editors/slurmdbdconfig.py +++ b/slurmutils/editors/slurmdbdconfig.py @@ -79,7 +79,7 @@ def edit(file: Union[str, os.PathLike]) -> SlurmdbdConfig: file: Path to slurmdbd.conf file to edit. If slurmdbd.conf does not exist at the specified file path, it will be created. """ - if os.path.exists(file): + if not os.path.exists(file): # Create an empty SlurmConfig that can be populated. config = SlurmdbdConfig() else: From f3e293576e780b5955936a846c1a43a70f54502f Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Tue, 30 Jan 2024 18:00:54 -0500 Subject: [PATCH 13/19] chore(meta): Remove old unit tests Signed-off-by: Jason C. Nucciarone --- tests/unit/test_slurmconf.py | 102 ----------------------------------- 1 file changed, 102 deletions(-) delete mode 100644 tests/unit/test_slurmconf.py diff --git a/tests/unit/test_slurmconf.py b/tests/unit/test_slurmconf.py deleted file mode 100644 index 6bda475..0000000 --- a/tests/unit/test_slurmconf.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Canonical Ltd. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License version 3 as published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . - -"""Test SLURM configuration API.""" - -import unittest -from pathlib import Path - -from slurmconf import SlurmConf - -example_conf = """ -# -# /etc/slurm/slurm.conf for juju-c9fc6f-[0-5].canonical.com -# Author: Jason C. Nucciarone -# Date: 17/07/2023 -# -SlurmctldHost=juju-c9fc6f-0(10.152.28.20) # Primary server -SlurmctldHost=juju-c9fc6f-1(10.152.28.100) # Backup server -# -ClusterName=charmed-hpc -AuthType=auth/munge -Epilog=/usr/local/slurm/epilog -Prolog=/usr/local/slurm/prolog -FirstJobId=65536 -InactiveLimit=120 -JobCompType=jobcomp/filetxt -JobCompLoc=/var/log/slurm/jobcomp -KillWait=30 -MaxJobCount=10000 -MinJobAge=3600 -PluginDir=/usr/local/lib:/usr/local/slurm/lib -ReturnToService=0 -SchedulerType=sched/backfill -SlurmctldLogFile=/var/log/slurm/slurmctld.log -SlurmdLogFile=/var/log/slurm/slurmd.log -SlurmctldPort=7002 -SlurmdPort=7003 -SlurmdSpoolDir=/var/spool/slurmd.spool -StateSaveLocation=/var/spool/slurm.state -SwitchType=switch/none -TmpFS=/tmp -WaitTime=30 -# -# Node Configurations -# -NodeName=juju-c9fc6f-2 NodeAddr=10.152.28.48 CPUs=1 RealMemory=1000 TmpDisk=10000 -NodeName=juju-c9fc6f-3 NodeAddr=10.152.28.49 CPUs=1 RealMemory=1000 TmpDisk=10000 -NodeName=juju-c9fc6f-4 NodeAddr=10.152.28.50 CPUs=1 RealMemory=1000 TmpDisk=10000 -NodeName=juju-c9fc6f-5 NodeAddr=10.152.28.51 CPUs=1 RealMemory=1000 TmpDisk=10000 -DownNodes=juju-c9fc6f-5 State=DOWN Reason="Maintenance Mode" -# -# Partition Configurations -# -PartitionName=DEFAULT MaxTime=30 MaxNodes=10 State=UP -PartitionName=batch Nodes=juju-c9fc6f-2,juju-c9fc6f-3,juju-c9fc6f-4,juju-c9fc6f-5 MinNodes=4 MaxTime=120 AllowGroups=admin -""" - - -class TestSlurmConf(unittest.TestCase): - """Unit tests for slurm.conf file editor.""" - - def setUp(self) -> None: - Path("slurm.conf").write_text(example_conf.strip()) - - def test_load(self) -> None: - """Test that SlurmConf can successfully load/parse example configuration file.""" - with SlurmConf("slurm.conf") as conf: - self.assertNotEqual(conf.comments, []) - self.assertNotEqual(conf.nodes, {}) - self.assertNotEqual(conf.down_nodes, []) - self.assertEqual(conf.frontend_nodes, {}) - self.assertEqual(conf.nodesets, {}) - self.assertNotEqual(conf.partitions, {}) - - def test_edit(self) -> None: - """Test if SlurmConf can successfully edit the example configuration file.""" - with SlurmConf("slurm.conf") as conf: - del conf.inactive_limit - conf.max_job_count = 20000 - conf.proctrack_type = "proctrack/linuxproc" - - conf = SlurmConf("slurm.conf") - conf.load() - self.assertIsNone(conf.inactive_limit) - self.assertEqual(conf.max_job_count, "20000") - self.assertEqual(conf.proctrack_type, "proctrack/linuxproc") - self.assertListEqual(conf.plugin_dir, ["/usr/local/lib", "/usr/local/slurm/lib"]) - - def tearDown(self) -> None: - Path("slurm.conf").unlink() From 3d1025687dfe0a5605146b1d88d736f07b0b1d00 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Tue, 30 Jan 2024 18:01:12 -0500 Subject: [PATCH 14/19] chore(lint): Fix spelling error in function docstring Signed-off-by: Jason C. Nucciarone --- slurmutils/editors/_editor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slurmutils/editors/_editor.py b/slurmutils/editors/_editor.py index 662640c..b8871c1 100644 --- a/slurmutils/editors/_editor.py +++ b/slurmutils/editors/_editor.py @@ -220,7 +220,7 @@ def marshal_model( _logger.debug("Ignoring option %s. Option is present in ignore set %s", option, ignore) if inline: - # Whitespace is the seperator in Slurm configuration syntax. + # Whitespace is the separator in Slurm configuration syntax. marshalled = " ".join(marshalled) + "\n" else: # Append newline character so that each configuration is on its own line. From 72cf0e6e96f931f54664a43184787811e808361f Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Wed, 31 Jan 2024 09:54:44 -0500 Subject: [PATCH 15/19] docs: Docstring and comment clean up in slurmconfig unit tests Signed-off-by: Jason C. Nucciarone --- tests/unit/editors/test_slurmconfig.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/unit/editors/test_slurmconfig.py b/tests/unit/editors/test_slurmconfig.py index 6c90d58..359fefd 100644 --- a/tests/unit/editors/test_slurmconfig.py +++ b/tests/unit/editors/test_slurmconfig.py @@ -78,7 +78,7 @@ def setUp(self) -> None: Path("slurm.conf").write_text(example_slurm_conf) def test_loads(self) -> None: - """Test `loads` method of slurmconfig module.""" + """Test `loads` method of the slurmconfig module.""" config = slurmconfig.loads(example_slurm_conf) self.assertListEqual( config.slurmctld_host, ["juju-c9fc6f-0(10.152.28.20)", "juju-c9fc6f-1(10.152.28.100)"] @@ -115,13 +115,14 @@ def test_loads(self) -> None: ) def test_dumps(self) -> None: - """Test `dumps` method of slurmconfig module.""" + """Test `dumps` method of the slurmconfig module.""" config = slurmconfig.loads(example_slurm_conf) - + # The new config and old config should not be equal since the + # timestamps in the header will be different. self.assertNotEqual(slurmconfig.dumps(config), example_slurm_conf) def test_edit(self) -> None: - """Test if SlurmConf can successfully edit the example configuration file.""" + """Test `edit` context manager from the slurmconfig module.""" with slurmconfig.edit("slurm.conf") as config: del config.inactive_limit config.max_job_count = 20000 From d750602a2e1bb038486e31a9e597234a2e6b3553 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Wed, 31 Jan 2024 09:59:13 -0500 Subject: [PATCH 16/19] feat: Add unit tests for slurmdbdconfig module Changes: - Changed callback for communication_parameters to SlurmDict. Originally used CommaSeparator, but remembered that there are some possible key=value options as well. Signed-off-by: Jason C. Nucciarone --- slurmutils/models/slurmdbd.py | 2 +- tests/unit/editors/test_slurmdbdconfig.py | 98 +++++++++++++++++++++++ 2 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 tests/unit/editors/test_slurmdbdconfig.py diff --git a/slurmutils/models/slurmdbd.py b/slurmutils/models/slurmdbd.py index 316d3ba..bf3b4e2 100644 --- a/slurmutils/models/slurmdbd.py +++ b/slurmutils/models/slurmdbd.py @@ -37,7 +37,7 @@ class SlurmdbdConfig(BaseModel): { "auth_alt_types": CommaSeparatorCallback, "auth_alt_parameters": SlurmDictCallback, - "communication_parameters": CommaSeparatorCallback, + "communication_parameters": SlurmDictCallback, "debug_flags": CommaSeparatorCallback, "parameters": CommaSeparatorCallback, "plugin_dir": ColonSeparatorCallback, diff --git a/tests/unit/editors/test_slurmdbdconfig.py b/tests/unit/editors/test_slurmdbdconfig.py new file mode 100644 index 0000000..4bbf1be --- /dev/null +++ b/tests/unit/editors/test_slurmdbdconfig.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# Copyright 2024 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +"""Unit tests for the slurmdbd.conf editor.""" + +import unittest +from pathlib import Path + +from slurmutils.editors import slurmdbdconfig + +example_slurmdbd_conf = """# +# `slurmdbd.conf` file generated at 2024-01-30 17:18:36.171652 by slurmutils. +# +ArchiveEvents=yes +ArchiveJobs=yes +ArchiveResvs=yes +ArchiveSteps=no +ArchiveTXN=no +ArchiveUsage=no +ArchiveScript=/usr/sbin/slurm.dbd.archive +AuthInfo=/var/run/munge/munge.socket.2 +AuthType=auth/munge +AuthAltTypes=auth/jwt +AuthAltParameters=jwt_key=16549684561684@ +DbdHost=slurmdbd-0 +DbdBackupHost=slurmdbd-1 +DebugLevel=info +PluginDir=/all/these/cool/plugins +PurgeEventAfter=1month +PurgeJobAfter=12month +PurgeResvAfter=1month +PurgeStepAfter=1month +PurgeSuspendAfter=1month +PurgeTXNAfter=12month +PurgeUsageAfter=24month +LogFile=/var/log/slurmdbd.log +PidFile=/var/run/slurmdbd.pid +SlurmUser=slurm +StoragePass=supersecretpasswd +StorageType=accounting_storage/mysql +StorageUser=slurm +StorageHost=127.0.0.1 +StoragePort=3306 +StorageLoc=slurm_acct_db +""" + + +class TestSlurmdbdConfigEditor(unittest.TestCase): + """Unit tests for the slurmdbd.conf file editor.""" + + def setUp(self) -> None: + Path("slurmdbd.conf").write_text(example_slurmdbd_conf) + + def test_loads(self) -> None: + """Test `loads` method of the slurmdbdconfig module.""" + config = slurmdbdconfig.loads(example_slurmdbd_conf) + self.assertListEqual(config.plugin_dir, ["/all/these/cool/plugins"]) + self.assertDictEqual(config.auth_alt_parameters, {"jwt_key": "16549684561684@"}) + self.assertEqual(config.slurm_user, "slurm") + self.assertEqual(config.log_file, "/var/log/slurmdbd.log") + + def test_dumps(self) -> None: + """Test `dumps` method of the slurmdbdconfig module.""" + config = slurmdbdconfig.loads(example_slurmdbd_conf) + # The new config and old config should not be equal since the + # timestamps in the header will be different. + self.assertNotEqual(slurmdbdconfig.dumps(config), example_slurmdbd_conf) + + def test_edit(self) -> None: + """Test `edit` context manager from the slurmdbdconfig module.""" + with slurmdbdconfig.edit("slurmdbd.conf") as config: + config.archive_usage = "yes" + config.log_file = "/var/spool/slurmdbd.log" + config.debug_flags = ["DB_EVENT", "DB_JOB", "DB_USAGE"] + del config.auth_alt_types + del config.auth_alt_parameters + + config = slurmdbdconfig.load("slurmdbd.conf") + self.assertEqual(config.archive_usage, "yes") + self.assertEqual(config.log_file, "/var/spool/slurmdbd.log") + self.assertListEqual(config.debug_flags, ["DB_EVENT", "DB_JOB", "DB_USAGE"]) + self.assertIsNone(config.auth_alt_types) + self.assertIsNone(config.auth_alt_parameters) + + def tearDown(self) -> None: + Path("slurmdbd.conf").unlink() From 04e81bf2fa0e468fff274a4cdbbabd5c8baacdf2 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Wed, 31 Jan 2024 11:31:02 -0500 Subject: [PATCH 17/19] docs: Update README Signed-off-by: Jason C. Nucciarone --- README.md | 104 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 71 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 642fded..c12abf7 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,29 @@ -

- slurmutils -

+
-

- Utilities and APIs for interacting with the SLURM workload manager. -

+# slurmutils + +Utilities and APIs for interfacing with the Slurm workload manager. + +[![Matrix](https://img.shields.io/matrix/ubuntu-hpc%3Amatrix.org?logo=matrix&label=ubuntu-hpc)](https://matrix.to/#/#ubuntu-hpc:matrix.org) + +
## Features -* `slurmconf`: An API for performing CRUD operations on the SLURM configuration file _slurm.conf_ +`slurmutils` is a collection of various utilities and APIs to make it easier +for you and your friends to interface with the Slurm workload manager, especially if you +are orchestrating deployments of new and current Slurm clusters. Gone are the days of +seething over incomplete Jinja2 templates. Current utilities and APIs shipped in the +`slurmutils` package include: + +#### `from slurmutils.editors import ...` + +* `slurmconfig`: An editor _slurm.conf_ and _Include_ files. +* `slurmdbdconfig`: An editor for _slurmdbd.conf_ files. ## Installation -#### Option 1: PyPI +#### Option 1: Install from PyPI ```shell $ python3 -m pip install slurmutils @@ -20,45 +31,72 @@ $ python3 -m pip install slurmutils #### Option 2: Install from source +We use the [Poetry](https://python-poetry.org) packaging and dependency manager to +manage this project. It must be installed on your system if installing `slurmutils` +from source. + ```shell $ git clone https://github.com/canonical/slurmutils.git $ cd slurmutils -$ python3 -m pip install . +$ poetry install ``` ## Usage -#### `slurmconf` +### Editors -This module provides an API for performing CRUD operations on the SLURM configuration file _slurm.conf_. -With this module, you can: +#### `slurmconfig` -##### Edit a pre-existing configuration +This module provides an API for editing both _slurm.conf_ and _Include_ files, +and can create new configuration files if they do not exist. Here's some common Slurm +lifecycle management operators you can perform using this editor: + +##### Edit a pre-existing _slurm.conf_ configuration file ```python -from slurmutils.slurmconf import SlurmConf +from slurmutils.editors import slurmconfig -with SlurmConf("/etc/slurm/slurm.conf") as conf: - del conf.inactive_limit - conf.max_job_count = 20000 - conf.proctrack_type = "proctrack/linuxproc" +# Open, edit, and save the slurm.conf file located at _/etc/slurm/slurm.conf_. +with slurmconfig.edit("/etc/slurm/slurm.conf") as config: + del config.inactive_limit + config.max_job_count = 20000 + config.proctrack_type = "proctrack/linuxproc" ``` -##### Add new nodes - -```python3 -from slurmutils.slurmconf import Node, SlurmConf - -with SlurmConf("/etc/slurm/slurm.conf") as conf: - node_name = "test-node" - node_conf = { - "NodeName": node_name, - "NodeAddr": "12.34.56.78", - "CPUs": 1, - "RealMemory": 1000, - "TmpDisk": 10000, - } - conf.nodes.update({node_name: Node(**node_conf)}) +##### Add a new node to the _slurm.conf_ file + +```python +from slurmutils.editors import slurmconfig +from slurmutils.models import Node + +with slurmconfig.edit("/etc/slurm/slurm.conf") as config: + node = Node( + NodeName="batch-[0-25]", + NodeAddr="12.34.56.78", + CPUs=1, + RealMemory=1000, + TmpDisk=10000, + ) + config.nodes[node.node_name] = node +``` + +#### `slurmdbdconfig` + +This module provides and API for editing _slurmdbd.conf_ files, and can create new +_slurmdbd.conf_ files if they do not exist. Here's some operations you can perform +on the _slurmdbd.conf_ file using this editor: + +##### Edit a pre-existing _slurmdbd.conf_ configuration file + +```python +from slurmutils.editors import slurmdbdconfig + +with slurmdbdconfig.edit("/etc/slurm/slurmdbd.conf") as config: + config.archive_usage = "yes" + config.log_file = "/var/spool/slurmdbd.log" + config.debug_flags = ["DB_EVENT", "DB_JOB", "DB_USAGE"] + del config.auth_alt_types + del config.auth_alt_parameters ``` ## Project & Community From 9523952355bbf64ca8f1facc8f4e344b6225df3a Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Wed, 31 Jan 2024 11:35:57 -0500 Subject: [PATCH 18/19] feat!: Replace setup.py with poetry BREAKING CHANGE: Since poetry is now the primary packaging and dependency manager, no longer using setuptools when installing from source. Folks installing from source will now need to install poetry if going to install slurmutils from source. Pulling packages from PyPI will remain the same however. Signed-off-by: Jason C. Nucciarone --- pyproject.toml | 49 ++++++++++++++++++++++++++++++++----------------- setup.py | 47 ----------------------------------------------- 2 files changed, 32 insertions(+), 64 deletions(-) delete mode 100644 setup.py diff --git a/pyproject.toml b/pyproject.toml index bbde4d1..b8a2a3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,32 +1,47 @@ -# Copyright 2023 Canonical Ltd. +# Copyright 2024 Canonical Ltd. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. # -# http://www.apache.org/licenses/LICENSE-2.0 +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" -# Poetry [tool.poetry] name = "slurmutils" -version = "0.1.0" -description = "Utilities and APIs for interfacing with the Slurm workload manager" +version = "0.2.0" +description = "Utilities and APIs for interfacing with the Slurm workload manager." +repository = "https://github.com/canonical/slurmutils" authors = ["Jason C. Nucciarone "] -license = "LGPL-3.0" +maintainers = ["Jason C. Nucciarone "] +license = "LGPL-3.0-only" readme = "README.md" +keywords = ["HPC", "administration", "orchestration", "utility"] +classifiers=[ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", +] [tool.poetry.dependencies] python = ">=3.8" -[build-system] -requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" +[tool.poetry.urls] +"Bug Tracker" = "https://github.com/canonical/slurmutils/issues" # Testing tools configuration [tool.coverage.run] diff --git a/setup.py b/setup.py deleted file mode 100644 index 83612e1..0000000 --- a/setup.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Canonical Ltd. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License version 3 as published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . - -"""setup.py for slurmutils package.""" - -from os.path import exists - -from setuptools import setup - -setup( - name="slurmutils", - version="0.1.0", - author="Jason C. Nucciarone", - author_email="jason.nucciarone@canonical.com", - license="LGPL-3.0", - url="https://github.com/canonical/slurmutils", - description="Utilities and APIs for interacting with the SLURM workload manager", - long_description=open("README.md").read() if exists("README.md") else "", - long_description_content_type="text/markdown", - python_requires=">=3.8", - classifiers=[ - "Development Status :: 3 - Alpha", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)", - "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - ], - packages=[ - "slurmutils", - "slurmutils.editors" - ] -) From c267301e1232dcfba57e5571fd5fc2d827f2ebf0 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Wed, 31 Jan 2024 12:17:57 -0500 Subject: [PATCH 19/19] feat: Update tox.ini to use poetry instead of twine for publishing Removes the dependency on the plain-text .pypirc file that can introduce potential security vulnerabilities. Signed-off-by: Jason C. Nucciarone --- tox.ini | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/tox.ini b/tox.ini index 3bd5826..c2dcebb 100644 --- a/tox.ini +++ b/tox.ini @@ -1,16 +1,16 @@ -# Copyright 2023 Canonical Ltd. +# Copyright 2024 Canonical Ltd. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. # -# http://www.apache.org/licenses/LICENSE-2.0 +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . [tox] skipsdist=True @@ -60,14 +60,15 @@ commands = coverage report [testenv:publish] -description = Publish slurmtools to PyPI. +description = Publish slurmutils to PyPI using poetry. allowlist_externals = /usr/bin/rm + /usr/bin/poetry deps = twine setuptools wheel commands = rm -rf {toxinidir}/dist - python setup.py sdist bdist_wheel - twine upload {toxinidir}/dist/* + poetry build + poetry publish