From acc645895f1890c3515bf64a5bca5c3e534d9d3e Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Wed, 18 Sep 2024 15:32:29 -0400 Subject: [PATCH 1/4] fix: properly iterate over option set when construction error message Signed-off-by: Jason C. Nucciarone --- slurmutils/editors/cgroupconfig.py | 0 slurmutils/editors/editor.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 slurmutils/editors/cgroupconfig.py diff --git a/slurmutils/editors/cgroupconfig.py b/slurmutils/editors/cgroupconfig.py new file mode 100644 index 0000000..e69de29 diff --git a/slurmutils/editors/editor.py b/slurmutils/editors/editor.py index c416fbb..6f059ea 100644 --- a/slurmutils/editors/editor.py +++ b/slurmutils/editors/editor.py @@ -49,7 +49,7 @@ def parse_line(options, line: str) -> Dict[str, Any]: raise EditorError( ( f"unable to parse configuration option {k}. " - + f"valid configuration options are {[option.name for option in options]}" + + f"valid configuration options are {[opt for opt in options.keys()]}" ) ) From 2a2b80b2a25b9669d5befa2317f51333f2e1085d Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Wed, 18 Sep 2024 15:33:33 -0400 Subject: [PATCH 2/4] feat(cgroup): add cgroup.conf editor and data model Signed-off-by: Jason C. Nucciarone --- slurmutils/editors/__init__.py | 1 + slurmutils/editors/cgroupconfig.py | 105 +++++++++++++++++++++++++++++ slurmutils/models/__init__.py | 1 + slurmutils/models/cgroup.py | 29 ++++++++ slurmutils/models/option.py | 24 +++++++ 5 files changed, 160 insertions(+) create mode 100644 slurmutils/models/cgroup.py diff --git a/slurmutils/editors/__init__.py b/slurmutils/editors/__init__.py index 4bf472e..0edb732 100644 --- a/slurmutils/editors/__init__.py +++ b/slurmutils/editors/__init__.py @@ -14,5 +14,6 @@ """Editors for Slurm workload manager configuration files.""" +from . import cgroupconfig as cgroupconfig from . import slurmconfig as slurmconfig from . import slurmdbdconfig as slurmdbdconfig diff --git a/slurmutils/editors/cgroupconfig.py b/slurmutils/editors/cgroupconfig.py index e69de29..5afa30b 100644 --- a/slurmutils/editors/cgroupconfig.py +++ b/slurmutils/editors/cgroupconfig.py @@ -0,0 +1,105 @@ +# Copyright 2024 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +"""Edit cgroup.conf files.""" + +__all__ = ["dump", "dumps", "load", "loads", "edit"] + +import logging +import os +from contextlib import contextmanager +from pathlib import Path +from typing import Union + +from ..models import CgroupConfig +from ..models.option import CgroupConfigOptionSet +from .editor import ( + clean, + dumper, + loader, + marshall_content, + parse_line, +) + +_logger = logging.getLogger("slurmutils") + + +@loader +def load(file: Union[str, os.PathLike]) -> CgroupConfig: + """Load `cgroup.conf` data model from cgroup.conf file.""" + return loads(Path(file).read_text()) + + +def loads(content: str) -> CgroupConfig: + """Load `cgroup.conf` data model from string.""" + return _parse(content) + + +@dumper +def dump(config: CgroupConfig, file: Union[str, os.PathLike]) -> None: + """Dump `cgroup.conf` data model into cgroup.conf file.""" + Path(file).write_text(dumps(config)) + + +def dumps(config: CgroupConfig) -> str: + """Dump `cgroup.conf` data model into a string.""" + return _marshall(config) + + +@contextmanager +def edit(file: Union[str, os.PathLike]) -> CgroupConfig: + """Edit a cgroup.conf file. + + Args: + file: Path to cgroup.conf file to edit. If cgroup.conf does + not exist at the specified file path, it will be created. + """ + if not os.path.exists(file): + _logger.warning("file %s not found. creating new empty cgroup.conf configuration", file) + config = CgroupConfig() + else: + config = load(file) + + yield config + dump(config, file) + + +def _parse(content: str) -> CgroupConfig: + """Parse contents of `cgroup.conf`. + + Args: + content: Contents of `cgroup.conf`. + """ + data = {} + lines = content.splitlines() + for index, line in enumerate(lines): + config = clean(line) + if config is None: + _logger.debug("ignoring line %s at index %s in cgroup.conf", line, index) + continue + + data.update(parse_line(CgroupConfigOptionSet, config)) + + return CgroupConfig.from_dict(data) + + +def _marshall(config: CgroupConfig) -> str: + """Marshall `cgroup.conf` data model back into cgroup.conf format. + + Args: + config: `cgroup.conf` data model to marshall. + """ + result = [] + result.extend(marshall_content(CgroupConfigOptionSet, config.dict())) + return "\n".join(result) diff --git a/slurmutils/models/__init__.py b/slurmutils/models/__init__.py index 3ff5c44..1bd6776 100644 --- a/slurmutils/models/__init__.py +++ b/slurmutils/models/__init__.py @@ -14,6 +14,7 @@ """Data models for common Slurm objects.""" +from .cgroup import CgroupConfig as CgroupConfig from .slurm import DownNodes as DownNodes from .slurm import FrontendNode as FrontendNode from .slurm import Node as Node diff --git a/slurmutils/models/cgroup.py b/slurmutils/models/cgroup.py new file mode 100644 index 0000000..ce82b9c --- /dev/null +++ b/slurmutils/models/cgroup.py @@ -0,0 +1,29 @@ +# Copyright 2024 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +"""Data models for `cgroup.conf` configuration file.""" + +from .model import BaseModel, format_key, generate_descriptors +from .option import CgroupConfigOptionSet + + +class CgroupConfig(BaseModel): + """`cgroup.conf` data model.""" + + def __init__(self, **kwargs) -> None: + super().__init__(CgroupConfigOptionSet, **kwargs) + + +for opt in CgroupConfigOptionSet.keys(): + setattr(CgroupConfig, format_key(opt), property(*generate_descriptors(opt))) diff --git a/slurmutils/models/option.py b/slurmutils/models/option.py index a07ae5e..f946174 100644 --- a/slurmutils/models/option.py +++ b/slurmutils/models/option.py @@ -15,6 +15,7 @@ """Configuration options for Slurm data models.""" __all__ = [ + "CgroupConfigOptionSet", "SlurmdbdConfigOptionSet", "SlurmConfigOptionSet", "NodeOptionSet", @@ -47,6 +48,29 @@ def keys(cls) -> Iterable[str]: yield field.name +@dataclass(frozen=True) +class CgroupConfigOptionSet(_OptionSet): + """`cgroup.conf` configuration options.""" + + CgroupMountpoint: Callback = Callback() + CgroupPlugin: Callback = Callback() + SystemdTimeout: Callback = Callback() + IgnoreSystemd: Callback = Callback() + IgnoreSystemdOnFailure: Callback = Callback() + EnableControllers: Callback = Callback() + AllowedRAMSpace: Callback = Callback() + AllowedSwapSpace: Callback = Callback() + ConstrainCores: Callback = Callback() + ConstrainDevices: Callback = Callback() + ConstrainRAMSpace: Callback = Callback() + ConstrainSwapSpace: Callback = Callback() + MaxRAMPercent: Callback = Callback() + MaxSwapPercent: Callback = Callback() + MemorySwappiness: Callback = Callback() + MinRAMSpace: Callback = Callback() + SignalChildrenProcesses: Callback = Callback() + + @dataclass(frozen=True) class SlurmdbdConfigOptionSet(_OptionSet): """`slurmdbd.conf` configuration options.""" From f49bb70e35dc5e172497b0decba272a01cac185b Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Wed, 18 Sep 2024 15:34:00 -0400 Subject: [PATCH 3/4] tests(cgroup): add unit tests for cgroup.conf editor Signed-off-by: Jason C. Nucciarone --- tests/unit/editors/test_cgroupconfig.py | 70 +++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 tests/unit/editors/test_cgroupconfig.py diff --git a/tests/unit/editors/test_cgroupconfig.py b/tests/unit/editors/test_cgroupconfig.py new file mode 100644 index 0000000..1a795fd --- /dev/null +++ b/tests/unit/editors/test_cgroupconfig.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# Copyright 2024 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +"""Unit tests for the cgroup.conf editor.""" + +import unittest +from pathlib import Path + +from slurmutils.editors import cgroupconfig + + +EXAMPLE_CGROUP_CONF = """# +# `cgroup.conf` file generated at 2024-09-18 15:10:44.652017 by slurmutils. +# +ConstrainCores=yes +ConstrainDevices=yes +ConstrainRAMSpace=yes +ConstrainSwapSpace=yes +""" + + +class TestCgroupConfigEditor(unittest.TestCase): + """Unit tests for cgroup.conf file editor.""" + + def setUp(self) -> None: + Path("cgroup.conf").write_text(EXAMPLE_CGROUP_CONF) + + def test_loads(self) -> None: + """Test `loads` method of the cgroupconfig module.""" + config = cgroupconfig.loads(EXAMPLE_CGROUP_CONF) + self.assertEqual(config.constrain_cores, "yes") + self.assertEqual(config.constrain_devices, "yes") + self.assertEqual(config.constrain_ram_space, "yes") + self.assertEqual(config.constrain_swap_space, "yes") + + def test_dumps(self) -> None: + """Test `dumps` method of the cgroupconfig module.""" + config = cgroupconfig.loads(EXAMPLE_CGROUP_CONF) + # The new config and old config should not be equal since the + # timestamps in the header will be different. + self.assertNotEqual(cgroupconfig.dumps(config), EXAMPLE_CGROUP_CONF) + + def test_edit(self) -> None: + """Test `edit` context manager from the cgroupconfig module.""" + with cgroupconfig.edit("cgroup.conf") as config: + config.constrain_cores = "no" + config.constrain_devices = "no" + config.constrain_ram_space = "no" + config.constrain_swap_space = "no" + + config = cgroupconfig.load("cgroup.conf") + self.assertEqual(config.constrain_cores, "no") + self.assertEqual(config.constrain_devices, "no") + self.assertEqual(config.constrain_ram_space, "no") + self.assertEqual(config.constrain_swap_space, "no") + + def tearDown(self) -> None: + Path("cgroup.conf").unlink() From 6db570cd43e3eceda6cb1bb712b03124e1a3b2d2 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Wed, 18 Sep 2024 15:41:25 -0400 Subject: [PATCH 4/4] chore(lint): apply formatting fixes and refactors Signed-off-by: Jason C. Nucciarone --- slurmutils/editors/cgroupconfig.py | 2 +- slurmutils/editors/editor.py | 2 +- tests/unit/editors/test_cgroupconfig.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/slurmutils/editors/cgroupconfig.py b/slurmutils/editors/cgroupconfig.py index 5afa30b..fb53fbf 100644 --- a/slurmutils/editors/cgroupconfig.py +++ b/slurmutils/editors/cgroupconfig.py @@ -22,7 +22,7 @@ from pathlib import Path from typing import Union -from ..models import CgroupConfig +from ..models import CgroupConfig from ..models.option import CgroupConfigOptionSet from .editor import ( clean, diff --git a/slurmutils/editors/editor.py b/slurmutils/editors/editor.py index 6f059ea..6544a04 100644 --- a/slurmutils/editors/editor.py +++ b/slurmutils/editors/editor.py @@ -49,7 +49,7 @@ def parse_line(options, line: str) -> Dict[str, Any]: raise EditorError( ( f"unable to parse configuration option {k}. " - + f"valid configuration options are {[opt for opt in options.keys()]}" + + f"valid configuration options are {list(options.keys())}" ) ) diff --git a/tests/unit/editors/test_cgroupconfig.py b/tests/unit/editors/test_cgroupconfig.py index 1a795fd..b0c8082 100644 --- a/tests/unit/editors/test_cgroupconfig.py +++ b/tests/unit/editors/test_cgroupconfig.py @@ -20,7 +20,6 @@ from slurmutils.editors import cgroupconfig - EXAMPLE_CGROUP_CONF = """# # `cgroup.conf` file generated at 2024-09-18 15:10:44.652017 by slurmutils. #