From 4cc7c2f9981cda5f373aaa46df5aafc768c5f0c3 Mon Sep 17 00:00:00 2001 From: hategan Date: Sun, 10 Mar 2024 19:20:07 -0700 Subject: [PATCH 01/51] Use packaging.versions insted of distutil.version, since the latter is deprecated in Python 3.1x ish. --- .gitignore | 3 ++- requirements.txt | 1 + src/psij-descriptors/aprun_descriptor.py | 5 ++--- src/psij-descriptors/cobalt_descriptor.py | 5 ++--- src/psij-descriptors/core_descriptors.py | 10 +++++----- src/psij-descriptors/flux_descriptor.py | 5 ++--- src/psij-descriptors/jsrun_descriptor.py | 5 ++--- src/psij-descriptors/lsf_descriptor.py | 5 ++--- src/psij-descriptors/pbs_descriptor.py | 7 +++---- src/psij-descriptors/rp_descriptor.py | 5 ++--- src/psij-descriptors/slurm_descriptor.py | 5 ++--- src/psij-descriptors/srun_descriptor.py | 5 ++--- src/psij/_plugins.py | 6 +++--- src/psij/descriptor.py | 4 ++-- tests/plugins1/psij-descriptors/descriptors.py | 16 ++++++++-------- tests/test_executor_versions.py | 3 +-- 16 files changed, 41 insertions(+), 49 deletions(-) diff --git a/.gitignore b/.gitignore index f6083982..4d8418e7 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,5 @@ venv* .venv* build/ docs/.web-build -web-build/ \ No newline at end of file +web-build/ +.packages/ \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index e552d33b..818f3752 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ psutil~=5.9 pystache>=0.6.0 typeguard~=2.12 typing-compat +packaging~=24.0 \ No newline at end of file diff --git a/src/psij-descriptors/aprun_descriptor.py b/src/psij-descriptors/aprun_descriptor.py index c4fb8f26..754f1450 100644 --- a/src/psij-descriptors/aprun_descriptor.py +++ b/src/psij-descriptors/aprun_descriptor.py @@ -1,8 +1,7 @@ -from distutils.version import StrictVersion - +from packaging.version import Version from psij.descriptor import Descriptor __PSI_J_LAUNCHERS__ = [ - Descriptor(name='aprun', version=StrictVersion('0.0.1'), + Descriptor(name='aprun', version=Version('0.0.1'), cls='psij.launchers.aprun.AprunLauncher'), ] diff --git a/src/psij-descriptors/cobalt_descriptor.py b/src/psij-descriptors/cobalt_descriptor.py index 7ea1ad27..f6fdfe04 100644 --- a/src/psij-descriptors/cobalt_descriptor.py +++ b/src/psij-descriptors/cobalt_descriptor.py @@ -1,7 +1,6 @@ -from distutils.version import StrictVersion - +from packaging.version import Version from psij.descriptor import Descriptor -__PSI_J_EXECUTORS__ = [Descriptor(name="cobalt", nice_name='Cobalt', version=StrictVersion("0.0.1"), +__PSI_J_EXECUTORS__ = [Descriptor(name="cobalt", nice_name='Cobalt', version=Version("0.0.1"), cls='psij.executors.batch.cobalt.CobaltJobExecutor')] diff --git a/src/psij-descriptors/core_descriptors.py b/src/psij-descriptors/core_descriptors.py index e5a1a741..b16c89a2 100644 --- a/src/psij-descriptors/core_descriptors.py +++ b/src/psij-descriptors/core_descriptors.py @@ -1,16 +1,16 @@ -from distutils.version import StrictVersion +from packaging.version import Version from psij.descriptor import Descriptor __PSI_J_EXECUTORS__ = [ - Descriptor(name='local', nice_name='Local', version=StrictVersion('0.0.1'), + Descriptor(name='local', nice_name='Local', version=Version('0.0.1'), cls='psij.executors.local.LocalJobExecutor') ] __PSI_J_LAUNCHERS__ = [ - Descriptor(name='single', version=StrictVersion('0.0.1'), + Descriptor(name='single', version=Version('0.0.1'), cls='psij.launchers.single.SingleLauncher'), - Descriptor(name='multiple', version=StrictVersion('0.0.1'), + Descriptor(name='multiple', version=Version('0.0.1'), cls='psij.launchers.multiple.MultipleLauncher'), - Descriptor(name='mpirun', version=StrictVersion('0.0.1'), + Descriptor(name='mpirun', version=Version('0.0.1'), cls='psij.launchers.mpirun.MPILauncher'), ] diff --git a/src/psij-descriptors/flux_descriptor.py b/src/psij-descriptors/flux_descriptor.py index 2087fe37..f1046ac6 100644 --- a/src/psij-descriptors/flux_descriptor.py +++ b/src/psij-descriptors/flux_descriptor.py @@ -1,7 +1,6 @@ -from distutils.version import StrictVersion - +from packaging.version import Version from psij.descriptor import Descriptor -__PSI_J_EXECUTORS__ = [Descriptor(name='flux', nice_name='Flux', version=StrictVersion('0.0.1'), +__PSI_J_EXECUTORS__ = [Descriptor(name='flux', nice_name='Flux', version=Version('0.0.1'), cls='psij.executors.flux.FluxJobExecutor')] diff --git a/src/psij-descriptors/jsrun_descriptor.py b/src/psij-descriptors/jsrun_descriptor.py index 756561d7..b9ceb5c4 100644 --- a/src/psij-descriptors/jsrun_descriptor.py +++ b/src/psij-descriptors/jsrun_descriptor.py @@ -1,8 +1,7 @@ -from distutils.version import StrictVersion - +from packaging.version import Version from psij.descriptor import Descriptor __PSI_J_LAUNCHERS__ = [ - Descriptor(name='jrun', version=StrictVersion('0.0.1'), + Descriptor(name='jrun', version=Version('0.0.1'), cls='psij.launchers.jsrun.JsrunLauncher'), ] diff --git a/src/psij-descriptors/lsf_descriptor.py b/src/psij-descriptors/lsf_descriptor.py index 8c002b30..87e2b64c 100644 --- a/src/psij-descriptors/lsf_descriptor.py +++ b/src/psij-descriptors/lsf_descriptor.py @@ -1,7 +1,6 @@ -from distutils.version import StrictVersion - +from packaging.version import Version from psij.descriptor import Descriptor -__PSI_J_EXECUTORS__ = [Descriptor(name='lsf', nice_name='LSF', version=StrictVersion('0.0.1'), +__PSI_J_EXECUTORS__ = [Descriptor(name='lsf', nice_name='LSF', version=Version('0.0.1'), cls='psij.executors.batch.lsf.LsfJobExecutor')] diff --git a/src/psij-descriptors/pbs_descriptor.py b/src/psij-descriptors/pbs_descriptor.py index d603f83c..8121e7bc 100644 --- a/src/psij-descriptors/pbs_descriptor.py +++ b/src/psij-descriptors/pbs_descriptor.py @@ -1,11 +1,10 @@ -from distutils.version import StrictVersion - +from packaging.version import Version from psij.descriptor import Descriptor __PSI_J_EXECUTORS__ = [Descriptor(name='pbs', nice_name='PBS Pro', aliases=['pbspro'], - version=StrictVersion('0.0.2'), + version=Version('0.0.2'), cls='psij.executors.batch.pbs.PBSJobExecutor'), Descriptor(name='pbs_classic', nice_name='PBS Classic', aliases=['torque'], - version=StrictVersion('0.0.2'), + version=Version('0.0.2'), cls='psij.executors.batch.pbs_classic.PBSClassicJobExecutor')] diff --git a/src/psij-descriptors/rp_descriptor.py b/src/psij-descriptors/rp_descriptor.py index 93468765..afe8f4f8 100644 --- a/src/psij-descriptors/rp_descriptor.py +++ b/src/psij-descriptors/rp_descriptor.py @@ -1,8 +1,7 @@ -from distutils.version import StrictVersion - +from packaging.version import Version from psij.descriptor import Descriptor __PSI_J_EXECUTORS__ = [Descriptor(name='rp', nice_name='Radical Pilot', - version=StrictVersion('0.0.1'), + version=Version('0.0.1'), cls='psij.executors.rp.RPJobExecutor')] diff --git a/src/psij-descriptors/slurm_descriptor.py b/src/psij-descriptors/slurm_descriptor.py index 5ec5465b..ff01129c 100644 --- a/src/psij-descriptors/slurm_descriptor.py +++ b/src/psij-descriptors/slurm_descriptor.py @@ -1,7 +1,6 @@ -from distutils.version import StrictVersion - +from packaging.version import Version from psij.descriptor import Descriptor -__PSI_J_EXECUTORS__ = [Descriptor(name='slurm', nice_name='Slurm', version=StrictVersion('0.0.1'), +__PSI_J_EXECUTORS__ = [Descriptor(name='slurm', nice_name='Slurm', version=Version('0.0.1'), cls='psij.executors.batch.slurm.SlurmJobExecutor')] diff --git a/src/psij-descriptors/srun_descriptor.py b/src/psij-descriptors/srun_descriptor.py index 2f8b4048..f226aad3 100644 --- a/src/psij-descriptors/srun_descriptor.py +++ b/src/psij-descriptors/srun_descriptor.py @@ -1,8 +1,7 @@ -from distutils.version import StrictVersion - +from packaging.version import Version from psij.descriptor import Descriptor __PSI_J_LAUNCHERS__ = [ - Descriptor(name='srun', version=StrictVersion('0.0.1'), + Descriptor(name='srun', version=Version('0.0.1'), cls='psij.launchers.srun.SrunLauncher'), ] diff --git a/src/psij/_plugins.py b/src/psij/_plugins.py index b0b3c863..4b8b9121 100644 --- a/src/psij/_plugins.py +++ b/src/psij/_plugins.py @@ -1,7 +1,7 @@ import importlib import logging from bisect import bisect_left -from distutils.versionpredicate import VersionPredicate +from packaging.specifiers import SpecifierSet from types import ModuleType from typing import Tuple, Dict, List, Union, Type, Any, Optional, TypeVar @@ -116,9 +116,9 @@ def _get_plugin_class(name: str, version_constraint: Optional[str], type: str, versions = store[name] selected = None if version_constraint: - pred = VersionPredicate('x(' + version_constraint + ')') + pred = SpecifierSet(version_constraint) for entry in reversed(versions): - if pred.satisfied_by(entry.version): + if entry.version in pred: selected = entry else: selected = versions[-1] diff --git a/src/psij/descriptor.py b/src/psij/descriptor.py index a52c46f4..a92b180c 100644 --- a/src/psij/descriptor.py +++ b/src/psij/descriptor.py @@ -1,6 +1,6 @@ """Executor/Launcher descriptor module.""" -from distutils.version import StrictVersion +from packaging.version import Version from typing import TypeVar, Generic, Optional, Type, List T = TypeVar('T') @@ -89,7 +89,7 @@ class name that implements the executor or launcher such as `psij.executors.local.LocalJobExecutor`. """ - def __init__(self, name: str, version: StrictVersion, cls: str, + def __init__(self, name: str, version: Version, cls: str, aliases: Optional[List[str]] = None, nice_name: Optional[str] = None) -> None: """ Parameters diff --git a/tests/plugins1/psij-descriptors/descriptors.py b/tests/plugins1/psij-descriptors/descriptors.py index 531be2a3..b0e3e6d8 100644 --- a/tests/plugins1/psij-descriptors/descriptors.py +++ b/tests/plugins1/psij-descriptors/descriptors.py @@ -1,29 +1,29 @@ -from distutils.version import StrictVersion +from packaging.version import Version from psij.descriptor import Descriptor __PSI_J_EXECUTORS__ = [ # executor in the same path as descriptor; should load - Descriptor(name='p1-tp1', version=StrictVersion('0.0.1'), + Descriptor(name='p1-tp1', version=Version('0.0.1'), cls='_test_plugins1.ex1._Executor1'), # executor in different path, but sharing module; should NOT load - Descriptor(name='p2-tp1', version=StrictVersion('0.0.1'), + Descriptor(name='p2-tp1', version=Version('0.0.1'), cls='_test_plugins1.ex2._Executor2'), # executor in different path with no shared module; should NOT load - Descriptor(name='p2-tp3', version=StrictVersion('0.0.1'), + Descriptor(name='p2-tp3', version=Version('0.0.1'), cls='_test_plugins3.ex3._Executor3'), # noop executor that should have no reason to not load - Descriptor(name='_always_loads', version=StrictVersion('0.0.1'), + Descriptor(name='_always_loads', version=Version('0.0.1'), cls='_test_plugins1._always_loads_executor.AlwaysLoadsExecutor'), # noop executor with an import of a package that does not exist - Descriptor(name='_never_loads', version=StrictVersion('0.0.1'), + Descriptor(name='_never_loads', version=Version('0.0.1'), cls='_test_plugins1._never_loads_executor.NeverLoadsExecutor'), # an executor that exercises some of the batch test stuff - Descriptor(name='batch-test', version=StrictVersion('0.0.1'), + Descriptor(name='batch-test', version=Version('0.0.1'), cls='_batch_test._batch_test._TestJobExecutor') ] __PSI_J_LAUNCHERS__ = [ - Descriptor(name='batch-test', version=StrictVersion('0.0.1'), + Descriptor(name='batch-test', version=Version('0.0.1'), cls='_batch_test._batch_test._TestLauncher') ] diff --git a/tests/test_executor_versions.py b/tests/test_executor_versions.py index a1fdee1c..9983e6a5 100755 --- a/tests/test_executor_versions.py +++ b/tests/test_executor_versions.py @@ -2,8 +2,7 @@ # This is meant as a simple test file to check if psi/j was installed successfully -from distutils.version import Version - +from packaging.version import Version from psij import JobExecutor From 8239a2f357c0b68fcf086ebceb24eb1e190f1a6a Mon Sep 17 00:00:00 2001 From: hategan Date: Sun, 10 Mar 2024 19:29:45 -0700 Subject: [PATCH 02/51] Fixed docs --- src/psij/descriptor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/psij/descriptor.py b/src/psij/descriptor.py index a92b180c..16dd3de8 100644 --- a/src/psij/descriptor.py +++ b/src/psij/descriptor.py @@ -68,17 +68,17 @@ class Descriptor(object): .. code-block:: python - from distutils.version import StrictVersion + from packaging.version import Version from psij.descriptor import Descriptor __PSI_J_EXECUTORS__ = [ - Descriptor(name=, version=StrictVersion(), + Descriptor(name=, version=Version(), cls=), ... ] __PSI_J_LAUNCHERS__ = [ - Descriptor(name=, version=StrictVersion(), + Descriptor(name=, version=Version(), cls=), ... ] From 160a4310397f3f812dbd607565b8beacd22a06d5 Mon Sep 17 00:00:00 2001 From: hategan Date: Sun, 10 Mar 2024 19:30:04 -0700 Subject: [PATCH 03/51] Add packaging.version.Version to sphinx nitpick ignore list. --- docs/conf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 0e3a5a5d..f486bdac 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,7 +31,8 @@ autodoc_mock_imports = ['flux'] nitpick_ignore = [ ('py:class', 'distutils.version.StrictVersion'), - ('py:class', 'distutils.version.Version') + ('py:class', 'distutils.version.Version'), + ('py:class', 'packaging.version.Version') ] if web_docs: From 8ba9db59a3ecb89cab94009767561444d67c601a Mon Sep 17 00:00:00 2001 From: hategan Date: Mon, 11 Mar 2024 09:59:25 -0700 Subject: [PATCH 04/51] Updated executor tutorial to use packaging.version --- docs/development/tutorial_add_executor.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/development/tutorial_add_executor.rst b/docs/development/tutorial_add_executor.rst index 5643c861..dcf68d4f 100644 --- a/docs/development/tutorial_add_executor.rst +++ b/docs/development/tutorial_add_executor.rst @@ -68,11 +68,11 @@ Create a simple BatchSchedulerExecutor subclass that does nothing new in `psijpb and create a descriptor file to tell PSI/J about this, ``psij-descriptors/pbspro.py``:: - from distutils.version import StrictVersion + from packaging.version import Version from psij._descriptor import _Descriptor - __PSI_J_EXECUTORS__ = [_Descriptor(name='pbspro', version=StrictVersion('0.0.1'), + __PSI_J_EXECUTORS__ = [_Descriptor(name='pbspro', version=Version('0.0.1'), cls='psijpbs.pbspro.PBSProJobExecutor')] Now, run the test suite. It should fail with an error reporting that the resource manager specific methods of BatchSchedulerExecutor have not been implemented:: From 889e70e9cc49e44522f62fdba484c30a5ebbcba5 Mon Sep 17 00:00:00 2001 From: hategan Date: Tue, 19 Mar 2024 22:18:20 -0700 Subject: [PATCH 05/51] Added `account` job attribute to use for accounting/billing. Deprecate the `project` attribute and re-direct it to `account`. --- QuickStart.md | 2 +- docs/_static/extras.js | 4 +-- docs/user_guide.rst | 2 +- .../executors/batch/cobalt/cobalt.mustache | 4 +-- src/psij/executors/batch/lsf/lsf.mustache | 4 +-- .../executors/batch/pbs/pbs_classic.mustache | 6 ++-- src/psij/executors/batch/pbs/pbspro.mustache | 6 ++-- src/psij/executors/batch/slurm/slurm.mustache | 4 +-- src/psij/job_attributes.py | 35 ++++++++++++++----- testing.conf | 2 +- tests/_test_tools.py | 4 +-- tests/ci_runner.py | 2 +- tests/conftest.py | 24 +++++++++++-- tests/executor_test_params.py | 8 ++--- tests/plugins1/_batch_test/test/test.mustache | 5 +-- tests/test_executor.py | 2 +- .../user_guide/test_scheduling_information.py | 2 +- 17 files changed, 77 insertions(+), 39 deletions(-) diff --git a/QuickStart.md b/QuickStart.md index 0dcea194..8a1fcaaa 100644 --- a/QuickStart.md +++ b/QuickStart.md @@ -58,7 +58,7 @@ def make_job(): spec.arguments = ['HELLO WORLD!'] # set project name if no default is specified - # spec.attributes.project_name = + # spec.attributes.account = # set queue if no default is specified # spec.attributes.queue_name = diff --git a/docs/_static/extras.js b/docs/_static/extras.js index 010e89c7..e8473a08 100644 --- a/docs/_static/extras.js +++ b/docs/_static/extras.js @@ -86,8 +86,8 @@ function detectAll(selectorType) { else if (text == "queue_name") { $(this).text("QUEUE_NAME"); } - else if (text == "project_name") { - $(this).text("PROJECT_NAME"); + else if (text == "account") { + $(this).text("ACCOUNT"); } } if (text == '_get_executor_instance') { diff --git a/docs/user_guide.rst b/docs/user_guide.rst index f0e63e39..b68cf665 100644 --- a/docs/user_guide.rst +++ b/docs/user_guide.rst @@ -335,7 +335,7 @@ and an instance of the :lines: 10-18,21-22 where `QUEUE_NAME` is the LRM queue where the job should be sent and -`PROJECT_NAME` is a project/account that may need to be specified for +`ACCOUNT` is a project/account that may need to be specified for accounting purposes. These values generally depend on the system and allocation being used. diff --git a/src/psij/executors/batch/cobalt/cobalt.mustache b/src/psij/executors/batch/cobalt/cobalt.mustache index 922e9eea..8281ed3e 100644 --- a/src/psij/executors/batch/cobalt/cobalt.mustache +++ b/src/psij/executors/batch/cobalt/cobalt.mustache @@ -25,9 +25,9 @@ {{#reservation_id}} #COBALT --queue={{.}} {{/reservation_id}} - {{#project_name}} + {{#account}} #COBALT --project={{.}} - {{/project_name}} + {{/account}} {{/job.spec.attributes}} {{#custom_attributes}} diff --git a/src/psij/executors/batch/lsf/lsf.mustache b/src/psij/executors/batch/lsf/lsf.mustache index 781e9f82..a616ad37 100644 --- a/src/psij/executors/batch/lsf/lsf.mustache +++ b/src/psij/executors/batch/lsf/lsf.mustache @@ -44,10 +44,10 @@ #BSUB -q "{{.}}" {{/queue_name}} - {{#project_name}} + {{#account}} #BSUB -G "{{.}}" #BSUB -P "{{.}}" - {{/project_name}} + {{/account}} {{#reservation_id}} #BSUB -U "{{.}}" diff --git a/src/psij/executors/batch/pbs/pbs_classic.mustache b/src/psij/executors/batch/pbs/pbs_classic.mustache index 3967d005..03a3d015 100644 --- a/src/psij/executors/batch/pbs/pbs_classic.mustache +++ b/src/psij/executors/batch/pbs/pbs_classic.mustache @@ -21,9 +21,9 @@ {{/formatted_job_duration}} {{#job.spec.attributes}} - {{#project_name}} -#PBS -P {{.}} - {{/project_name}} + {{#account}} +#PBS -A {{.}} + {{/account}} {{#queue_name}} #PBS -q {{.}} {{/queue_name}} diff --git a/src/psij/executors/batch/pbs/pbspro.mustache b/src/psij/executors/batch/pbs/pbspro.mustache index 2d44fae9..19176687 100644 --- a/src/psij/executors/batch/pbs/pbspro.mustache +++ b/src/psij/executors/batch/pbs/pbspro.mustache @@ -24,9 +24,9 @@ {{/formatted_job_duration}} {{#job.spec.attributes}} - {{#project_name}} -#PBS -P {{.}} - {{/project_name}} + {{#account}} +#PBS -A {{.}} + {{/account}} {{#queue_name}} #PBS -q {{.}} {{/queue_name}} diff --git a/src/psij/executors/batch/slurm/slurm.mustache b/src/psij/executors/batch/slurm/slurm.mustache index d6f850fc..b044ba27 100644 --- a/src/psij/executors/batch/slurm/slurm.mustache +++ b/src/psij/executors/batch/slurm/slurm.mustache @@ -48,9 +48,9 @@ #SBATCH --partition="{{.}}" {{/queue_name}} - {{#project_name}} + {{#account}} #SBATCH --account="{{.}}" - {{/project_name}} + {{/account}} {{#reservation_id}} #SBATCH --reservation="{{.}}" diff --git a/src/psij/job_attributes.py b/src/psij/job_attributes.py index e481ee73..02263896 100644 --- a/src/psij/job_attributes.py +++ b/src/psij/job_attributes.py @@ -1,3 +1,4 @@ +import logging import re from datetime import timedelta from typing import Optional, Dict @@ -5,6 +6,9 @@ from typeguard import check_argument_types +logger = logging.getLogger(__name__) + + _WALLTIME_FMT_ERROR = 'Unknown walltime format: %s. Accepted formats are hh:mm:ss, ' \ 'hh:mm, mm, or n\\s*[h|m|s].' @@ -13,17 +17,19 @@ class JobAttributes(object): """A class containing ancillary job information that describes how a job is to be run.""" def __init__(self, duration: timedelta = timedelta(minutes=10), - queue_name: Optional[str] = None, project_name: Optional[str] = None, + queue_name: Optional[str] = None, account: Optional[str] = None, reservation_id: Optional[str] = None, - custom_attributes: Optional[Dict[str, object]] = None) -> None: + custom_attributes: Optional[Dict[str, object]] = None, + project_name: Optional[str] = None) -> None: """ :param duration: Specifies the duration (walltime) of the job. A job whose execution exceeds its walltime can be terminated forcefully. :param queue_name: If a backend supports multiple queues, this parameter can be used to instruct the backend to send this job to a particular queue. - :param project_name: If a backend supports multiple projects for billing purposes, setting - this attribute instructs the backend to bill the indicated project for the resources - consumed by this job. + :param account: An account to use for billing purposes. Please note that the executor + implementation (or batch scheduler) may use a different term for the option used for + accounting/billing purposes, such as `project`. However, scheduler must map this + attribute to the accounting/billing option in the underlying execution mechanism. :param reservation_id: Allows specifying an advanced reservation ID. Advanced reservations enable the pre-allocation of a set of resources/compute nodes for a certain duration such that jobs can be run immediately, without waiting in the queue for resources to @@ -39,11 +45,13 @@ def __init__(self, duration: timedelta = timedelta(minutes=10), `pbs.c`, `lsf.core_isolation`) and translate them into the corresponding batch scheduler directives (e.g., `#SLURM --constraint=...`, `#PBS -c ...`, `#BSUB -core_isolation ...`). + :param project_name: Deprecated. Please use the `account` attribute. All constructor parameters are accessible as properties. """ assert check_argument_types() + self.account = account self.duration = duration self.queue_name = queue_name self.project_name = project_name @@ -86,8 +94,8 @@ def custom_attributes(self, attrs: Optional[Dict[str, object]]) -> None: def __repr__(self) -> str: """Returns a string representation of this object.""" - return 'JobAttributes(duration={}, queue_name={}, project_name={}, reservation_id={}, ' \ - 'custom_attributes={})'.format(self.duration, self.queue_name, self.project_name, + return 'JobAttributes(duration={}, queue_name={}, account={}, reservation_id={}, ' \ + 'custom_attributes={})'.format(self.duration, self.queue_name, self.account, self.reservation_id, self._custom_attributes) def __eq__(self, o: object) -> bool: @@ -99,7 +107,7 @@ def __eq__(self, o: object) -> bool: if not isinstance(o, JobAttributes): return False - for prop_name in ['duration', 'queue_name', 'project_name', 'reservation_id', + for prop_name in ['duration', 'queue_name', 'account', 'reservation_id', 'custom_attributes']: if getattr(self, prop_name) != getattr(o, prop_name): return False @@ -150,3 +158,14 @@ def parse_walltime(walltime: str) -> timedelta: elif unit == 's': return timedelta(seconds=val) raise ValueError(_WALLTIME_FMT_ERROR % walltime) + + @property + def project_name(self) -> Optional[str]: + """Deprecated. Please use the `account` attribute.""" + return self.account + + @project_name.setter + def project_name(self, project_name: Optional[str]) -> None: + logger.warning('The "project_name" attribute is deprecated. Please use the "account" ' + 'attribute instead.') + self.account = project_name diff --git a/testing.conf b/testing.conf index d0340789..2b04247e 100644 --- a/testing.conf +++ b/testing.conf @@ -119,7 +119,7 @@ queue_name = # If you need a project/account for billing purposes, enter it here. # -project_name = +account = diff --git a/tests/_test_tools.py b/tests/_test_tools.py index b8c00f0a..2517bd00 100644 --- a/tests/_test_tools.py +++ b/tests/_test_tools.py @@ -52,8 +52,8 @@ def _get_executor_instance(ep: ExecutorTestParams, job: Optional[Job] = None) -> assert job.spec is not None job.spec.launcher = ep.launcher job.spec.attributes = JobAttributes(custom_attributes=ep.custom_attributes) - if ep.project_name is not None: - job.spec.attributes.project_name = ep.project_name + if ep.account is not None: + job.spec.attributes.account = ep.account if ep.queue_name is not None: job.spec.attributes.queue_name = ep.queue_name return JobExecutor.get_instance(ep.executor, url=ep.url) diff --git a/tests/ci_runner.py b/tests/ci_runner.py index a1a9420e..47507e29 100755 --- a/tests/ci_runner.py +++ b/tests/ci_runner.py @@ -147,7 +147,7 @@ def run_branch_tests(conf: Dict[str, str], dir: Path, run_id: str, clone: bool = args.append('--branch-name-override') args.append(fake_branch_name) for opt in ['maintainer_email', 'executors', 'server_url', 'key', 'max_age', - 'custom_attributes', 'queue_name', 'project_name']: + 'custom_attributes', 'queue_name', 'project_name', 'account']: try: val = get_conf(conf, opt) args.append('--' + opt.replace('_', '-')) diff --git a/tests/conftest.py b/tests/conftest.py index d2a1c456..3e0ca9a4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -69,6 +69,8 @@ def pytest_addoption(parser): help='A queue to run the batch jobs in.') parser.addoption('--project-name', action='store', default=None, help='A project/account name to associate the batch jobs with.') + parser.addoption('--account', action='store', default=None, + help='An account to use for billing purposes.') parser.addoption('--custom-attributes', action='store', default=None, help='A set of custom attributes to pass to jobs.') parser.addoption('--minimal-uploads', action='store_true', default=False, @@ -152,14 +154,30 @@ def _translate_launcher(config: Dict[str, str], exec: str, launcher: str) -> str return launcher +_WARNED_ABOUT_ACCOUNT_CLASH = False + + +def _get_account(options): + global _WARNED_ABOUT_ACCOUNT_CLASH + if options.account: + if options.project_name and not _WARNED_ABOUT_ACCOUNT_CLASH: + _WARNED_ABOUT_ACCOUNT_CLASH = True + logger.warning('Both "account" and "project_name" are specified. Ignoring ' + '"project_name".') + return options.account + else: + return options.project_name + + def pytest_generate_tests(metafunc): + options = metafunc.config.option if 'execparams' in metafunc.fixturenames: etps = [] for x in _get_executors((metafunc.config)): - etp = ExecutorTestParams(x, queue_name=metafunc.config.option.queue_name, - project_name=metafunc.config.option.project_name, - custom_attributes_raw=metafunc.config.option.custom_attributes) + etp = ExecutorTestParams(x, queue_name=options.queue_name, + account=_get_account(options), + custom_attributes_raw=options.custom_attributes) etps.append(etp) metafunc.parametrize('execparams', etps, ids=lambda x: str(x)) diff --git a/tests/executor_test_params.py b/tests/executor_test_params.py index 9796d0fa..6c864a94 100644 --- a/tests/executor_test_params.py +++ b/tests/executor_test_params.py @@ -6,7 +6,7 @@ class ExecutorTestParams: """A class holding executor, launcher, url pairs.""" def __init__(self, spec: str, queue_name: Optional[str] = None, - project_name: Optional[str] = None, + account: Optional[str] = None, custom_attributes_raw: Optional[Dict[str, Dict[str, object]]] = None) \ -> None: """ @@ -19,8 +19,8 @@ def __init__(self, spec: str, queue_name: Optional[str] = None, url are specified, the string should be formatted as "::". queue_name An optional queue to submit the job to - project_name - An optional project name to associate the job with + account + An optional account to use for billing purposes. custom_attributes """ spec_l = re.split(':', spec, maxsplit=2) @@ -35,7 +35,7 @@ def __init__(self, spec: str, queue_name: Optional[str] = None, self.url = None self.queue_name = queue_name - self.project_name = project_name + self.account = account self.custom_attributes_raw = custom_attributes_raw self.custom_attributes: Dict[str, object] = {} diff --git a/tests/plugins1/_batch_test/test/test.mustache b/tests/plugins1/_batch_test/test/test.mustache index ae51ae68..bfd09a45 100644 --- a/tests/plugins1/_batch_test/test/test.mustache +++ b/tests/plugins1/_batch_test/test/test.mustache @@ -18,9 +18,10 @@ export PSIJ_TEST_BATCH_EXEC_COUNT={{.}} {{#queue_name}} export PSIJ_TEST_BATCH_EXEC_QUEUE="{{.}}" {{/queue_name}} - {{#project_name}} + {{#account}} export PSIJ_TEST_BATCH_EXEC_PROJECT="{{.}}" - {{/project_name}} +export PSIJ_TEST_BATCH_EXEC_ACCOUNT="{{.}}" + {{/account}} {{#reservation_id}} export PSIJ_TEST_BATCH_EXEC_RES_ID="{{.}}" {{/reservation_id}} diff --git a/tests/test_executor.py b/tests/test_executor.py index e425a030..0cc8b510 100644 --- a/tests/test_executor.py +++ b/tests/test_executor.py @@ -220,7 +220,7 @@ def test_submit_script_generation(exec_name: str) -> None: prefix = _get_attr_prefix(exec_name) _check_str_attrs(ex, job, ['executable', 'directory'], lambda k, v: setattr(spec, k, v)) - _check_str_attrs(ex, job, ['queue_name', 'project_name', 'reservation_id'], + _check_str_attrs(ex, job, ['queue_name', 'account', 'reservation_id'], lambda k, v: setattr(attrs, k, v)) _check_str_attrs(ex, job, [prefix + '.cust_attr1', prefix + '.cust_attr2'], lambda k, v: c_attrs.__setitem__(k, v)) diff --git a/tests/user_guide/test_scheduling_information.py b/tests/user_guide/test_scheduling_information.py index ad6d226d..367db1d1 100644 --- a/tests/user_guide/test_scheduling_information.py +++ b/tests/user_guide/test_scheduling_information.py @@ -12,7 +12,7 @@ def test_user_guide_scheduling_info(execparams: ExecutorTestParams) -> None: executable="/bin/date", attributes=JobAttributes( queue_name=execparams.queue_name, - project_name=execparams.project_name + account=execparams.account ) ) ) From e61784f0d439f545998dbf303e29befb08d6cec9 Mon Sep 17 00:00:00 2001 From: hategan Date: Tue, 2 Apr 2024 14:24:01 -0700 Subject: [PATCH 06/51] When testing, always put the src dir before .packages in PYTHONPATH. We don't want a dependency to install something that may have the same name as something in psij (including possibly a different version of psij itself) and hijack things. --- tests/ci_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci_runner.py b/tests/ci_runner.py index a1a9420e..7faa6684 100755 --- a/tests/ci_runner.py +++ b/tests/ci_runner.py @@ -164,8 +164,8 @@ def run_branch_tests(conf: Dict[str, str], dir: Path, run_id: str, clone: bool = cwd = (dir / 'code') if clone else Path('.') env = dict(os.environ) - env['PYTHONPATH'] = str(cwd.resolve() / '.packages') \ - + ':' + str(cwd.resolve() / 'src') \ + env['PYTHONPATH'] = str(cwd.resolve() / 'src') \ + + ':' + str(cwd.resolve() / '.packages') \ + (':' + env['PYTHONPATH'] if 'PYTHONPATH' in env else '') subprocess.run(args, cwd=cwd.resolve(), env=env) From 3648ca5fdc17631d2ae00df48ee6b911cce3049a Mon Sep 17 00:00:00 2001 From: hategan Date: Mon, 8 Apr 2024 10:02:06 -0700 Subject: [PATCH 07/51] Replaced lingering `distutils.Version` --- src/psij/job_executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/psij/job_executor.py b/src/psij/job_executor.py index b9f9a226..9f686b73 100644 --- a/src/psij/job_executor.py +++ b/src/psij/job_executor.py @@ -1,6 +1,6 @@ import logging from abc import ABC, abstractmethod -from distutils.version import Version +from packaging.version import Version from threading import RLock from typing import Optional, Dict, List, Type, cast, Union, Callable, Set From b16c3ba39fa992dcc0ef32e7d6a5be76d72df516 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Tue, 9 Apr 2024 00:03:04 +0200 Subject: [PATCH 08/51] typeguard update --- src/psij/job_attributes.py | 4 ++-- src/psij/job_spec.py | 4 ++-- src/psij/resource_spec.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/psij/job_attributes.py b/src/psij/job_attributes.py index e481ee73..1ed2b6a7 100644 --- a/src/psij/job_attributes.py +++ b/src/psij/job_attributes.py @@ -2,7 +2,7 @@ from datetime import timedelta from typing import Optional, Dict -from typeguard import check_argument_types +from typeguard import typechecked _WALLTIME_FMT_ERROR = 'Unknown walltime format: %s. Accepted formats are hh:mm:ss, ' \ @@ -12,6 +12,7 @@ class JobAttributes(object): """A class containing ancillary job information that describes how a job is to be run.""" + @typechecked def __init__(self, duration: timedelta = timedelta(minutes=10), queue_name: Optional[str] = None, project_name: Optional[str] = None, reservation_id: Optional[str] = None, @@ -42,7 +43,6 @@ def __init__(self, duration: timedelta = timedelta(minutes=10), All constructor parameters are accessible as properties. """ - assert check_argument_types() self.duration = duration self.queue_name = queue_name diff --git a/src/psij/job_spec.py b/src/psij/job_spec.py index 7eec03ec..d6e08a7b 100644 --- a/src/psij/job_spec.py +++ b/src/psij/job_spec.py @@ -5,7 +5,7 @@ import pathlib from typing import Dict, List, Optional, Union -from typeguard import check_argument_types +from typeguard import typechecked import psij.resource_spec import psij.job_attributes @@ -36,6 +36,7 @@ def _to_env_dict(arg: Union[Dict[str, Union[str, int]], None]) -> Optional[Dict[ class JobSpec(object): """A class that describes the details of a job.""" + @typechecked def __init__(self, executable: Optional[str] = None, arguments: Optional[List[str]] = None, # For some odd reason, and only in the constructor, if Path is used directly, # sphinx fails to find the class. Using Path in the getters and setters does not @@ -129,7 +130,6 @@ def __init__(self, executable: Optional[str] = None, arguments: Optional[List[st the scheduler. In such a case, one must leave the `spec.directory` attribute empty and refer to files inside the job directory using relative paths. """ - assert check_argument_types() self._name = name self.executable = executable diff --git a/src/psij/resource_spec.py b/src/psij/resource_spec.py index 2d17f3c9..2bb3a6fa 100644 --- a/src/psij/resource_spec.py +++ b/src/psij/resource_spec.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from typing import Optional, List -from typeguard import check_argument_types +from typeguard import typechecked from psij.exceptions import InvalidJobException @@ -48,6 +48,7 @@ def get_instance(version: int) -> 'ResourceSpec': class ResourceSpecV1(ResourceSpec): """This class implements V1 of the PSI/J resource specification.""" + @typechecked def __init__(self, node_count: Optional[int] = None, process_count: Optional[int] = None, processes_per_node: Optional[int] = None, @@ -81,7 +82,6 @@ def __init__(self, node_count: Optional[int] = None, All constructor parameters are accessible as properties. """ - assert check_argument_types() self.node_count = node_count self.process_count = process_count From 94dceddeb002e71a5f2a3c2ea8df94a765a091b2 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Tue, 9 Apr 2024 00:05:44 +0200 Subject: [PATCH 09/51] dep update --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 818f3752..d626d85c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ filelock~=3.4 psutil~=5.9 pystache>=0.6.0 -typeguard~=2.12 +typeguard>=3.0.1 typing-compat -packaging~=24.0 \ No newline at end of file +packaging~=24.0 From 392cc35b1d1aabc958ecb2f3be2eba367da457be Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Tue, 9 Apr 2024 21:11:43 +0200 Subject: [PATCH 10/51] move to class decorator for `typechecked` --- src/psij/resource_spec.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/psij/resource_spec.py b/src/psij/resource_spec.py index 2bb3a6fa..68aa680f 100644 --- a/src/psij/resource_spec.py +++ b/src/psij/resource_spec.py @@ -45,10 +45,10 @@ def get_instance(version: int) -> 'ResourceSpec': raise ValueError() +@typechecked class ResourceSpecV1(ResourceSpec): """This class implements V1 of the PSI/J resource specification.""" - @typechecked def __init__(self, node_count: Optional[int] = None, process_count: Optional[int] = None, processes_per_node: Optional[int] = None, @@ -82,7 +82,6 @@ def __init__(self, node_count: Optional[int] = None, All constructor parameters are accessible as properties. """ - self.node_count = node_count self.process_count = process_count self.processes_per_node = processes_per_node From 9fc83bac2d36788ca806b7bfcc68a790b3607da5 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Tue, 9 Apr 2024 21:35:33 +0200 Subject: [PATCH 11/51] linting, disable typeguard on type specific tests --- src/psij/job_attributes.py | 1 - src/psij/job_spec.py | 1 - tests/test_job_spec.py | 7 +++++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/psij/job_attributes.py b/src/psij/job_attributes.py index 1ed2b6a7..f55c0cf5 100644 --- a/src/psij/job_attributes.py +++ b/src/psij/job_attributes.py @@ -43,7 +43,6 @@ def __init__(self, duration: timedelta = timedelta(minutes=10), All constructor parameters are accessible as properties. """ - self.duration = duration self.queue_name = queue_name self.project_name = project_name diff --git a/src/psij/job_spec.py b/src/psij/job_spec.py index d6e08a7b..e3585d0d 100644 --- a/src/psij/job_spec.py +++ b/src/psij/job_spec.py @@ -130,7 +130,6 @@ def __init__(self, executable: Optional[str] = None, arguments: Optional[List[st the scheduler. In such a case, one must leave the `spec.directory` attribute empty and refer to files inside the job directory using relative paths. """ - self._name = name self.executable = executable self.arguments = arguments diff --git a/tests/test_job_spec.py b/tests/test_job_spec.py index 9fa57a49..f02b6305 100644 --- a/tests/test_job_spec.py +++ b/tests/test_job_spec.py @@ -1,6 +1,8 @@ import os from pathlib import Path +from typeguard import suppress_type_checks + import pytest from psij import Job, JobExecutor, JobSpec @@ -13,8 +15,9 @@ def _test_spec(spec: JobSpec) -> None: def test_environment_types() -> None: - with pytest.raises(TypeError): - _test_spec(JobSpec(executable='true', environment={1: 'foo'})) # type: ignore + with suppress_type_checks(): + with pytest.raises(TypeError): + _test_spec(JobSpec(executable='true', environment={1: 'foo'})) # type: ignore with pytest.raises(TypeError): spec = JobSpec(executable='true') From ca90feb1b414a9e90dc9202a50fe96d81f3a109f Mon Sep 17 00:00:00 2001 From: hategan Date: Tue, 30 Apr 2024 10:55:21 -0700 Subject: [PATCH 12/51] Check that twine is accessible before doing a release. --- release.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/release.sh b/release.sh index 28a8b444..615d1be5 100755 --- a/release.sh +++ b/release.sh @@ -48,6 +48,12 @@ if [ "$LAST" != "$TARGET_VERSION" ]; then error "Version $TARGET_VERSION is lower than the latest tagged version ($LAST)." fi +if which twine >/dev/null; then + echo "Found twine" +else + error "Twine was not found. Please install it and then re-run this script" +fi + echo "This will tag and release psij-python to version $TARGET_VERSION." echo -n "Type 'yes' if you want to continue: " read REPLY From 96ef5d333203fc8011769e723f01bdd85c6d8290 Mon Sep 17 00:00:00 2001 From: hategan Date: Tue, 30 Apr 2024 10:55:38 -0700 Subject: [PATCH 13/51] Description should not contain new lines --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index d841b2a7..c9df126d 100644 --- a/setup.py +++ b/setup.py @@ -12,8 +12,7 @@ name='psij-python', version=VERSION, - description='''This is an implementation of the PSI/J (Portable Submission Interface for Jobs) - specification.''', + description='''This is an implementation of the PSI/J (Portable Submission Interface for Jobs) specification.''', author='The ExaWorks Team', author_email='hategan@mcs.anl.gov', From 8f9411d8bfce0489e3afa6ae77da4f82c3b5a98c Mon Sep 17 00:00:00 2001 From: hategan Date: Tue, 30 Apr 2024 10:56:29 -0700 Subject: [PATCH 14/51] Include requirements.txt in source packages. --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..f9bd1455 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include requirements.txt From ff675ab258f44fc1fa47ee9a0c15fae820589f08 Mon Sep 17 00:00:00 2001 From: hategan Date: Mon, 6 May 2024 14:40:06 -0700 Subject: [PATCH 15/51] Only trigger warning about `project_name` if it's being set to something that isn't `None`. --- src/psij/job_attributes.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/psij/job_attributes.py b/src/psij/job_attributes.py index 02263896..157da5e9 100644 --- a/src/psij/job_attributes.py +++ b/src/psij/job_attributes.py @@ -166,6 +166,7 @@ def project_name(self) -> Optional[str]: @project_name.setter def project_name(self, project_name: Optional[str]) -> None: - logger.warning('The "project_name" attribute is deprecated. Please use the "account" ' - 'attribute instead.') + if project_name is not None: + logger.warning('The "project_name" attribute is deprecated. Please use the "account" ' + 'attribute instead.') self.account = project_name From 39e26a14390ee1bcaccf81324cc04a1f29db29cc Mon Sep 17 00:00:00 2001 From: hategan Date: Mon, 13 May 2024 09:56:27 -0700 Subject: [PATCH 16/51] Move filelock requirement from requirements.txt to requirements-tests.txt since it appears to be the only place where it is actually used. --- requirements-tests.txt | 3 ++- requirements.txt | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index aab4fb0b..7cc475ae 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -6,4 +6,5 @@ pytest >= 6.2.0 requests >= 2.25.1 pytest-cov -pytest-timeout \ No newline at end of file +pytest-timeout +filelock~=3.4 diff --git a/requirements.txt b/requirements.txt index 818f3752..9e703381 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ -filelock~=3.4 psutil~=5.9 pystache>=0.6.0 typeguard~=2.12 typing-compat -packaging~=24.0 \ No newline at end of file +packaging~=24.0 From 5545e9ce6d1ad35aedcd76149dfedd5c3e01a945 Mon Sep 17 00:00:00 2001 From: hategan Date: Tue, 14 May 2024 13:29:30 -0700 Subject: [PATCH 17/51] Trigger tests/checks From aabf1463db7efabbb7210ec33d6a4a495b33eb61 Mon Sep 17 00:00:00 2001 From: Mihael Hategan Date: Sun, 30 Jun 2024 16:14:37 -0700 Subject: [PATCH 18/51] Create dependabot.yml --- .github/dependabot.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..ddb33180 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" # Location of package manifests + schedule: + interval: "weekly" + versioning-strategy: widen From cb372615d17b2f850e620eb40e7425815216374b Mon Sep 17 00:00:00 2001 From: Mihael Hategan Date: Sun, 30 Jun 2024 16:20:21 -0700 Subject: [PATCH 19/51] Update dependabot.yml --- .github/dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ddb33180..6634c32e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,4 +9,4 @@ updates: directory: "/" # Location of package manifests schedule: interval: "weekly" - versioning-strategy: widen + versioning-strategy: auto From 891608e2dd807b49f1fb36b76fc6022ae2c1d511 Mon Sep 17 00:00:00 2001 From: Mihael Hategan Date: Sun, 30 Jun 2024 16:26:48 -0700 Subject: [PATCH 20/51] Update dependabot.yml --- .github/dependabot.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6634c32e..74abab25 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -10,3 +10,11 @@ updates: schedule: interval: "weekly" versioning-strategy: auto + allow: + - dependency-name: "filelock" + - dependency-name: "psutil" + - dependency-name: "pystache" + - dependency-name: "typeguard" + - dependency-name: "typing-compat" + - dependency-name: "packaging" + From 0bba163b3e5ef7ec2aa5b7f476b601769b339c17 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 30 Jun 2024 23:33:15 +0000 Subject: [PATCH 21/51] Update filelock requirement from ~=3.4 to ~=3.15 Updates the requirements on [filelock](https://github.com/tox-dev/py-filelock) to permit the latest version. - [Release notes](https://github.com/tox-dev/py-filelock/releases) - [Changelog](https://github.com/tox-dev/filelock/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/py-filelock/compare/3.4.0...3.15.4) --- updated-dependencies: - dependency-name: filelock dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 7cc475ae..786f3363 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -7,4 +7,4 @@ pytest >= 6.2.0 requests >= 2.25.1 pytest-cov pytest-timeout -filelock~=3.4 +filelock~=3.15 From f046653051f94a96b19b755f0170812acd6f01ff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 30 Jun 2024 23:33:17 +0000 Subject: [PATCH 22/51] Update packaging requirement from ~=24.0 to ~=24.1 Updates the requirements on [packaging](https://github.com/pypa/packaging) to permit the latest version. - [Release notes](https://github.com/pypa/packaging/releases) - [Changelog](https://github.com/pypa/packaging/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pypa/packaging/compare/24.0...24.1) --- updated-dependencies: - dependency-name: packaging dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9d159867..83f28869 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ psutil~=5.9 pystache>=0.6.0 typeguard>=3.0.1 typing-compat -packaging~=24.0 +packaging~=24.1 From 2359b35441e61640db3f66a7dc9b2fdce6297c68 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 30 Jun 2024 23:33:20 +0000 Subject: [PATCH 23/51] Update psutil requirement from ~=5.9 to ~=6.0 Updates the requirements on [psutil](https://github.com/giampaolo/psutil) to permit the latest version. - [Changelog](https://github.com/giampaolo/psutil/blob/master/HISTORY.rst) - [Commits](https://github.com/giampaolo/psutil/compare/release-5.9.0...release-6.0.0) --- updated-dependencies: - dependency-name: psutil dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9d159867..15d42edf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -psutil~=5.9 +psutil~=6.0 pystache>=0.6.0 typeguard>=3.0.1 typing-compat From aaa72b6e57b9dad96210362b031b0edbf579bc80 Mon Sep 17 00:00:00 2001 From: Mihael Hategan Date: Sun, 30 Jun 2024 16:51:51 -0700 Subject: [PATCH 24/51] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 15d42edf..e30295db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -psutil~=6.0 +psutil >=5.9, <=6.0 pystache>=0.6.0 typeguard>=3.0.1 typing-compat From 993427f38388ed187c66bccbacdc3c52b475f9da Mon Sep 17 00:00:00 2001 From: Mihael Hategan Date: Sun, 30 Jun 2024 16:52:42 -0700 Subject: [PATCH 25/51] Update requirements-tests.txt --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 786f3363..d5018b16 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -7,4 +7,4 @@ pytest >= 6.2.0 requests >= 2.25.1 pytest-cov pytest-timeout -filelock~=3.15 +filelock >= 3.4, <= 3.15 From 92047f94d5dc41d45ab904ac61805067982d3c38 Mon Sep 17 00:00:00 2001 From: Mihael Hategan Date: Sun, 30 Jun 2024 16:53:24 -0700 Subject: [PATCH 26/51] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 83f28869..d497b37b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ psutil~=5.9 pystache>=0.6.0 typeguard>=3.0.1 typing-compat -packaging~=24.1 +packaging >= 24.0, <= 24.1 From 26d3959d6070ef03483feb21d0c96849289b54fc Mon Sep 17 00:00:00 2001 From: hategan Date: Sun, 30 Jun 2024 17:03:47 -0700 Subject: [PATCH 27/51] Newer versions of codespell seem to get triggered on pytest's assertIn method, so add it to the codespell ignore list. --- .github/workflows/codespell.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 36f59c8f..447176bc 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -15,3 +15,5 @@ jobs: steps: - uses: actions/checkout@v2 - uses: codespell-project/actions-codespell@master + with: + ignore_words_list: assertIn From 30a47c23378c7071d2c19ab6d156d9129f7c070d Mon Sep 17 00:00:00 2001 From: hategan Date: Sun, 30 Jun 2024 17:20:12 -0700 Subject: [PATCH 28/51] `squeue` fails if a job id is passed to `-j` for a job that is not in the queue (any more). Because we cannot guarantee that the query intervals are smaller than the intervals at which jobs get purged from the queue, failed `squeue` invocations can happen. This change replaces the `-j` option with `--me`, which lists all jobs for a user. This has the potential of listing more jobs than necessary (since a user can have multiple PSI/J instances managing jobs). --- src/psij/executors/batch/slurm.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/psij/executors/batch/slurm.py b/src/psij/executors/batch/slurm.py index 7778b5b8..ffdb3a3e 100644 --- a/src/psij/executors/batch/slurm.py +++ b/src/psij/executors/batch/slurm.py @@ -147,12 +147,11 @@ def process_cancel_command_output(self, exit_code: int, out: str) -> None: def get_status_command(self, native_ids: Collection[str]) -> List[str]: """See :meth:`~.BatchSchedulerExecutor.get_status_command`.""" - ids = ','.join(native_ids) # we're not really using job arrays, so this is equivalent to the job ID. However, if # we were to use arrays, this would return one ID for the entire array rather than # listing each element of the array independently - return [_SQUEUE_COMMAND, '-O', 'JobArrayID,StateCompact,Reason', '-t', 'all', '-j', ids] + return [_SQUEUE_COMMAND, '-O', 'JobArrayID,StateCompact,Reason', '-t', 'all', '--me'] def parse_status_output(self, exit_code: int, out: str) -> Dict[str, JobStatus]: """See :meth:`~.BatchSchedulerExecutor.parse_status_output`.""" From ffcf4c024daebbd7ac9644118ddf936a31d079df Mon Sep 17 00:00:00 2001 From: hategan Date: Sun, 30 Jun 2024 21:13:29 -0700 Subject: [PATCH 29/51] Use uid for batch-test scheduler files stored in `/tmp` in order to avoid issues when multiple users use it. --- tests/plugins1/_batch_test/test/qlib.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/plugins1/_batch_test/test/qlib.py b/tests/plugins1/_batch_test/test/qlib.py index 5379f381..9e3764a7 100644 --- a/tests/plugins1/_batch_test/test/qlib.py +++ b/tests/plugins1/_batch_test/test/qlib.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 import json import logging +import os import tempfile from datetime import timedelta, datetime from pathlib import Path @@ -8,10 +9,12 @@ from filelock import FileLock + +uid = os.getuid() tmp = tempfile.gettempdir() -lock_file = Path(tmp) / 'qlist.lock' -state_file = Path(tmp) / 'qlist' -log_file = Path(tmp) / 'qlist.log' +lock_file = Path(tmp) / 'qlist-%s.lock' % uid +state_file = Path(tmp) / 'qlist-%s' % uid +log_file = Path(tmp) / 'qlist-%s.log' % uid my_dir = Path(__file__).parent From 0efccd420583d7d1510ad0172e9c4ad65bc86550 Mon Sep 17 00:00:00 2001 From: hategan Date: Sun, 30 Jun 2024 21:41:53 -0700 Subject: [PATCH 30/51] Turns out '/' and '%' have the same precedence. --- tests/plugins1/_batch_test/test/qlib.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/plugins1/_batch_test/test/qlib.py b/tests/plugins1/_batch_test/test/qlib.py index 9e3764a7..e2a240f6 100644 --- a/tests/plugins1/_batch_test/test/qlib.py +++ b/tests/plugins1/_batch_test/test/qlib.py @@ -12,9 +12,9 @@ uid = os.getuid() tmp = tempfile.gettempdir() -lock_file = Path(tmp) / 'qlist-%s.lock' % uid -state_file = Path(tmp) / 'qlist-%s' % uid -log_file = Path(tmp) / 'qlist-%s.log' % uid +lock_file = Path(tmp) / ('qlist-%s.lock' % uid) +state_file = Path(tmp) / ('qlist-%s' % uid) +log_file = Path(tmp) / ('qlist-%s.log' % uid) my_dir = Path(__file__).parent From 160382cb6cc8b07df486d7f90dcaaa7cea0ee974 Mon Sep 17 00:00:00 2001 From: hategan Date: Sun, 30 Jun 2024 21:42:56 -0700 Subject: [PATCH 31/51] Linting fix --- src/psij/executors/batch/slurm.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/psij/executors/batch/slurm.py b/src/psij/executors/batch/slurm.py index ffdb3a3e..6f0963ee 100644 --- a/src/psij/executors/batch/slurm.py +++ b/src/psij/executors/batch/slurm.py @@ -147,7 +147,6 @@ def process_cancel_command_output(self, exit_code: int, out: str) -> None: def get_status_command(self, native_ids: Collection[str]) -> List[str]: """See :meth:`~.BatchSchedulerExecutor.get_status_command`.""" - # we're not really using job arrays, so this is equivalent to the job ID. However, if # we were to use arrays, this would return one ID for the entire array rather than # listing each element of the array independently From c6ff2677cad4483645fd8c9f7b0c5baa434c3877 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2024 05:26:15 +0000 Subject: [PATCH 32/51] Update filelock requirement from <=3.15,>=3.4 to <=3.15.4,>=3.15.4 Updates the requirements on [filelock](https://github.com/tox-dev/py-filelock) to permit the latest version. - [Release notes](https://github.com/tox-dev/py-filelock/releases) - [Changelog](https://github.com/tox-dev/filelock/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/py-filelock/compare/3.4.0...3.15.4) --- updated-dependencies: - dependency-name: filelock dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index d5018b16..9599db01 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -7,4 +7,4 @@ pytest >= 6.2.0 requests >= 2.25.1 pytest-cov pytest-timeout -filelock >= 3.4, <= 3.15 +filelock <= 3.15.4, >= 3.15.4 From 486df412b28818a277c754e41b51041138f7fb03 Mon Sep 17 00:00:00 2001 From: Mihael Hategan Date: Sun, 30 Jun 2024 22:30:30 -0700 Subject: [PATCH 33/51] Update requirements-tests.txt --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 9599db01..b3c3b428 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -7,4 +7,4 @@ pytest >= 6.2.0 requests >= 2.25.1 pytest-cov pytest-timeout -filelock <= 3.15.4, >= 3.15.4 +filelock >= 3.4, <= 3.15.4 From 9ae441e19d03493a4a9fed5f678bd0745ef51cb7 Mon Sep 17 00:00:00 2001 From: Mihael Hategan Date: Sun, 30 Jun 2024 22:33:26 -0700 Subject: [PATCH 34/51] Update requirements-tests.txt --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index b3c3b428..863c4f25 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -7,4 +7,4 @@ pytest >= 6.2.0 requests >= 2.25.1 pytest-cov pytest-timeout -filelock >= 3.4, <= 3.15.4 +filelock >= 3.4, < 3.16 From ad71427343fbda9c5f8c994ff2d17dcb62313c65 Mon Sep 17 00:00:00 2001 From: hategan Date: Mon, 8 Jul 2024 15:43:44 -0700 Subject: [PATCH 35/51] The "comment" field is not necessarily present for all states. --- src/psij/executors/batch/pbs_base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/psij/executors/batch/pbs_base.py b/src/psij/executors/batch/pbs_base.py index 2bbe0c04..0183b06b 100644 --- a/src/psij/executors/batch/pbs_base.py +++ b/src/psij/executors/batch/pbs_base.py @@ -138,7 +138,10 @@ def parse_status_output(self, exit_code: int, out: str) -> Dict[str, JobStatus]: elif 'Exit_status' in job_report and job_report['Exit_status'] != 0: state = JobState.FAILED - msg = job_report["comment"] + try: + msg = job_report['comment'] + except KeyError: + msg = None r[native_id] = JobStatus(state, message=msg) return r From 9e7e31fef1fa065756c1a56b7828032d423a83e7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Sep 2024 05:32:21 +0000 Subject: [PATCH 36/51] Update filelock requirement from <3.16,>=3.4 to >=3.4,<3.17 Updates the requirements on [filelock](https://github.com/tox-dev/py-filelock) to permit the latest version. - [Release notes](https://github.com/tox-dev/py-filelock/releases) - [Changelog](https://github.com/tox-dev/filelock/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/py-filelock/compare/3.4.0...3.16.0) --- updated-dependencies: - dependency-name: filelock dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 863c4f25..36cf4553 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -7,4 +7,4 @@ pytest >= 6.2.0 requests >= 2.25.1 pytest-cov pytest-timeout -filelock >= 3.4, < 3.16 +filelock >= 3.4, < 3.17 From 7eb68927ed79c2806491249f45156569d8d9e84e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Oct 2024 06:04:48 +0000 Subject: [PATCH 37/51] Update psutil requirement from <=6.0,>=5.9 to >=5.9,<=6.1.0 Updates the requirements on [psutil](https://github.com/giampaolo/psutil) to permit the latest version. - [Changelog](https://github.com/giampaolo/psutil/blob/master/HISTORY.rst) - [Commits](https://github.com/giampaolo/psutil/compare/release-5.9.0...release-6.1.0) --- updated-dependencies: - dependency-name: psutil dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2e07c81e..e3f68f7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -psutil >=5.9, <=6.0 +psutil >=5.9, <=6.1.0 pystache>=0.6.0 typeguard>=3.0.1 typing-compat From 7d651f73f6c389b2b5c5f17d2cef5b2395cd0e9a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Nov 2024 06:05:19 +0000 Subject: [PATCH 38/51] Update packaging requirement from <=24.1,>=24.0 to >=24.0,<=24.2 Updates the requirements on [packaging](https://github.com/pypa/packaging) to permit the latest version. - [Release notes](https://github.com/pypa/packaging/releases) - [Changelog](https://github.com/pypa/packaging/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pypa/packaging/compare/24.0...24.2) --- updated-dependencies: - dependency-name: packaging dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e3f68f7a..f6ea94b2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ psutil >=5.9, <=6.1.0 pystache>=0.6.0 typeguard>=3.0.1 typing-compat -packaging >= 24.0, <= 24.1 +packaging >= 24.0, <= 24.2 From 3c4401bf62d89414584284a52e5103e8af36f84a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2024 05:55:10 +0000 Subject: [PATCH 39/51] Update psutil requirement from <=6.1.0,>=5.9 to >=5.9,<=6.1.1 Updates the requirements on [psutil](https://github.com/giampaolo/psutil) to permit the latest version. - [Changelog](https://github.com/giampaolo/psutil/blob/master/HISTORY.rst) - [Commits](https://github.com/giampaolo/psutil/compare/release-5.9.0...release-6.1.1) --- updated-dependencies: - dependency-name: psutil dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f6ea94b2..7656d0f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -psutil >=5.9, <=6.1.0 +psutil >=5.9, <=6.1.1 pystache>=0.6.0 typeguard>=3.0.1 typing-compat From 41d4ee7c1b7c245be37d38e60df5777d904e4029 Mon Sep 17 00:00:00 2001 From: hategan Date: Thu, 9 Jan 2025 15:19:53 -0800 Subject: [PATCH 40/51] On Frontier, the node count is mandatory. We should probably discuss having one node as a default and specify that in the spec. In the mean time, add one node as default for slurm. --- src/psij/executors/batch/slurm/slurm.mustache | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/psij/executors/batch/slurm/slurm.mustache b/src/psij/executors/batch/slurm/slurm.mustache index b044ba27..962bc2ab 100644 --- a/src/psij/executors/batch/slurm/slurm.mustache +++ b/src/psij/executors/batch/slurm/slurm.mustache @@ -38,6 +38,9 @@ #SBATCH --cpus-per-task={{.}} {{/cpu_cores_per_process}} {{/job.spec.resources}} +{{^job.spec.resources}} +#SBATCH --nodes=1 +{{/job.spec.resources}} {{#formatted_job_duration}} #SBATCH --time={{.}} From 8fe07286c272fad4765025f96a76a94cfea691e8 Mon Sep 17 00:00:00 2001 From: hategan Date: Thu, 9 Jan 2025 18:45:46 -0800 Subject: [PATCH 41/51] Set default resources on submit and use calculated resource numbers in templates. --- src/psij/executors/batch/cobalt/cobalt.mustache | 8 ++++---- src/psij/executors/batch/lsf/lsf.mustache | 8 ++++---- src/psij/executors/batch/slurm/slurm.mustache | 15 ++++++--------- src/psij/job_executor.py | 6 ++++++ tests/plugins1/_batch_test/test/test.mustache | 4 ++-- 5 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/psij/executors/batch/cobalt/cobalt.mustache b/src/psij/executors/batch/cobalt/cobalt.mustache index 8281ed3e..90d9b73c 100644 --- a/src/psij/executors/batch/cobalt/cobalt.mustache +++ b/src/psij/executors/batch/cobalt/cobalt.mustache @@ -6,12 +6,12 @@ #COBALT --cwd={{.}} {{/job.spec.directory}} {{#job.spec.resources}} - {{#node_count}} + {{#computed_node_count}} #COBALT --nodecount={{.}} - {{/node_count}} - {{#process_count}} + {{/computed_node_count}} + {{#computed_process_count}} #COBALT --proccount={{.}} - {{/process_count}} + {{/computed_process_count}} {{/job.spec.resources}} {{#formatted_job_duration}} #COBALT --time={{duration}} diff --git a/src/psij/executors/batch/lsf/lsf.mustache b/src/psij/executors/batch/lsf/lsf.mustache index a616ad37..aaa5f319 100644 --- a/src/psij/executors/batch/lsf/lsf.mustache +++ b/src/psij/executors/batch/lsf/lsf.mustache @@ -20,13 +20,13 @@ {{#job.spec.resources}} - {{#node_count}} + {{#computed_node_count}} #BSUB -nnodes {{.}} - {{/node_count}} + {{/computed_node_count}} - {{#process_count}} + {{#computed_process_count}} #BSUB -n {{.}} - {{/process_count}} + {{/computed_process_count}} {{#gpu_cores_per_process}} #BSUB -gpu num={{.}}/task diff --git a/src/psij/executors/batch/slurm/slurm.mustache b/src/psij/executors/batch/slurm/slurm.mustache index 962bc2ab..b9862d1b 100644 --- a/src/psij/executors/batch/slurm/slurm.mustache +++ b/src/psij/executors/batch/slurm/slurm.mustache @@ -14,17 +14,17 @@ #SBATCH --exclusive {{/exclusive_node_use}} - {{#node_count}} + {{#computed_node_count}} #SBATCH --nodes={{.}} - {{/node_count}} + {{/computed_node_count}} - {{#process_count}} + {{#computed_process_count}} #SBATCH --ntasks={{.}} - {{/process_count}} + {{/computed_process_count}} - {{#processes_per_node}} + {{#computed_ppn}} #SBATCH --ntasks-per-node={{.}} - {{/processes_per_node}} + {{/computed_ppn}} {{#gpu_cores_per_process}} #SBATCH --gpus-per-task={{.}} @@ -38,9 +38,6 @@ #SBATCH --cpus-per-task={{.}} {{/cpu_cores_per_process}} {{/job.spec.resources}} -{{^job.spec.resources}} -#SBATCH --nodes=1 -{{/job.spec.resources}} {{#formatted_job_duration}} #SBATCH --time={{.}} diff --git a/src/psij/job_executor.py b/src/psij/job_executor.py index 9f686b73..286fc315 100644 --- a/src/psij/job_executor.py +++ b/src/psij/job_executor.py @@ -14,11 +14,15 @@ from psij.job_executor_config import JobExecutorConfig from psij.job_launcher import Launcher from psij.job_spec import JobSpec +from psij.resource_spec import ResourceSpecV1 logger = logging.getLogger(__name__) +_DEFAULT_RESOURCES = ResourceSpecV1() + + class JobExecutor(ABC): """An abstract base class for all JobExecutor implementations.""" @@ -92,6 +96,8 @@ def _check_job(self, job: Job) -> JobSpec: spec = job.spec if not spec: raise InvalidJobException('Missing specification') + if not spec.resources: + spec.resources = _DEFAULT_RESOURCES if __debug__: if spec.environment is not None: diff --git a/tests/plugins1/_batch_test/test/test.mustache b/tests/plugins1/_batch_test/test/test.mustache index bfd09a45..0759e0af 100644 --- a/tests/plugins1/_batch_test/test/test.mustache +++ b/tests/plugins1/_batch_test/test/test.mustache @@ -9,9 +9,9 @@ cd "{{.}}" export PSIJ_TEST_BATCH_EXEC_COUNT=1 {{#job.spec.resources}} - {{#process_count}} + {{#computed_process_count}} export PSIJ_TEST_BATCH_EXEC_COUNT={{.}} - {{/process_count}} + {{/computed_process_count}} {{/job.spec.resources}} {{#job.spec.attributes}} From acaa1627c3e85d2bffee32d9affaa2add11fd6e6 Mon Sep 17 00:00:00 2001 From: hategan Date: Fri, 10 Jan 2025 12:43:35 -0800 Subject: [PATCH 42/51] "computed_ppn" is not a thing. --- src/psij/executors/batch/slurm/slurm.mustache | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/psij/executors/batch/slurm/slurm.mustache b/src/psij/executors/batch/slurm/slurm.mustache index b9862d1b..68e86271 100644 --- a/src/psij/executors/batch/slurm/slurm.mustache +++ b/src/psij/executors/batch/slurm/slurm.mustache @@ -22,9 +22,9 @@ #SBATCH --ntasks={{.}} {{/computed_process_count}} - {{#computed_ppn}} + {{#computed_processes_per_node}} #SBATCH --ntasks-per-node={{.}} - {{/computed_ppn}} + {{/computed_processes_per_node}} {{#gpu_cores_per_process}} #SBATCH --gpus-per-task={{.}} From 9e2354d44f505acade95416f39d4bfa4a989a4ca Mon Sep 17 00:00:00 2001 From: hategan Date: Fri, 10 Jan 2025 14:16:07 -0800 Subject: [PATCH 43/51] Set `exclusive_node_use` default to `False` (for some reason it was `True`) and add a test (for slurm only at this time) to ensure that the corresponding parameter is not generated in the submit script. --- src/psij/resource_spec.py | 2 +- tests/test_executor.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/psij/resource_spec.py b/src/psij/resource_spec.py index 68aa680f..7b565fe4 100644 --- a/src/psij/resource_spec.py +++ b/src/psij/resource_spec.py @@ -54,7 +54,7 @@ def __init__(self, node_count: Optional[int] = None, processes_per_node: Optional[int] = None, cpu_cores_per_process: Optional[int] = None, gpu_cores_per_process: Optional[int] = None, - exclusive_node_use: bool = True) -> None: + exclusive_node_use: bool = False) -> None: """ Some of the properties of this class are constrained. Specifically, `process_count = node_count * processes_per_node`. Specifying all constrained properties diff --git a/tests/test_executor.py b/tests/test_executor.py index 0cc8b510..b940b40a 100644 --- a/tests/test_executor.py +++ b/tests/test_executor.py @@ -224,3 +224,16 @@ def test_submit_script_generation(exec_name: str) -> None: lambda k, v: setattr(attrs, k, v)) _check_str_attrs(ex, job, [prefix + '.cust_attr1', prefix + '.cust_attr2'], lambda k, v: c_attrs.__setitem__(k, v)) + + +def test_resource_generation1() -> None: + res = ResourceSpecV1() + spec = JobSpec('/bin/date', resources=res) + job = Job(spec=spec) + ex = JobExecutor.get_instance('slurm') + with TemporaryFile(mode='w+') as f: + ex.generate_submit_script(job, ex._create_script_context(job), f) + f.seek(0) + contents = f.read() + if contents.find('--exclusive') != -1: + pytest.fail('Spurious exclusive flag') From b0d3253c7a2d65e62db49602b914a74915bf822c Mon Sep 17 00:00:00 2001 From: hategan Date: Fri, 10 Jan 2025 14:18:12 -0800 Subject: [PATCH 44/51] Typechecking --- tests/test_executor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_executor.py b/tests/test_executor.py index b940b40a..af2bbd69 100644 --- a/tests/test_executor.py +++ b/tests/test_executor.py @@ -231,6 +231,7 @@ def test_resource_generation1() -> None: spec = JobSpec('/bin/date', resources=res) job = Job(spec=spec) ex = JobExecutor.get_instance('slurm') + assert isinstance(ex, BatchSchedulerExecutor) with TemporaryFile(mode='w+') as f: ex.generate_submit_script(job, ex._create_script_context(job), f) f.seek(0) From 82967484089cb3712fb95e50f959d887fc33356e Mon Sep 17 00:00:00 2001 From: hategan Date: Wed, 15 Jan 2025 13:01:38 -0800 Subject: [PATCH 45/51] Removed dependency on typing_compat and bumped the minimum python version to 3.8 --- .github/workflows/python-package.yml | 2 +- requirements.txt | 1 - setup.py | 2 +- src/psij/serialize.py | 15 +++++++-------- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 74dab345..70d66fa9 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.7, 3.8, 3.9, 3.10] + python-version: [3.8, 3.9, 3.10, 3.11, 3.12] runs-on: ubuntu-latest steps: diff --git a/requirements.txt b/requirements.txt index 7656d0f8..92185d2f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ psutil >=5.9, <=6.1.1 pystache>=0.6.0 typeguard>=3.0.1 -typing-compat packaging >= 24.0, <= 24.2 diff --git a/setup.py b/setup.py index c9df126d..d9f3ee7f 100644 --- a/setup.py +++ b/setup.py @@ -42,5 +42,5 @@ }, install_requires=install_requires, - python_requires='>=3.7' + python_requires='>=3.8' ) diff --git a/src/psij/serialize.py b/src/psij/serialize.py index 2cab84da..80ec771b 100644 --- a/src/psij/serialize.py +++ b/src/psij/serialize.py @@ -6,7 +6,6 @@ from io import StringIO, TextIOBase from pathlib import Path from typing import Optional, Dict, Union, List, IO, AnyStr, TextIO -import typing_compat from psij import ResourceSpec from psij.job_attributes import JobAttributes @@ -135,12 +134,12 @@ def _from_psij_object(self, o: Union[JobSpec, JobAttributes, ResourceSpec]) \ def _canonicalize_type(self, t: object) -> object: # generics don't appear to be subclasses of Type, so we can't really use Type for t - origin = typing_compat.get_origin(t) + origin = typing.get_origin(t) if origin == Optional: # Python converts Optional[T] to Union[T, None], so this shouldn't happen - return typing_compat.get_args(t)[0] + return typing.get_args(t)[0] elif origin == Union: - args = typing_compat.get_args(t) + args = typing.get_args(t) if args[0] == NoneType: return args[1] elif args[1] == NoneType: @@ -171,10 +170,10 @@ def _from_object(self, o: object, t: object) -> object: else: if t == Union[str, Path] or t == Optional[Union[str, Path]]: return str(o) - if typing_compat.get_origin(t) == dict: + if typing.get_origin(t) == dict: assert isinstance(o, dict) return self._from_dict(o) - if typing_compat.get_origin(t) == list: + if typing.get_origin(t) == list: assert isinstance(o, list) return self._from_list(o) raise ValueError('Cannot convert type "%s".' % t) @@ -249,10 +248,10 @@ def _to_object(self, s: object, t: object) -> object: if t == Union[str, Path] or t == Optional[Union[str, Path]]: assert isinstance(s, str) return Path(s) - if typing_compat.get_origin(t) == dict: + if typing.get_origin(t) == dict: assert isinstance(s, dict) return self._to_dict(s) - if typing_compat.get_origin(t) == list: + if typing.get_origin(t) == list: assert isinstance(s, list) return self._to_list(s) raise ValueError('Cannot convert type "%s".' % t) From d303a8d235420dcf26be363b07942bf9eaa75069 Mon Sep 17 00:00:00 2001 From: hategan Date: Wed, 15 Jan 2025 14:59:54 -0800 Subject: [PATCH 46/51] Removed more typing_compat references. --- .github/dependabot.yml | 1 - .mypy | 3 --- 2 files changed, 4 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 74abab25..ed04926b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -15,6 +15,5 @@ updates: - dependency-name: "psutil" - dependency-name: "pystache" - dependency-name: "typeguard" - - dependency-name: "typing-compat" - dependency-name: "packaging" diff --git a/.mypy b/.mypy index 131f5287..b374bf53 100644 --- a/.mypy +++ b/.mypy @@ -16,6 +16,3 @@ ignore_missing_imports = True [mypy-pystache.*] ignore_missing_imports = True - -[mypy-typing_compat.*] -ignore_missing_imports = True \ No newline at end of file From 538220a7a9337141af99f893a3198d9af92ec8fa Mon Sep 17 00:00:00 2001 From: hategan Date: Wed, 15 Jan 2025 15:03:43 -0800 Subject: [PATCH 47/51] Updated web/docs to reflect python >= 3.8. --- QuickStart.md | 4 ++-- docs/getting_started.rst | 2 +- web/install.html | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/QuickStart.md b/QuickStart.md index 8a1fcaaa..ce4de4c8 100644 --- a/QuickStart.md +++ b/QuickStart.md @@ -7,7 +7,7 @@ This document will guide you through the install procedure and your first Hello - [Hello World example](#hello-world) ## Requirements -- python3.7+ +- python3.8+ ## Install PSI/J @@ -35,7 +35,7 @@ Install PSI/J from the GitHub repository: ## Hello World **Requirements** -- python3.7 +- python3.8 - Job executor, e.g. Slurm in this example **Steps** diff --git a/docs/getting_started.rst b/docs/getting_started.rst index feaba64c..caec7816 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -10,7 +10,7 @@ or from source. Requirements ^^^^^^^^^^^^ -The only requirements are Python 3.7+ and pip, which almost always +The only requirements are Python 3.8+ and pip, which almost always comes with Python. Install from PIP diff --git a/web/install.html b/web/install.html index b9b5ee11..60344e73 100644 --- a/web/install.html +++ b/web/install.html @@ -13,7 +13,7 @@

Installing PSI/J The psij-python library has the following requirements:
    -
  • Python 3.7 or later +
  • Python 3.8 or later
  • Ubuntu 16.04 or later (or an equivalent Linux distribution)
  • OS X 10.10 or later
From 0cdcf01e867ab8a6df1faf3043e41b7c5e1eb554 Mon Sep 17 00:00:00 2001 From: hategan Date: Wed, 15 Jan 2025 15:04:55 -0800 Subject: [PATCH 48/51] One more --- QuickStart.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/QuickStart.md b/QuickStart.md index ce4de4c8..7b610c90 100644 --- a/QuickStart.md +++ b/QuickStart.md @@ -13,7 +13,7 @@ This document will guide you through the install procedure and your first Hello If you have conda installed you might want to start from a fresh environment. This part is not installing PSI/J but setting up a new environment with the specified python version: -1. `conda create -n psij python=3.7` +1. `conda create -n psij python=3.8` 2. `conda activate psij` From fbfccc33cb6841858dd937564c81ca283596d8ff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 05:12:35 +0000 Subject: [PATCH 49/51] Update filelock requirement from <3.17,>=3.4 to >=3.4,<3.18 Updates the requirements on [filelock](https://github.com/tox-dev/py-filelock) to permit the latest version. - [Release notes](https://github.com/tox-dev/py-filelock/releases) - [Changelog](https://github.com/tox-dev/filelock/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/py-filelock/compare/3.4.0...3.17.0) --- updated-dependencies: - dependency-name: filelock dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 36cf4553..cf9ed31a 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -7,4 +7,4 @@ pytest >= 6.2.0 requests >= 2.25.1 pytest-cov pytest-timeout -filelock >= 3.4, < 3.17 +filelock >= 3.4, < 3.18 From 65a5c83c944c234f07672c14c998c2edeb17a8bc Mon Sep 17 00:00:00 2001 From: hategan Date: Mon, 10 Feb 2025 13:21:14 -0800 Subject: [PATCH 50/51] Keep a weakref from the QueuePollThread to the executor to avoid circular references and shut down the QueuePollThread when the executor is garbage collected. --- .../batch/batch_scheduler_executor.py | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/src/psij/executors/batch/batch_scheduler_executor.py b/src/psij/executors/batch/batch_scheduler_executor.py index 3a7cc327..20ba5764 100644 --- a/src/psij/executors/batch/batch_scheduler_executor.py +++ b/src/psij/executors/batch/batch_scheduler_executor.py @@ -3,6 +3,7 @@ import subprocess import time import traceback +import weakref from abc import abstractmethod from datetime import timedelta from pathlib import Path @@ -639,7 +640,8 @@ def __init__(self, name: str, config: BatchSchedulerExecutorConfig, self.name = name self.daemon = True self.config = config - self.executor = executor + self.done = False + self.executor = weakref.ref(executor, self._stop) # native_id -> job self._jobs: Dict[str, List[Job]] = {} # counts consecutive errors while invoking qstat or equivalent @@ -647,26 +649,32 @@ def __init__(self, name: str, config: BatchSchedulerExecutorConfig, self._jobs_lock = RLock() def run(self) -> None: - logger.debug('Executor %s: queue poll thread started', self.executor) + logger.debug('Executor %s: queue poll thread started', self.executor()) time.sleep(self.config.initial_queue_polling_delay) - while True: + while not self.done: self._poll() time.sleep(self.config.queue_polling_interval) + def _stop(self, exec: object) -> None: + self.done = True + def _poll(self) -> None: + exec = self.executor() + if exec is None: + return with self._jobs_lock: if len(self._jobs) == 0: return jobs_copy = dict(self._jobs) logger.info('Polling for %s jobs', len(jobs_copy)) try: - out = self.executor._run_command(self.executor.get_status_command(jobs_copy.keys())) + if exec: + out = exec._run_command(exec.get_status_command(jobs_copy.keys())) except subprocess.CalledProcessError as ex: out = ex.output exit_code = ex.returncode except Exception as ex: - self._handle_poll_error(True, - ex, + self._handle_poll_error(exec, True, ex, f'Failed to poll for job status: {traceback.format_exc()}') return else: @@ -674,10 +682,9 @@ def _poll(self) -> None: self._poll_error_count = 0 logger.debug('Output from status command: %s', out) try: - status_map = self.executor.parse_status_output(exit_code, out) + status_map = exec.parse_status_output(exit_code, out) except Exception as ex: - self._handle_poll_error(False, - ex, + self._handle_poll_error(exec, False, ex, f'Failed to poll for job status: {traceback.format_exc()}') return try: @@ -689,13 +696,13 @@ def _poll(self) -> None: message='Failed to update job status: %s' % traceback.format_exc()) for job in job_list: - self.executor._set_job_status(job, status) + exec._set_job_status(job, status) if status.state.final: with self._jobs_lock: del self._jobs[native_id] except Exception as ex: msg = traceback.format_exc() - self._handle_poll_error(True, ex, 'Error updating job statuses {}'.format(msg)) + self._handle_poll_error(exec, True, ex, 'Error updating job statuses {}'.format(msg)) def _get_job_status(self, native_id: str, status_map: Dict[str, JobStatus]) -> JobStatus: if native_id in status_map: @@ -703,7 +710,8 @@ def _get_job_status(self, native_id: str, status_map: Dict[str, JobStatus]) -> J else: return JobStatus(JobState.COMPLETED) - def _handle_poll_error(self, immediate: bool, ex: Exception, msg: str) -> None: + def _handle_poll_error(self, exec: BatchSchedulerExecutor, immediate: bool, ex: Exception, + msg: str) -> None: logger.warning('Polling error: %s', msg) self._poll_error_count += 1 if immediate or (self._poll_error_count > self.config.queue_polling_error_threshold): @@ -720,7 +728,7 @@ def _handle_poll_error(self, immediate: bool, ex: Exception, msg: str) -> None: self._jobs.clear() for job_list in jobs_copy.values(): for job in job_list: - self.executor._set_job_status(job, JobStatus(JobState.FAILED, message=msg)) + exec._set_job_status(job, JobStatus(JobState.FAILED, message=msg)) def register_job(self, job: Job) -> None: assert job.native_id From 421d1877ebcb326b8fc689c00d1a0e2f4bcbe9f3 Mon Sep 17 00:00:00 2001 From: hategan Date: Thu, 13 Feb 2025 00:38:45 -0800 Subject: [PATCH 51/51] We already check for exec not being null earlier. --- src/psij/executors/batch/batch_scheduler_executor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/psij/executors/batch/batch_scheduler_executor.py b/src/psij/executors/batch/batch_scheduler_executor.py index 20ba5764..5a211030 100644 --- a/src/psij/executors/batch/batch_scheduler_executor.py +++ b/src/psij/executors/batch/batch_scheduler_executor.py @@ -668,8 +668,7 @@ def _poll(self) -> None: jobs_copy = dict(self._jobs) logger.info('Polling for %s jobs', len(jobs_copy)) try: - if exec: - out = exec._run_command(exec.get_status_command(jobs_copy.keys())) + out = exec._run_command(exec.get_status_command(jobs_copy.keys())) except subprocess.CalledProcessError as ex: out = ex.output exit_code = ex.returncode