Merge branch 'main' into file_staging

.github/dependabot.yml

@@ -0,0 +1,19 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"
+    versioning-strategy: auto
+    allow:
+      - dependency-name: "filelock"
+      - dependency-name: "psutil"
+      - dependency-name: "pystache"
+      - dependency-name: "typeguard"
+      - dependency-name: "packaging"
+

.github/workflows/codespell.yml

@@ -15,3 +15,5 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: codespell-project/actions-codespell@master
+        with:
+          ignore_words_list: assertIn

.github/workflows/python-package.yml

@@ -14,7 +14,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.7, 3.8, 3.9, 3.10]
+        python-version: [3.8, 3.9, 3.10, 3.11, 3.12]
     runs-on: ubuntu-latest
 
     steps:

.gitignore

@@ -20,3 +20,4 @@ build/
 .packages/
 docs/.web-build
 web-build/
+.packages/

.mypy

@@ -16,6 +16,3 @@ ignore_missing_imports = True
 
 [mypy-pystache.*]
 ignore_missing_imports = True
-
-[mypy-typing_compat.*]
-ignore_missing_imports = True

MANIFEST.in

@@ -0,0 +1 @@
+include requirements.txt

QuickStart.md

@@ -7,13 +7,13 @@ This document will guide you through the install procedure and your first Hello
 - [Hello World example](#hello-world)
 
 ## Requirements
-- python3.7+
+- python3.8+
 
 ## Install PSI/J
 
 If you have conda installed you might want to start from a fresh environment. This part is not installing PSI/J but setting up a new environment with the specified python version:
 
-1. `conda create -n psij python=3.7`
+1. `conda create -n psij python=3.8`
 2. `conda activate psij`
 
 
@@ -35,7 +35,7 @@ Install PSI/J from the GitHub repository:
 ## Hello World
 
 **Requirements**
-- python3.7
+- python3.8
 - Job executor, e.g. Slurm in this example
 
 **Steps**
@@ -58,7 +58,7 @@ def make_job():
     spec.arguments = ['HELLO WORLD!']
 
     # set project name if no default is specified
-    # spec.attributes.project_name = <PROJECT_NAME>
+    # spec.attributes.account = <PROJECT_NAME>
 
     # set queue if no default is specified
     # spec.attributes.queue_name = <QUEUE_NAME>

docs/_static/extras.js

@@ -86,8 +86,8 @@ function detectAll(selectorType) {
             else if (text == "queue_name") {
                 $(this).text("QUEUE_NAME");
             }
-            else if (text == "project_name") {
-                $(this).text("PROJECT_NAME");
+            else if (text == "account") {
+                $(this).text("ACCOUNT");
             }
         }
         if (text == '_get_executor_instance') {

docs/conf.py

@@ -31,7 +31,8 @@
 autodoc_mock_imports = ['flux']
 nitpick_ignore = [
     ('py:class', 'distutils.version.StrictVersion'),
-    ('py:class', 'distutils.version.Version')
+    ('py:class', 'distutils.version.Version'),
+    ('py:class', 'packaging.version.Version')
 ]
 
 if web_docs:

docs/development/tutorial_add_executor.rst

@@ -68,11 +68,11 @@ Create a simple BatchSchedulerExecutor subclass that does nothing new in `psijpb
 
 and create a descriptor file to tell PSI/J about this, ``psij-descriptors/pbspro.py``::
 
-  from distutils.version import StrictVersion
+  from packaging.version import Version
 
   from psij._descriptor import _Descriptor
 
-  __PSI_J_EXECUTORS__ = [_Descriptor(name='pbspro', version=StrictVersion('0.0.1'),
+  __PSI_J_EXECUTORS__ = [_Descriptor(name='pbspro', version=Version('0.0.1'),
                                      cls='psijpbs.pbspro.PBSProJobExecutor')]
 
 Now, run the test suite. It should fail with an error reporting that the resource manager specific methods of BatchSchedulerExecutor have not been implemented::

docs/getting_started.rst

@@ -10,7 +10,7 @@ or from source.
 Requirements
 ^^^^^^^^^^^^
 
-The only requirements are Python 3.7+ and pip, which almost always
+The only requirements are Python 3.8+ and pip, which almost always
 comes with Python.
 
 Install from PIP

docs/user_guide.rst

@@ -335,7 +335,7 @@ and an instance of the
     :lines: 10-18,21-22
 
 where `QUEUE_NAME` is the LRM queue where the job should be sent and
-`PROJECT_NAME` is a project/account that may need to be specified for
+`ACCOUNT` is a project/account that may need to be specified for
 accounting purposes. These values generally depend on the system and allocation
 being used.
 

release.sh

@@ -48,6 +48,12 @@ if [ "$LAST" != "$TARGET_VERSION" ]; then
     error "Version $TARGET_VERSION is lower than the latest tagged version ($LAST)."
 fi
 
+if which twine >/dev/null; then
+	echo "Found twine"
+else
+	error "Twine was not found. Please install it and then re-run this script"
+fi
+
 echo "This will tag and release psij-python to version $TARGET_VERSION."
 echo -n "Type 'yes' if you want to continue: "
 read REPLY

requirements-tests.txt

@@ -6,4 +6,5 @@
 pytest >= 6.2.0
 requests >= 2.25.1
 pytest-cov
-pytest-timeout
+pytest-timeout
+filelock >= 3.4, < 3.18

requirements.txt

@@ -1,5 +1,4 @@
-filelock~=3.4
-psutil~=5.9
+psutil >=5.9, <=6.1.1
 pystache>=0.6.0
-typeguard~=2.12
-typing-compat
+typeguard>=3.0.1
+packaging >= 24.0, <= 24.2

setup.py

@@ -12,8 +12,7 @@
         name='psij-python',
         version=VERSION,
 
-        description='''This is an implementation of the PSI/J (Portable Submission Interface for Jobs)
-        specification.''',
+        description='''This is an implementation of the PSI/J (Portable Submission Interface for Jobs) specification.''',
 
         author='The ExaWorks Team',
         author_email='[email protected]',
@@ -43,5 +42,5 @@
         },
 
         install_requires=install_requires,
-        python_requires='>=3.7'
+        python_requires='>=3.8'
     )

src/psij-descriptors/aprun_descriptor.py

@@ -1,8 +1,7 @@
-from distutils.version import StrictVersion
-
+from packaging.version import Version
 from psij.descriptor import Descriptor
 
 __PSI_J_LAUNCHERS__ = [
-    Descriptor(name='aprun', version=StrictVersion('0.0.1'),
+    Descriptor(name='aprun', version=Version('0.0.1'),
                cls='psij.launchers.aprun.AprunLauncher'),
 ]

src/psij-descriptors/cobalt_descriptor.py

@@ -1,7 +1,6 @@
-from distutils.version import StrictVersion
-
+from packaging.version import Version
 from psij.descriptor import Descriptor
 
 
-__PSI_J_EXECUTORS__ = [Descriptor(name="cobalt", nice_name='Cobalt', version=StrictVersion("0.2.0"),
+__PSI_J_EXECUTORS__ = [Descriptor(name="cobalt", nice_name='Cobalt', version=Version("0.2.0"),
                                   cls='psij.executors.batch.cobalt.CobaltJobExecutor')]

src/psij-descriptors/core_descriptors.py

@@ -1,16 +1,16 @@
-from distutils.version import StrictVersion
+from packaging.version import Version
 from psij.descriptor import Descriptor
 
 __PSI_J_EXECUTORS__ = [
-    Descriptor(name='local', nice_name='Local', version=StrictVersion('0.2.0'),
+    Descriptor(name='local', nice_name='Local', version=Version('0.2.0'),
                cls='psij.executors.local.LocalJobExecutor')
 ]
 
 __PSI_J_LAUNCHERS__ = [
-    Descriptor(name='single', version=StrictVersion('0.2.0'),
+    Descriptor(name='single', version=Version('0.2.0'),
                cls='psij.launchers.single.SingleLauncher'),
-    Descriptor(name='multiple', version=StrictVersion('0.2.0'),
+    Descriptor(name='multiple', version=Version('0.2.0'),
                cls='psij.launchers.multiple.MultipleLauncher'),
-    Descriptor(name='mpirun', version=StrictVersion('0.2.0'),
+    Descriptor(name='mpirun', version=Version('0.2.0'),
                cls='psij.launchers.mpirun.MPILauncher'),
 ]

src/psij-descriptors/flux_descriptor.py

@@ -1,7 +1,6 @@
-from distutils.version import StrictVersion
-
+from packaging.version import Version
 from psij.descriptor import Descriptor
 
 
-__PSI_J_EXECUTORS__ = [Descriptor(name='flux', nice_name='Flux', version=StrictVersion('0.0.1'),
+__PSI_J_EXECUTORS__ = [Descriptor(name='flux', nice_name='Flux', version=Version('0.0.1'),
                                   cls='psij.executors.flux.FluxJobExecutor')]

src/psij-descriptors/jsrun_descriptor.py

@@ -1,8 +1,7 @@
-from distutils.version import StrictVersion
-
+from packaging.version import Version
 from psij.descriptor import Descriptor
 
 __PSI_J_LAUNCHERS__ = [
-    Descriptor(name='jrun', version=StrictVersion('0.0.1'),
+    Descriptor(name='jrun', version=Version('0.0.1'),
                cls='psij.launchers.jsrun.JsrunLauncher'),
 ]

src/psij-descriptors/lsf_descriptor.py

@@ -1,7 +1,6 @@
-from distutils.version import StrictVersion
-
+from packaging.version import Version
 from psij.descriptor import Descriptor
 
 
-__PSI_J_EXECUTORS__ = [Descriptor(name='lsf', nice_name='LSF', version=StrictVersion('0.2.0'),
+__PSI_J_EXECUTORS__ = [Descriptor(name='lsf', nice_name='LSF', version=Version('0.2.0'),
                                   cls='psij.executors.batch.lsf.LsfJobExecutor')]

src/psij-descriptors/pbs_descriptor.py

@@ -1,11 +1,10 @@
-from distutils.version import StrictVersion
-
+from packaging.version import Version
 from psij.descriptor import Descriptor
 
 
 __PSI_J_EXECUTORS__ = [Descriptor(name='pbs', nice_name='PBS Pro', aliases=['pbspro'],
-                                  version=StrictVersion('0.2.0'),
+                                  version=Version('0.2.0'),
                                   cls='psij.executors.batch.pbs.PBSJobExecutor'),
                        Descriptor(name='pbs_classic', nice_name='PBS Classic', aliases=['torque'],
-                                  version=StrictVersion('0.2.0'),
+                                  version=Version('0.2.0'),
                                   cls='psij.executors.batch.pbs_classic.PBSClassicJobExecutor')]

src/psij-descriptors/rp_descriptor.py

@@ -1,8 +1,7 @@
-from distutils.version import StrictVersion
-
+from packaging.version import Version
 from psij.descriptor import Descriptor
 
 
 __PSI_J_EXECUTORS__ = [Descriptor(name='rp', nice_name='Radical Pilot',
-                                  version=StrictVersion('0.0.1'),
+                                  version=Version('0.0.1'),
                                   cls='psij.executors.rp.RPJobExecutor')]

src/psij-descriptors/slurm_descriptor.py

@@ -1,7 +1,6 @@
-from distutils.version import StrictVersion
-
+from packaging.version import Version
 from psij.descriptor import Descriptor
 
 
-__PSI_J_EXECUTORS__ = [Descriptor(name='slurm', nice_name='Slurm', version=StrictVersion('0.2.0'),
+__PSI_J_EXECUTORS__ = [Descriptor(name='slurm', nice_name='Slurm', version=Version('0.2.0'),
                                   cls='psij.executors.batch.slurm.SlurmJobExecutor')]

src/psij-descriptors/srun_descriptor.py

@@ -1,8 +1,7 @@
-from distutils.version import StrictVersion
-
+from packaging.version import Version
 from psij.descriptor import Descriptor
 
 __PSI_J_LAUNCHERS__ = [
-    Descriptor(name='srun', version=StrictVersion('0.0.1'),
+    Descriptor(name='srun', version=Version('0.0.1'),
                cls='psij.launchers.srun.SrunLauncher'),
 ]

src/psij/_plugins.py

@@ -1,7 +1,7 @@
 import importlib
 import logging
 from bisect import bisect_left
-from distutils.versionpredicate import VersionPredicate
+from packaging.specifiers import SpecifierSet
 from types import ModuleType
 from typing import Tuple, Dict, List, Union, Type, Any, Optional, TypeVar
 
@@ -116,9 +116,9 @@ def _get_plugin_class(name: str, version_constraint: Optional[str], type: str,
     versions = store[name]
     selected = None
     if version_constraint:
-        pred = VersionPredicate('x(' + version_constraint + ')')
+        pred = SpecifierSet(version_constraint)
         for entry in reversed(versions):
-            if pred.satisfied_by(entry.version):
+            if entry.version in pred:
                 selected = entry
     else:
         selected = versions[-1]

src/psij/descriptor.py

@@ -1,6 +1,6 @@
 """Executor/Launcher descriptor module."""
 
-from distutils.version import StrictVersion
+from packaging.version import Version
 from typing import TypeVar, Generic, Optional, Type, List
 
 T = TypeVar('T')
@@ -68,17 +68,17 @@ class Descriptor(object):
 
     .. code-block:: python
 
-        from distutils.version import StrictVersion
+        from packaging.version import Version
         from psij.descriptor import Descriptor
 
         __PSI_J_EXECUTORS__ = [
-            Descriptor(name=<name>, version=StrictVersion(<version_str>),
+            Descriptor(name=<name>, version=Version(<version_str>),
                        cls=<fqn_str>),
             ...
         ]
 
         __PSI_J_LAUNCHERS__ = [
-            Descriptor(name=<name>, version=StrictVersion(<version_str>),
+            Descriptor(name=<name>, version=Version(<version_str>),
                        cls=<fqn_str>),
             ...
         ]
@@ -89,7 +89,7 @@ class name that implements the executor or launcher such as
     `psij.executors.local.LocalJobExecutor`.
     """
 
-    def __init__(self, name: str, version: StrictVersion, cls: str,
+    def __init__(self, name: str, version: Version, cls: str,
                  aliases: Optional[List[str]] = None, nice_name: Optional[str] = None) -> None:
         """
         Parameters