diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..ed04926b --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,19 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" # Location of package manifests + schedule: + interval: "weekly" + versioning-strategy: auto + allow: + - dependency-name: "filelock" + - dependency-name: "psutil" + - dependency-name: "pystache" + - dependency-name: "typeguard" + - dependency-name: "packaging" + diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 36f59c8f..447176bc 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -15,3 +15,5 @@ jobs: steps: - uses: actions/checkout@v2 - uses: codespell-project/actions-codespell@master + with: + ignore_words_list: assertIn diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index edab8dad..135625c3 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.7, 3.8, 3.9, 3.10] + python-version: [3.8, 3.9, 3.10, 3.11, 3.12] runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index 10ba1bba..76944679 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ build/ .packages/ docs/.web-build web-build/ +.packages/ diff --git a/.mypy b/.mypy index 131f5287..b374bf53 100644 --- a/.mypy +++ b/.mypy @@ -16,6 +16,3 @@ ignore_missing_imports = True [mypy-pystache.*] ignore_missing_imports = True - -[mypy-typing_compat.*] -ignore_missing_imports = True \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..f9bd1455 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include requirements.txt diff --git a/QuickStart.md b/QuickStart.md index 0dcea194..7b610c90 100644 --- a/QuickStart.md +++ b/QuickStart.md @@ -7,13 +7,13 @@ This document will guide you through the install procedure and your first Hello - [Hello World example](#hello-world) ## Requirements -- python3.7+ +- python3.8+ ## Install PSI/J If you have conda installed you might want to start from a fresh environment. This part is not installing PSI/J but setting up a new environment with the specified python version: -1. `conda create -n psij python=3.7` +1. `conda create -n psij python=3.8` 2. `conda activate psij` @@ -35,7 +35,7 @@ Install PSI/J from the GitHub repository: ## Hello World **Requirements** -- python3.7 +- python3.8 - Job executor, e.g. Slurm in this example **Steps** @@ -58,7 +58,7 @@ def make_job(): spec.arguments = ['HELLO WORLD!'] # set project name if no default is specified - # spec.attributes.project_name = + # spec.attributes.account = # set queue if no default is specified # spec.attributes.queue_name = diff --git a/docs/_static/extras.js b/docs/_static/extras.js index 010e89c7..e8473a08 100644 --- a/docs/_static/extras.js +++ b/docs/_static/extras.js @@ -86,8 +86,8 @@ function detectAll(selectorType) { else if (text == "queue_name") { $(this).text("QUEUE_NAME"); } - else if (text == "project_name") { - $(this).text("PROJECT_NAME"); + else if (text == "account") { + $(this).text("ACCOUNT"); } } if (text == '_get_executor_instance') { diff --git a/docs/conf.py b/docs/conf.py index 0e3a5a5d..f486bdac 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,7 +31,8 @@ autodoc_mock_imports = ['flux'] nitpick_ignore = [ ('py:class', 'distutils.version.StrictVersion'), - ('py:class', 'distutils.version.Version') + ('py:class', 'distutils.version.Version'), + ('py:class', 'packaging.version.Version') ] if web_docs: diff --git a/docs/development/tutorial_add_executor.rst b/docs/development/tutorial_add_executor.rst index 5643c861..dcf68d4f 100644 --- a/docs/development/tutorial_add_executor.rst +++ b/docs/development/tutorial_add_executor.rst @@ -68,11 +68,11 @@ Create a simple BatchSchedulerExecutor subclass that does nothing new in `psijpb and create a descriptor file to tell PSI/J about this, ``psij-descriptors/pbspro.py``:: - from distutils.version import StrictVersion + from packaging.version import Version from psij._descriptor import _Descriptor - __PSI_J_EXECUTORS__ = [_Descriptor(name='pbspro', version=StrictVersion('0.0.1'), + __PSI_J_EXECUTORS__ = [_Descriptor(name='pbspro', version=Version('0.0.1'), cls='psijpbs.pbspro.PBSProJobExecutor')] Now, run the test suite. It should fail with an error reporting that the resource manager specific methods of BatchSchedulerExecutor have not been implemented:: diff --git a/docs/getting_started.rst b/docs/getting_started.rst index feaba64c..caec7816 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -10,7 +10,7 @@ or from source. Requirements ^^^^^^^^^^^^ -The only requirements are Python 3.7+ and pip, which almost always +The only requirements are Python 3.8+ and pip, which almost always comes with Python. Install from PIP diff --git a/docs/user_guide.rst b/docs/user_guide.rst index f0e63e39..b68cf665 100644 --- a/docs/user_guide.rst +++ b/docs/user_guide.rst @@ -335,7 +335,7 @@ and an instance of the :lines: 10-18,21-22 where `QUEUE_NAME` is the LRM queue where the job should be sent and -`PROJECT_NAME` is a project/account that may need to be specified for +`ACCOUNT` is a project/account that may need to be specified for accounting purposes. These values generally depend on the system and allocation being used. diff --git a/release.sh b/release.sh index 28a8b444..615d1be5 100755 --- a/release.sh +++ b/release.sh @@ -48,6 +48,12 @@ if [ "$LAST" != "$TARGET_VERSION" ]; then error "Version $TARGET_VERSION is lower than the latest tagged version ($LAST)." fi +if which twine >/dev/null; then + echo "Found twine" +else + error "Twine was not found. Please install it and then re-run this script" +fi + echo "This will tag and release psij-python to version $TARGET_VERSION." echo -n "Type 'yes' if you want to continue: " read REPLY diff --git a/requirements-tests.txt b/requirements-tests.txt index aab4fb0b..cf9ed31a 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -6,4 +6,5 @@ pytest >= 6.2.0 requests >= 2.25.1 pytest-cov -pytest-timeout \ No newline at end of file +pytest-timeout +filelock >= 3.4, < 3.18 diff --git a/requirements.txt b/requirements.txt index e552d33b..92185d2f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ -filelock~=3.4 -psutil~=5.9 +psutil >=5.9, <=6.1.1 pystache>=0.6.0 -typeguard~=2.12 -typing-compat +typeguard>=3.0.1 +packaging >= 24.0, <= 24.2 diff --git a/setup.py b/setup.py index d841b2a7..d9f3ee7f 100644 --- a/setup.py +++ b/setup.py @@ -12,8 +12,7 @@ name='psij-python', version=VERSION, - description='''This is an implementation of the PSI/J (Portable Submission Interface for Jobs) - specification.''', + description='''This is an implementation of the PSI/J (Portable Submission Interface for Jobs) specification.''', author='The ExaWorks Team', author_email='hategan@mcs.anl.gov', @@ -43,5 +42,5 @@ }, install_requires=install_requires, - python_requires='>=3.7' + python_requires='>=3.8' ) diff --git a/src/psij-descriptors/aprun_descriptor.py b/src/psij-descriptors/aprun_descriptor.py index c4fb8f26..754f1450 100644 --- a/src/psij-descriptors/aprun_descriptor.py +++ b/src/psij-descriptors/aprun_descriptor.py @@ -1,8 +1,7 @@ -from distutils.version import StrictVersion - +from packaging.version import Version from psij.descriptor import Descriptor __PSI_J_LAUNCHERS__ = [ - Descriptor(name='aprun', version=StrictVersion('0.0.1'), + Descriptor(name='aprun', version=Version('0.0.1'), cls='psij.launchers.aprun.AprunLauncher'), ] diff --git a/src/psij-descriptors/cobalt_descriptor.py b/src/psij-descriptors/cobalt_descriptor.py index eb73a3a1..ea27c601 100644 --- a/src/psij-descriptors/cobalt_descriptor.py +++ b/src/psij-descriptors/cobalt_descriptor.py @@ -1,7 +1,6 @@ -from distutils.version import StrictVersion - +from packaging.version import Version from psij.descriptor import Descriptor -__PSI_J_EXECUTORS__ = [Descriptor(name="cobalt", nice_name='Cobalt', version=StrictVersion("0.2.0"), +__PSI_J_EXECUTORS__ = [Descriptor(name="cobalt", nice_name='Cobalt', version=Version("0.2.0"), cls='psij.executors.batch.cobalt.CobaltJobExecutor')] diff --git a/src/psij-descriptors/core_descriptors.py b/src/psij-descriptors/core_descriptors.py index 5212f204..cf1183a1 100644 --- a/src/psij-descriptors/core_descriptors.py +++ b/src/psij-descriptors/core_descriptors.py @@ -1,16 +1,16 @@ -from distutils.version import StrictVersion +from packaging.version import Version from psij.descriptor import Descriptor __PSI_J_EXECUTORS__ = [ - Descriptor(name='local', nice_name='Local', version=StrictVersion('0.2.0'), + Descriptor(name='local', nice_name='Local', version=Version('0.2.0'), cls='psij.executors.local.LocalJobExecutor') ] __PSI_J_LAUNCHERS__ = [ - Descriptor(name='single', version=StrictVersion('0.2.0'), + Descriptor(name='single', version=Version('0.2.0'), cls='psij.launchers.single.SingleLauncher'), - Descriptor(name='multiple', version=StrictVersion('0.2.0'), + Descriptor(name='multiple', version=Version('0.2.0'), cls='psij.launchers.multiple.MultipleLauncher'), - Descriptor(name='mpirun', version=StrictVersion('0.2.0'), + Descriptor(name='mpirun', version=Version('0.2.0'), cls='psij.launchers.mpirun.MPILauncher'), ] diff --git a/src/psij-descriptors/flux_descriptor.py b/src/psij-descriptors/flux_descriptor.py index 2087fe37..f1046ac6 100644 --- a/src/psij-descriptors/flux_descriptor.py +++ b/src/psij-descriptors/flux_descriptor.py @@ -1,7 +1,6 @@ -from distutils.version import StrictVersion - +from packaging.version import Version from psij.descriptor import Descriptor -__PSI_J_EXECUTORS__ = [Descriptor(name='flux', nice_name='Flux', version=StrictVersion('0.0.1'), +__PSI_J_EXECUTORS__ = [Descriptor(name='flux', nice_name='Flux', version=Version('0.0.1'), cls='psij.executors.flux.FluxJobExecutor')] diff --git a/src/psij-descriptors/jsrun_descriptor.py b/src/psij-descriptors/jsrun_descriptor.py index 756561d7..b9ceb5c4 100644 --- a/src/psij-descriptors/jsrun_descriptor.py +++ b/src/psij-descriptors/jsrun_descriptor.py @@ -1,8 +1,7 @@ -from distutils.version import StrictVersion - +from packaging.version import Version from psij.descriptor import Descriptor __PSI_J_LAUNCHERS__ = [ - Descriptor(name='jrun', version=StrictVersion('0.0.1'), + Descriptor(name='jrun', version=Version('0.0.1'), cls='psij.launchers.jsrun.JsrunLauncher'), ] diff --git a/src/psij-descriptors/lsf_descriptor.py b/src/psij-descriptors/lsf_descriptor.py index c0677ec9..68b19cfd 100644 --- a/src/psij-descriptors/lsf_descriptor.py +++ b/src/psij-descriptors/lsf_descriptor.py @@ -1,7 +1,6 @@ -from distutils.version import StrictVersion - +from packaging.version import Version from psij.descriptor import Descriptor -__PSI_J_EXECUTORS__ = [Descriptor(name='lsf', nice_name='LSF', version=StrictVersion('0.2.0'), +__PSI_J_EXECUTORS__ = [Descriptor(name='lsf', nice_name='LSF', version=Version('0.2.0'), cls='psij.executors.batch.lsf.LsfJobExecutor')] diff --git a/src/psij-descriptors/pbs_descriptor.py b/src/psij-descriptors/pbs_descriptor.py index 29f22d9a..6ee0dbdd 100644 --- a/src/psij-descriptors/pbs_descriptor.py +++ b/src/psij-descriptors/pbs_descriptor.py @@ -1,11 +1,10 @@ -from distutils.version import StrictVersion - +from packaging.version import Version from psij.descriptor import Descriptor __PSI_J_EXECUTORS__ = [Descriptor(name='pbs', nice_name='PBS Pro', aliases=['pbspro'], - version=StrictVersion('0.2.0'), + version=Version('0.2.0'), cls='psij.executors.batch.pbs.PBSJobExecutor'), Descriptor(name='pbs_classic', nice_name='PBS Classic', aliases=['torque'], - version=StrictVersion('0.2.0'), + version=Version('0.2.0'), cls='psij.executors.batch.pbs_classic.PBSClassicJobExecutor')] diff --git a/src/psij-descriptors/rp_descriptor.py b/src/psij-descriptors/rp_descriptor.py index 93468765..afe8f4f8 100644 --- a/src/psij-descriptors/rp_descriptor.py +++ b/src/psij-descriptors/rp_descriptor.py @@ -1,8 +1,7 @@ -from distutils.version import StrictVersion - +from packaging.version import Version from psij.descriptor import Descriptor __PSI_J_EXECUTORS__ = [Descriptor(name='rp', nice_name='Radical Pilot', - version=StrictVersion('0.0.1'), + version=Version('0.0.1'), cls='psij.executors.rp.RPJobExecutor')] diff --git a/src/psij-descriptors/slurm_descriptor.py b/src/psij-descriptors/slurm_descriptor.py index 732f4af6..c9af34dc 100644 --- a/src/psij-descriptors/slurm_descriptor.py +++ b/src/psij-descriptors/slurm_descriptor.py @@ -1,7 +1,6 @@ -from distutils.version import StrictVersion - +from packaging.version import Version from psij.descriptor import Descriptor -__PSI_J_EXECUTORS__ = [Descriptor(name='slurm', nice_name='Slurm', version=StrictVersion('0.2.0'), +__PSI_J_EXECUTORS__ = [Descriptor(name='slurm', nice_name='Slurm', version=Version('0.2.0'), cls='psij.executors.batch.slurm.SlurmJobExecutor')] diff --git a/src/psij-descriptors/srun_descriptor.py b/src/psij-descriptors/srun_descriptor.py index 2f8b4048..f226aad3 100644 --- a/src/psij-descriptors/srun_descriptor.py +++ b/src/psij-descriptors/srun_descriptor.py @@ -1,8 +1,7 @@ -from distutils.version import StrictVersion - +from packaging.version import Version from psij.descriptor import Descriptor __PSI_J_LAUNCHERS__ = [ - Descriptor(name='srun', version=StrictVersion('0.0.1'), + Descriptor(name='srun', version=Version('0.0.1'), cls='psij.launchers.srun.SrunLauncher'), ] diff --git a/src/psij/_plugins.py b/src/psij/_plugins.py index b0b3c863..4b8b9121 100644 --- a/src/psij/_plugins.py +++ b/src/psij/_plugins.py @@ -1,7 +1,7 @@ import importlib import logging from bisect import bisect_left -from distutils.versionpredicate import VersionPredicate +from packaging.specifiers import SpecifierSet from types import ModuleType from typing import Tuple, Dict, List, Union, Type, Any, Optional, TypeVar @@ -116,9 +116,9 @@ def _get_plugin_class(name: str, version_constraint: Optional[str], type: str, versions = store[name] selected = None if version_constraint: - pred = VersionPredicate('x(' + version_constraint + ')') + pred = SpecifierSet(version_constraint) for entry in reversed(versions): - if pred.satisfied_by(entry.version): + if entry.version in pred: selected = entry else: selected = versions[-1] diff --git a/src/psij/descriptor.py b/src/psij/descriptor.py index a52c46f4..16dd3de8 100644 --- a/src/psij/descriptor.py +++ b/src/psij/descriptor.py @@ -1,6 +1,6 @@ """Executor/Launcher descriptor module.""" -from distutils.version import StrictVersion +from packaging.version import Version from typing import TypeVar, Generic, Optional, Type, List T = TypeVar('T') @@ -68,17 +68,17 @@ class Descriptor(object): .. code-block:: python - from distutils.version import StrictVersion + from packaging.version import Version from psij.descriptor import Descriptor __PSI_J_EXECUTORS__ = [ - Descriptor(name=, version=StrictVersion(), + Descriptor(name=, version=Version(), cls=), ... ] __PSI_J_LAUNCHERS__ = [ - Descriptor(name=, version=StrictVersion(), + Descriptor(name=, version=Version(), cls=), ... ] @@ -89,7 +89,7 @@ class name that implements the executor or launcher such as `psij.executors.local.LocalJobExecutor`. """ - def __init__(self, name: str, version: StrictVersion, cls: str, + def __init__(self, name: str, version: Version, cls: str, aliases: Optional[List[str]] = None, nice_name: Optional[str] = None) -> None: """ Parameters diff --git a/src/psij/executors/batch/batch_scheduler_executor.py b/src/psij/executors/batch/batch_scheduler_executor.py index 2e503a66..a0b82ba7 100644 --- a/src/psij/executors/batch/batch_scheduler_executor.py +++ b/src/psij/executors/batch/batch_scheduler_executor.py @@ -1,10 +1,10 @@ import logging import os -import weakref import subprocess import time import traceback +import weakref from abc import abstractmethod from datetime import timedelta from pathlib import Path @@ -641,39 +641,36 @@ def __init__(self, name: str, config: BatchSchedulerExecutorConfig, super().__init__() self.name = name self.daemon = True - # We don't at this time cache executor instances. Even if we did, it may be wise - # to shut down queue polling threads when their executors (the only entities that - # use them) are garbage collected. So we wrap the references to the executor and - # config in a weak ref and exit when the ref becomes invalid. - self.config = weakref.ref(config) - self.executor = weakref.ref(executor) + self.config = config + self.done = False + self.executor = weakref.ref(executor, self._stop) # native_id -> job self._jobs: Dict[str, Set[Job]] = {} # counts consecutive errors while invoking qstat or equivalent self._poll_error_count = 0 self._jobs_lock = RLock() self.status_updater = cast(_StatusUpdater, _StatusUpdater.get_instance()) - self.active = True def run(self) -> None: logger.debug('Executor %s: queue poll thread started', self.executor) - try: - time.sleep(self.get_config().initial_queue_polling_delay) - while self.active: - self._poll() - start = time.time() - now = start - while now - start < self.get_config().queue_polling_interval: - time.sleep(1) - now = time.time() - except StopIteration: - logger.info('Thread %s exiting due to executor collection' % self) - - def stop(self) -> None: - self.active = False + + time.sleep(self.config.initial_queue_polling_delay) + while not self.done: + self._poll() + start = time.time() + now = start + while now - start < self.config.queue_polling_interval: + time.sleep(1) + now = time.time() + logger.info('Thread %s exiting due to executor collection' % self) + + def _stop(self, executor: object) -> None: + self.done = True def _poll(self) -> None: - executor = self.get_executor() + executor = self.executor() + if executor is None: + return with self._jobs_lock: if len(self._jobs) == 0: return @@ -685,8 +682,7 @@ def _poll(self) -> None: out = ex.output exit_code = ex.returncode except Exception as ex: - self._handle_poll_error(True, - ex, + self._handle_poll_error(executor, True, ex, f'Failed to poll for job status: {traceback.format_exc()}') return else: @@ -696,8 +692,7 @@ def _poll(self) -> None: try: status_map = executor.parse_status_output(exit_code, out) except Exception as ex: - self._handle_poll_error(False, - ex, + self._handle_poll_error(executor, False, ex, f'Failed to poll for job status: {traceback.format_exc()}') return try: @@ -708,12 +703,11 @@ def _poll(self) -> None: status = JobStatus(JobState.FAILED, message='Failed to update job status: %s' % traceback.format_exc()) - for job in job_set: executor._set_job_status(job, status) except Exception as ex: msg = traceback.format_exc() - self._handle_poll_error(True, ex, 'Error updating job statuses {}'.format(msg)) + self._handle_poll_error(executor, True, ex, f'Error updating job statuses {msg}') def _get_job_status(self, native_id: str, status_map: Dict[str, JobStatus]) -> JobStatus: if native_id in status_map: @@ -721,10 +715,11 @@ def _get_job_status(self, native_id: str, status_map: Dict[str, JobStatus]) -> J else: return JobStatus(JobState.COMPLETED) - def _handle_poll_error(self, immediate: bool, ex: Exception, msg: str) -> None: + def _handle_poll_error(self, executor: BatchSchedulerExecutor, immediate: bool, ex: Exception, + msg: str) -> None: logger.warning('Polling error: %s', msg) self._poll_error_count += 1 - if immediate or (self._poll_error_count > self.get_config().queue_polling_error_threshold): + if immediate or (self._poll_error_count > self.config.queue_polling_error_threshold): self._poll_error_count = 0 # fail all jobs with self._jobs_lock: @@ -739,11 +734,14 @@ def _handle_poll_error(self, immediate: bool, ex: Exception, msg: str) -> None: for job_set in jobs_copy.values(): for job in job_set: self.unregister_job(job) - self.get_executor()._set_job_status(job, JobStatus(JobState.FAILED, - message=msg)) + executor._set_job_status(job, JobStatus(JobState.FAILED, message=msg)) def register_job(self, job: Job) -> None: - self.status_updater.register_job(job, self.get_executor()) + executor = self.executor() + # This method is only called from the executor. It stands to reason that the + # executor cannot have been GC-ed. + assert executor is not None + self.status_updater.register_job(job, executor) assert job.native_id logger.info('Job %s: registering', job.id) with self._jobs_lock: @@ -766,17 +764,3 @@ def unregister_job(self, job: Job) -> None: # first one being unregistered would already have removed # the dict entry pass - - def get_config(self) -> BatchSchedulerExecutorConfig: - config = self.config() - if config: - return config - else: - raise StopIteration() - - def get_executor(self) -> BatchSchedulerExecutor: - ex = self.executor() - if ex: - return ex - else: - raise StopIteration() diff --git a/src/psij/executors/batch/cobalt/cobalt.mustache b/src/psij/executors/batch/cobalt/cobalt.mustache index eb5af85f..444de0ae 100644 --- a/src/psij/executors/batch/cobalt/cobalt.mustache +++ b/src/psij/executors/batch/cobalt/cobalt.mustache @@ -6,12 +6,12 @@ #COBALT --cwd={{.}} {{/job.spec.directory}} {{#job.spec.resources}} - {{#node_count}} + {{#computed_node_count}} #COBALT --nodecount={{.}} - {{/node_count}} - {{#process_count}} + {{/computed_node_count}} + {{#computed_process_count}} #COBALT --proccount={{.}} - {{/process_count}} + {{/computed_process_count}} {{/job.spec.resources}} {{#formatted_job_duration}} #COBALT --time={{duration}} @@ -25,9 +25,9 @@ {{#reservation_id}} #COBALT --queue={{.}} {{/reservation_id}} - {{#project_name}} + {{#account}} #COBALT --project={{.}} - {{/project_name}} + {{/account}} {{/job.spec.attributes}} {{#custom_attributes}} diff --git a/src/psij/executors/batch/lsf/lsf.mustache b/src/psij/executors/batch/lsf/lsf.mustache index e21820c3..3823dbbd 100644 --- a/src/psij/executors/batch/lsf/lsf.mustache +++ b/src/psij/executors/batch/lsf/lsf.mustache @@ -20,13 +20,13 @@ {{#job.spec.resources}} - {{#node_count}} + {{#computed_node_count}} #BSUB -nnodes {{.}} - {{/node_count}} + {{/computed_node_count}} - {{#process_count}} + {{#computed_process_count}} #BSUB -n {{.}} - {{/process_count}} + {{/computed_process_count}} {{#gpu_cores_per_process}} #BSUB -gpu num={{.}}/task @@ -44,10 +44,10 @@ #BSUB -q "{{.}}" {{/queue_name}} - {{#project_name}} + {{#account}} #BSUB -G "{{.}}" #BSUB -P "{{.}}" - {{/project_name}} + {{/account}} {{#reservation_id}} #BSUB -U "{{.}}" diff --git a/src/psij/executors/batch/pbs/pbs_classic.mustache b/src/psij/executors/batch/pbs/pbs_classic.mustache index 39c80a81..b4e2a4ff 100644 --- a/src/psij/executors/batch/pbs/pbs_classic.mustache +++ b/src/psij/executors/batch/pbs/pbs_classic.mustache @@ -21,9 +21,9 @@ {{/formatted_job_duration}} {{#job.spec.attributes}} - {{#project_name}} -#PBS -P {{.}} - {{/project_name}} + {{#account}} +#PBS -A {{.}} + {{/account}} {{#queue_name}} #PBS -q {{.}} {{/queue_name}} diff --git a/src/psij/executors/batch/pbs/pbspro.mustache b/src/psij/executors/batch/pbs/pbspro.mustache index 4bf262a1..2dc83768 100644 --- a/src/psij/executors/batch/pbs/pbspro.mustache +++ b/src/psij/executors/batch/pbs/pbspro.mustache @@ -24,9 +24,9 @@ {{/formatted_job_duration}} {{#job.spec.attributes}} - {{#project_name}} -#PBS -P {{.}} - {{/project_name}} + {{#account}} +#PBS -A {{.}} + {{/account}} {{#queue_name}} #PBS -q {{.}} {{/queue_name}} diff --git a/src/psij/executors/batch/pbs_base.py b/src/psij/executors/batch/pbs_base.py index 2bbe0c04..0183b06b 100644 --- a/src/psij/executors/batch/pbs_base.py +++ b/src/psij/executors/batch/pbs_base.py @@ -138,7 +138,10 @@ def parse_status_output(self, exit_code: int, out: str) -> Dict[str, JobStatus]: elif 'Exit_status' in job_report and job_report['Exit_status'] != 0: state = JobState.FAILED - msg = job_report["comment"] + try: + msg = job_report['comment'] + except KeyError: + msg = None r[native_id] = JobStatus(state, message=msg) return r diff --git a/src/psij/executors/batch/slurm.py b/src/psij/executors/batch/slurm.py index 7778b5b8..6f0963ee 100644 --- a/src/psij/executors/batch/slurm.py +++ b/src/psij/executors/batch/slurm.py @@ -147,12 +147,10 @@ def process_cancel_command_output(self, exit_code: int, out: str) -> None: def get_status_command(self, native_ids: Collection[str]) -> List[str]: """See :meth:`~.BatchSchedulerExecutor.get_status_command`.""" - ids = ','.join(native_ids) - # we're not really using job arrays, so this is equivalent to the job ID. However, if # we were to use arrays, this would return one ID for the entire array rather than # listing each element of the array independently - return [_SQUEUE_COMMAND, '-O', 'JobArrayID,StateCompact,Reason', '-t', 'all', '-j', ids] + return [_SQUEUE_COMMAND, '-O', 'JobArrayID,StateCompact,Reason', '-t', 'all', '--me'] def parse_status_output(self, exit_code: int, out: str) -> Dict[str, JobStatus]: """See :meth:`~.BatchSchedulerExecutor.parse_status_output`.""" diff --git a/src/psij/executors/batch/slurm/slurm.mustache b/src/psij/executors/batch/slurm/slurm.mustache index 7cedfbe8..6afa46c6 100644 --- a/src/psij/executors/batch/slurm/slurm.mustache +++ b/src/psij/executors/batch/slurm/slurm.mustache @@ -14,17 +14,17 @@ #SBATCH --exclusive {{/exclusive_node_use}} - {{#node_count}} + {{#computed_node_count}} #SBATCH --nodes={{.}} - {{/node_count}} + {{/computed_node_count}} - {{#process_count}} + {{#computed_process_count}} #SBATCH --ntasks={{.}} - {{/process_count}} + {{/computed_process_count}} - {{#processes_per_node}} + {{#computed_processes_per_node}} #SBATCH --ntasks-per-node={{.}} - {{/processes_per_node}} + {{/computed_processes_per_node}} {{#gpu_cores_per_process}} #SBATCH --gpus-per-task={{.}} @@ -48,9 +48,9 @@ #SBATCH --partition="{{.}}" {{/queue_name}} - {{#project_name}} + {{#account}} #SBATCH --account="{{.}}" - {{/project_name}} + {{/account}} {{#reservation_id}} #SBATCH --reservation="{{.}}" diff --git a/src/psij/job_attributes.py b/src/psij/job_attributes.py index e481ee73..91fcd02b 100644 --- a/src/psij/job_attributes.py +++ b/src/psij/job_attributes.py @@ -1,8 +1,12 @@ +import logging import re from datetime import timedelta from typing import Optional, Dict -from typeguard import check_argument_types +from typeguard import typechecked + + +logger = logging.getLogger(__name__) _WALLTIME_FMT_ERROR = 'Unknown walltime format: %s. Accepted formats are hh:mm:ss, ' \ @@ -12,18 +16,21 @@ class JobAttributes(object): """A class containing ancillary job information that describes how a job is to be run.""" + @typechecked def __init__(self, duration: timedelta = timedelta(minutes=10), - queue_name: Optional[str] = None, project_name: Optional[str] = None, + queue_name: Optional[str] = None, account: Optional[str] = None, reservation_id: Optional[str] = None, - custom_attributes: Optional[Dict[str, object]] = None) -> None: + custom_attributes: Optional[Dict[str, object]] = None, + project_name: Optional[str] = None) -> None: """ :param duration: Specifies the duration (walltime) of the job. A job whose execution exceeds its walltime can be terminated forcefully. :param queue_name: If a backend supports multiple queues, this parameter can be used to instruct the backend to send this job to a particular queue. - :param project_name: If a backend supports multiple projects for billing purposes, setting - this attribute instructs the backend to bill the indicated project for the resources - consumed by this job. + :param account: An account to use for billing purposes. Please note that the executor + implementation (or batch scheduler) may use a different term for the option used for + accounting/billing purposes, such as `project`. However, scheduler must map this + attribute to the accounting/billing option in the underlying execution mechanism. :param reservation_id: Allows specifying an advanced reservation ID. Advanced reservations enable the pre-allocation of a set of resources/compute nodes for a certain duration such that jobs can be run immediately, without waiting in the queue for resources to @@ -39,11 +46,11 @@ def __init__(self, duration: timedelta = timedelta(minutes=10), `pbs.c`, `lsf.core_isolation`) and translate them into the corresponding batch scheduler directives (e.g., `#SLURM --constraint=...`, `#PBS -c ...`, `#BSUB -core_isolation ...`). + :param project_name: Deprecated. Please use the `account` attribute. All constructor parameters are accessible as properties. """ - assert check_argument_types() - + self.account = account self.duration = duration self.queue_name = queue_name self.project_name = project_name @@ -86,8 +93,8 @@ def custom_attributes(self, attrs: Optional[Dict[str, object]]) -> None: def __repr__(self) -> str: """Returns a string representation of this object.""" - return 'JobAttributes(duration={}, queue_name={}, project_name={}, reservation_id={}, ' \ - 'custom_attributes={})'.format(self.duration, self.queue_name, self.project_name, + return 'JobAttributes(duration={}, queue_name={}, account={}, reservation_id={}, ' \ + 'custom_attributes={})'.format(self.duration, self.queue_name, self.account, self.reservation_id, self._custom_attributes) def __eq__(self, o: object) -> bool: @@ -99,7 +106,7 @@ def __eq__(self, o: object) -> bool: if not isinstance(o, JobAttributes): return False - for prop_name in ['duration', 'queue_name', 'project_name', 'reservation_id', + for prop_name in ['duration', 'queue_name', 'account', 'reservation_id', 'custom_attributes']: if getattr(self, prop_name) != getattr(o, prop_name): return False @@ -150,3 +157,15 @@ def parse_walltime(walltime: str) -> timedelta: elif unit == 's': return timedelta(seconds=val) raise ValueError(_WALLTIME_FMT_ERROR % walltime) + + @property + def project_name(self) -> Optional[str]: + """Deprecated. Please use the `account` attribute.""" + return self.account + + @project_name.setter + def project_name(self, project_name: Optional[str]) -> None: + if project_name is not None: + logger.warning('The "project_name" attribute is deprecated. Please use the "account" ' + 'attribute instead.') + self.account = project_name diff --git a/src/psij/job_executor.py b/src/psij/job_executor.py index b9f9a226..286fc315 100644 --- a/src/psij/job_executor.py +++ b/src/psij/job_executor.py @@ -1,6 +1,6 @@ import logging from abc import ABC, abstractmethod -from distutils.version import Version +from packaging.version import Version from threading import RLock from typing import Optional, Dict, List, Type, cast, Union, Callable, Set @@ -14,11 +14,15 @@ from psij.job_executor_config import JobExecutorConfig from psij.job_launcher import Launcher from psij.job_spec import JobSpec +from psij.resource_spec import ResourceSpecV1 logger = logging.getLogger(__name__) +_DEFAULT_RESOURCES = ResourceSpecV1() + + class JobExecutor(ABC): """An abstract base class for all JobExecutor implementations.""" @@ -92,6 +96,8 @@ def _check_job(self, job: Job) -> JobSpec: spec = job.spec if not spec: raise InvalidJobException('Missing specification') + if not spec.resources: + spec.resources = _DEFAULT_RESOURCES if __debug__: if spec.environment is not None: diff --git a/src/psij/job_spec.py b/src/psij/job_spec.py index 8ebdb2b5..ff16c26d 100644 --- a/src/psij/job_spec.py +++ b/src/psij/job_spec.py @@ -5,7 +5,7 @@ import pathlib from typing import Dict, List, Optional, Union, Set -from typeguard import check_argument_types +from typeguard import typechecked import psij.resource_spec import psij.job_attributes @@ -48,6 +48,7 @@ def _all_to_path(s: Optional[Set[Union[str, pathlib.Path]]]) -> Optional[Set[pat class JobSpec(object): """A class that describes the details of a job.""" + @typechecked def __init__(self, executable: Optional[str] = None, arguments: Optional[List[str]] = None, # For some odd reason, and only in the constructor, if Path is used directly, # sphinx fails to find the class. Using Path in the getters and setters does not @@ -153,8 +154,6 @@ def __init__(self, executable: Optional[str] = None, arguments: Optional[List[st the scheduler. In such a case, one must leave the `spec.directory` attribute empty and refer to files inside the job directory using relative paths. """ - assert check_argument_types() - self._name = name self.executable = executable self.arguments = arguments diff --git a/src/psij/resource_spec.py b/src/psij/resource_spec.py index 2d17f3c9..7b565fe4 100644 --- a/src/psij/resource_spec.py +++ b/src/psij/resource_spec.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from typing import Optional, List -from typeguard import check_argument_types +from typeguard import typechecked from psij.exceptions import InvalidJobException @@ -45,6 +45,7 @@ def get_instance(version: int) -> 'ResourceSpec': raise ValueError() +@typechecked class ResourceSpecV1(ResourceSpec): """This class implements V1 of the PSI/J resource specification.""" @@ -53,7 +54,7 @@ def __init__(self, node_count: Optional[int] = None, processes_per_node: Optional[int] = None, cpu_cores_per_process: Optional[int] = None, gpu_cores_per_process: Optional[int] = None, - exclusive_node_use: bool = True) -> None: + exclusive_node_use: bool = False) -> None: """ Some of the properties of this class are constrained. Specifically, `process_count = node_count * processes_per_node`. Specifying all constrained properties @@ -81,8 +82,6 @@ def __init__(self, node_count: Optional[int] = None, All constructor parameters are accessible as properties. """ - assert check_argument_types() - self.node_count = node_count self.process_count = process_count self.processes_per_node = processes_per_node diff --git a/src/psij/serialize.py b/src/psij/serialize.py index 2cab84da..80ec771b 100644 --- a/src/psij/serialize.py +++ b/src/psij/serialize.py @@ -6,7 +6,6 @@ from io import StringIO, TextIOBase from pathlib import Path from typing import Optional, Dict, Union, List, IO, AnyStr, TextIO -import typing_compat from psij import ResourceSpec from psij.job_attributes import JobAttributes @@ -135,12 +134,12 @@ def _from_psij_object(self, o: Union[JobSpec, JobAttributes, ResourceSpec]) \ def _canonicalize_type(self, t: object) -> object: # generics don't appear to be subclasses of Type, so we can't really use Type for t - origin = typing_compat.get_origin(t) + origin = typing.get_origin(t) if origin == Optional: # Python converts Optional[T] to Union[T, None], so this shouldn't happen - return typing_compat.get_args(t)[0] + return typing.get_args(t)[0] elif origin == Union: - args = typing_compat.get_args(t) + args = typing.get_args(t) if args[0] == NoneType: return args[1] elif args[1] == NoneType: @@ -171,10 +170,10 @@ def _from_object(self, o: object, t: object) -> object: else: if t == Union[str, Path] or t == Optional[Union[str, Path]]: return str(o) - if typing_compat.get_origin(t) == dict: + if typing.get_origin(t) == dict: assert isinstance(o, dict) return self._from_dict(o) - if typing_compat.get_origin(t) == list: + if typing.get_origin(t) == list: assert isinstance(o, list) return self._from_list(o) raise ValueError('Cannot convert type "%s".' % t) @@ -249,10 +248,10 @@ def _to_object(self, s: object, t: object) -> object: if t == Union[str, Path] or t == Optional[Union[str, Path]]: assert isinstance(s, str) return Path(s) - if typing_compat.get_origin(t) == dict: + if typing.get_origin(t) == dict: assert isinstance(s, dict) return self._to_dict(s) - if typing_compat.get_origin(t) == list: + if typing.get_origin(t) == list: assert isinstance(s, list) return self._to_list(s) raise ValueError('Cannot convert type "%s".' % t) diff --git a/testing.conf b/testing.conf index d0340789..2b04247e 100644 --- a/testing.conf +++ b/testing.conf @@ -119,7 +119,7 @@ queue_name = # If you need a project/account for billing purposes, enter it here. # -project_name = +account = diff --git a/tests/_test_tools.py b/tests/_test_tools.py index 7ee08150..b9dac7b9 100644 --- a/tests/_test_tools.py +++ b/tests/_test_tools.py @@ -58,8 +58,8 @@ def _get_executor_instance(ep: ExecutorTestParams, job: Optional[Job] = None) -> assert job.spec is not None job.spec.launcher = ep.launcher job.spec.attributes = JobAttributes(custom_attributes=ep.custom_attributes) - if ep.project_name is not None: - job.spec.attributes.project_name = ep.project_name + if ep.account is not None: + job.spec.attributes.account = ep.account if ep.queue_name is not None: job.spec.attributes.queue_name = ep.queue_name return JobExecutor.get_instance(ep.executor, url=ep.url) diff --git a/tests/ci_runner.py b/tests/ci_runner.py index a1a9420e..5af69a36 100755 --- a/tests/ci_runner.py +++ b/tests/ci_runner.py @@ -147,7 +147,7 @@ def run_branch_tests(conf: Dict[str, str], dir: Path, run_id: str, clone: bool = args.append('--branch-name-override') args.append(fake_branch_name) for opt in ['maintainer_email', 'executors', 'server_url', 'key', 'max_age', - 'custom_attributes', 'queue_name', 'project_name']: + 'custom_attributes', 'queue_name', 'project_name', 'account']: try: val = get_conf(conf, opt) args.append('--' + opt.replace('_', '-')) @@ -164,8 +164,8 @@ def run_branch_tests(conf: Dict[str, str], dir: Path, run_id: str, clone: bool = cwd = (dir / 'code') if clone else Path('.') env = dict(os.environ) - env['PYTHONPATH'] = str(cwd.resolve() / '.packages') \ - + ':' + str(cwd.resolve() / 'src') \ + env['PYTHONPATH'] = str(cwd.resolve() / 'src') \ + + ':' + str(cwd.resolve() / '.packages') \ + (':' + env['PYTHONPATH'] if 'PYTHONPATH' in env else '') subprocess.run(args, cwd=cwd.resolve(), env=env) diff --git a/tests/conftest.py b/tests/conftest.py index a7d8fab9..886cddbb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -71,6 +71,8 @@ def pytest_addoption(parser): help='A queue to run the batch jobs in.') parser.addoption('--project-name', action='store', default=None, help='A project/account name to associate the batch jobs with.') + parser.addoption('--account', action='store', default=None, + help='An account to use for billing purposes.') parser.addoption('--custom-attributes', action='store', default=None, help='A set of custom attributes to pass to jobs.') parser.addoption('--minimal-uploads', action='store_true', default=False, @@ -172,14 +174,30 @@ def _translate_launcher(config: Dict[str, str], exec: str, launcher: str) -> str return launcher +_WARNED_ABOUT_ACCOUNT_CLASH = False + + +def _get_account(options): + global _WARNED_ABOUT_ACCOUNT_CLASH + if options.account: + if options.project_name and not _WARNED_ABOUT_ACCOUNT_CLASH: + _WARNED_ABOUT_ACCOUNT_CLASH = True + logger.warning('Both "account" and "project_name" are specified. Ignoring ' + '"project_name".') + return options.account + else: + return options.project_name + + def pytest_generate_tests(metafunc): + options = metafunc.config.option if 'execparams' in metafunc.fixturenames: etps = [] for x in _get_executors((metafunc.config)): - etp = ExecutorTestParams(x, queue_name=metafunc.config.option.queue_name, - project_name=metafunc.config.option.project_name, - custom_attributes_raw=metafunc.config.option.custom_attributes) + etp = ExecutorTestParams(x, queue_name=options.queue_name, + account=_get_account(options), + custom_attributes_raw=options.custom_attributes) etps.append(etp) metafunc.parametrize('execparams', etps, ids=lambda x: str(x)) diff --git a/tests/executor_test_params.py b/tests/executor_test_params.py index 9796d0fa..6c864a94 100644 --- a/tests/executor_test_params.py +++ b/tests/executor_test_params.py @@ -6,7 +6,7 @@ class ExecutorTestParams: """A class holding executor, launcher, url pairs.""" def __init__(self, spec: str, queue_name: Optional[str] = None, - project_name: Optional[str] = None, + account: Optional[str] = None, custom_attributes_raw: Optional[Dict[str, Dict[str, object]]] = None) \ -> None: """ @@ -19,8 +19,8 @@ def __init__(self, spec: str, queue_name: Optional[str] = None, url are specified, the string should be formatted as "::". queue_name An optional queue to submit the job to - project_name - An optional project name to associate the job with + account + An optional account to use for billing purposes. custom_attributes """ spec_l = re.split(':', spec, maxsplit=2) @@ -35,7 +35,7 @@ def __init__(self, spec: str, queue_name: Optional[str] = None, self.url = None self.queue_name = queue_name - self.project_name = project_name + self.account = account self.custom_attributes_raw = custom_attributes_raw self.custom_attributes: Dict[str, object] = {} diff --git a/tests/plugins1/_batch_test/test/qlib.py b/tests/plugins1/_batch_test/test/qlib.py index 5379f381..e2a240f6 100644 --- a/tests/plugins1/_batch_test/test/qlib.py +++ b/tests/plugins1/_batch_test/test/qlib.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 import json import logging +import os import tempfile from datetime import timedelta, datetime from pathlib import Path @@ -8,10 +9,12 @@ from filelock import FileLock + +uid = os.getuid() tmp = tempfile.gettempdir() -lock_file = Path(tmp) / 'qlist.lock' -state_file = Path(tmp) / 'qlist' -log_file = Path(tmp) / 'qlist.log' +lock_file = Path(tmp) / ('qlist-%s.lock' % uid) +state_file = Path(tmp) / ('qlist-%s' % uid) +log_file = Path(tmp) / ('qlist-%s.log' % uid) my_dir = Path(__file__).parent diff --git a/tests/plugins1/_batch_test/test/test.mustache b/tests/plugins1/_batch_test/test/test.mustache index 88d0b4d8..5fc00bfe 100644 --- a/tests/plugins1/_batch_test/test/test.mustache +++ b/tests/plugins1/_batch_test/test/test.mustache @@ -11,18 +11,19 @@ cd "{{.}}" export PSIJ_TEST_BATCH_EXEC_COUNT=1 {{#job.spec.resources}} - {{#process_count}} + {{#computed_process_count}} export PSIJ_TEST_BATCH_EXEC_COUNT={{.}} - {{/process_count}} + {{/computed_process_count}} {{/job.spec.resources}} {{#job.spec.attributes}} {{#queue_name}} export PSIJ_TEST_BATCH_EXEC_QUEUE="{{.}}" {{/queue_name}} - {{#project_name}} + {{#account}} export PSIJ_TEST_BATCH_EXEC_PROJECT="{{.}}" - {{/project_name}} +export PSIJ_TEST_BATCH_EXEC_ACCOUNT="{{.}}" + {{/account}} {{#reservation_id}} export PSIJ_TEST_BATCH_EXEC_RES_ID="{{.}}" {{/reservation_id}} diff --git a/tests/plugins1/psij-descriptors/descriptors.py b/tests/plugins1/psij-descriptors/descriptors.py index 531be2a3..b0e3e6d8 100644 --- a/tests/plugins1/psij-descriptors/descriptors.py +++ b/tests/plugins1/psij-descriptors/descriptors.py @@ -1,29 +1,29 @@ -from distutils.version import StrictVersion +from packaging.version import Version from psij.descriptor import Descriptor __PSI_J_EXECUTORS__ = [ # executor in the same path as descriptor; should load - Descriptor(name='p1-tp1', version=StrictVersion('0.0.1'), + Descriptor(name='p1-tp1', version=Version('0.0.1'), cls='_test_plugins1.ex1._Executor1'), # executor in different path, but sharing module; should NOT load - Descriptor(name='p2-tp1', version=StrictVersion('0.0.1'), + Descriptor(name='p2-tp1', version=Version('0.0.1'), cls='_test_plugins1.ex2._Executor2'), # executor in different path with no shared module; should NOT load - Descriptor(name='p2-tp3', version=StrictVersion('0.0.1'), + Descriptor(name='p2-tp3', version=Version('0.0.1'), cls='_test_plugins3.ex3._Executor3'), # noop executor that should have no reason to not load - Descriptor(name='_always_loads', version=StrictVersion('0.0.1'), + Descriptor(name='_always_loads', version=Version('0.0.1'), cls='_test_plugins1._always_loads_executor.AlwaysLoadsExecutor'), # noop executor with an import of a package that does not exist - Descriptor(name='_never_loads', version=StrictVersion('0.0.1'), + Descriptor(name='_never_loads', version=Version('0.0.1'), cls='_test_plugins1._never_loads_executor.NeverLoadsExecutor'), # an executor that exercises some of the batch test stuff - Descriptor(name='batch-test', version=StrictVersion('0.0.1'), + Descriptor(name='batch-test', version=Version('0.0.1'), cls='_batch_test._batch_test._TestJobExecutor') ] __PSI_J_LAUNCHERS__ = [ - Descriptor(name='batch-test', version=StrictVersion('0.0.1'), + Descriptor(name='batch-test', version=Version('0.0.1'), cls='_batch_test._batch_test._TestLauncher') ] diff --git a/tests/test_executor.py b/tests/test_executor.py index e425a030..af2bbd69 100644 --- a/tests/test_executor.py +++ b/tests/test_executor.py @@ -220,7 +220,21 @@ def test_submit_script_generation(exec_name: str) -> None: prefix = _get_attr_prefix(exec_name) _check_str_attrs(ex, job, ['executable', 'directory'], lambda k, v: setattr(spec, k, v)) - _check_str_attrs(ex, job, ['queue_name', 'project_name', 'reservation_id'], + _check_str_attrs(ex, job, ['queue_name', 'account', 'reservation_id'], lambda k, v: setattr(attrs, k, v)) _check_str_attrs(ex, job, [prefix + '.cust_attr1', prefix + '.cust_attr2'], lambda k, v: c_attrs.__setitem__(k, v)) + + +def test_resource_generation1() -> None: + res = ResourceSpecV1() + spec = JobSpec('/bin/date', resources=res) + job = Job(spec=spec) + ex = JobExecutor.get_instance('slurm') + assert isinstance(ex, BatchSchedulerExecutor) + with TemporaryFile(mode='w+') as f: + ex.generate_submit_script(job, ex._create_script_context(job), f) + f.seek(0) + contents = f.read() + if contents.find('--exclusive') != -1: + pytest.fail('Spurious exclusive flag') diff --git a/tests/test_executor_versions.py b/tests/test_executor_versions.py index a1fdee1c..9983e6a5 100755 --- a/tests/test_executor_versions.py +++ b/tests/test_executor_versions.py @@ -2,8 +2,7 @@ # This is meant as a simple test file to check if psi/j was installed successfully -from distutils.version import Version - +from packaging.version import Version from psij import JobExecutor diff --git a/tests/test_job_spec.py b/tests/test_job_spec.py index 9fa57a49..f02b6305 100644 --- a/tests/test_job_spec.py +++ b/tests/test_job_spec.py @@ -1,6 +1,8 @@ import os from pathlib import Path +from typeguard import suppress_type_checks + import pytest from psij import Job, JobExecutor, JobSpec @@ -13,8 +15,9 @@ def _test_spec(spec: JobSpec) -> None: def test_environment_types() -> None: - with pytest.raises(TypeError): - _test_spec(JobSpec(executable='true', environment={1: 'foo'})) # type: ignore + with suppress_type_checks(): + with pytest.raises(TypeError): + _test_spec(JobSpec(executable='true', environment={1: 'foo'})) # type: ignore with pytest.raises(TypeError): spec = JobSpec(executable='true') diff --git a/tests/user_guide/test_scheduling_information.py b/tests/user_guide/test_scheduling_information.py index ad6d226d..367db1d1 100644 --- a/tests/user_guide/test_scheduling_information.py +++ b/tests/user_guide/test_scheduling_information.py @@ -12,7 +12,7 @@ def test_user_guide_scheduling_info(execparams: ExecutorTestParams) -> None: executable="/bin/date", attributes=JobAttributes( queue_name=execparams.queue_name, - project_name=execparams.project_name + account=execparams.account ) ) ) diff --git a/web/install.html b/web/install.html index b9b5ee11..60344e73 100644 --- a/web/install.html +++ b/web/install.html @@ -13,7 +13,7 @@

Installing PSI/J The psij-python library has the following requirements:
    -
  • Python 3.7 or later +
  • Python 3.8 or later
  • Ubuntu 16.04 or later (or an equivalent Linux distribution)
  • OS X 10.10 or later