From 671d59f2217a4edb1fea42bc132078b5f90de26a Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 22 Jan 2025 10:20:58 +0100 Subject: [PATCH] Making the build work --- .github/workflows/ci.yaml | 15 ++++++++++----- README-dev.md | 24 ++++++++++++++++-------- notebooks/test_notebooks.py | 17 +++++++++++++++++ src/sensai/util/version.py | 3 ++- 4 files changed, 45 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 13bbd3ea..2fab33ea 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -13,7 +13,7 @@ jobs: strategy: matrix: # py_backwardscompat is not included because there is also an issue with numpy compatibility on Python 3.10 - env_name: [py_pinned_dependencies, py_latest_dependencies] # removed py_backwardscompat + env_name: [py_pinned_dependencies, py_latest_dependencies] steps: - name: Checkout Code uses: actions/checkout@v3 @@ -26,23 +26,28 @@ jobs: - name: Install Dependencies run: | python -m pip install --upgrade pip - if [[ "${{ matrix.env_name }}" == "py" ]]; then + if [[ "${{ matrix.env_name }}" == "py_pinned_dependencies" ]]; then pip install -v -r requirements.txt -r requirements_torch.txt -r requirements_lightgbm.txt -r requirements_geoanalytics.txt -r requirements_xgboost.txt pytest pytest-cov pytest-xdist pytorch-lightning~=1.1.0; elif [[ "${{ matrix.env_name }}" == "py_latest_dependencies" ]]; then pip install pytest jupyter==1.0.0 nbconvert==6.5.0 clearml==0.17.1 pytorch-lightning>=1.1 ".[full]"; elif [[ "${{ matrix.env_name }}" == "py_backwardscompat" ]]; then - pip install pytest scikit-learn==1.0.2 pytorch-lightning>=1.1 ".[torch]"; + pip install pytest scikit-learn==1.0.2 "numpy<1.21" ".[torch]"; fi pip install --no-deps -e . + - name: Run Notebook Tests + run: | + if [[ "${{ matrix.env_name }}" == "py_latest_dependencies" ]]; then + pytest notebooks; + fi + - name: Run Tests run: | - if [[ "${{ matrix.env_name }}" == "py" ]]; then + if [[ "${{ matrix.env_name }}" == "py_pinned_dependencies" ]]; then coverage erase pytest -n 4 --cov --cov-append --cov-report=term-missing tests; elif [[ "${{ matrix.env_name }}" == "py_latest_dependencies" ]]; then pytest - pytest notebooks; elif [[ "${{ matrix.env_name }}" == "py_backwardscompat" ]]; then pytest tests/backwardscompat; fi diff --git a/README-dev.md b/README-dev.md index a66b6196..93d26d7e 100644 --- a/README-dev.md +++ b/README-dev.md @@ -18,24 +18,32 @@ Use conda to set up the Python environment: Solving the environment may take several minutes (but should ultimately work). -NOTE: versions are mostly unpinned in the environment specification, because this facilitates conda dependency resolution. Also, sensAI is intended to be compatible with *all* (newer) versions of the dependencies. If it isn't, we need to specify an upper version bound in `setup.py` (where it matters the most) as well as in `environment.yml`. Compatibility with old (pinned) versions and the latest versions is tested in the tox build (see below). +NOTE: versions are mostly unpinned in the environment specification, because this facilitates conda dependency resolution. +Also, sensAI is intended to be compatible with *all* (newer) versions of the dependencies. +If it isn't, we need to specify an upper version bound in `setup.py` (where it matters the most) as well as in `environment.yml`. +Compatibility with old (pinned) versions and the latest versions is tested in the GitHub build (see below). # Build and Test Pipeline -The tests and docs build are executed via **tox** in several environments: -* `py`: the "regular" test environment, where we test against the pinned dependencies (by explicitly including `requirements.txt` with the pinned versions; this is also the environment in which we test the execution of notebooks -* `py_latest_dependencies`: the environment where we use the latest versions of all dependencies (except where we have identified an incompatibility; see `setup.py` definitions `DEPS_VERSION_LOWER_BOUND` and `DEPS_VERSION_UPPER_BOUND_EXCLUSIVE`); by not including `requirements.txt`, we depend on the latest admissible versions according to `setup.py` -* `docs`: the environment in which docs are built via sphinx +The tests and docs build are executed in several environments: +* `py_pinned_dependencies`: the "regular" test environment, where we test against the pinned dependencies + (by explicitly including `requirements.txt` with the pinned versions; this is also the environment in which we test the + execution of notebooks +* `py_latest_dependencies`: the environment where we use the latest versions of all dependencies (except where we have + identified an incompatibility; see `setup.py` definitions `DEPS_VERSION_LOWER_BOUND` and `DEPS_VERSION_UPPER_BOUND_EXCLUSIVE`); + by not including `requirements.txt`, we depend on the latest admissible versions according to `setup.py` +* `py_backwardscompa`: a special environment with old versions of some critical dependences where we can test backwards compatibility + with persisted models of very old sensAI versions (that used older versions of the dependencies, e.g. sklearn) ## Automated Tests -The tests can be locally run without tox via +The tests can be locally via sh run_pytest_tests.sh ## Docs Build -Docs are automatically created during the GitHub build via tox. +Docs are automatically created during the GitHub build. All .rst files are auto-generated (by `build_scripts/update_docs.py`), with the exception of the root index file `index.rst`. @@ -56,7 +64,7 @@ For changes in notebooks to be reflected in the docs build, the test needs to be ### Manually Running the Docs Build -The docs build can be run without tox via +The docs build can be run via sh build-docs.sh diff --git a/notebooks/test_notebooks.py b/notebooks/test_notebooks.py index c01dde94..25b4225f 100644 --- a/notebooks/test_notebooks.py +++ b/notebooks/test_notebooks.py @@ -75,6 +75,23 @@ def test_notebook(notebook): :param notebook: :return: """ + from sensai.util import logging + #logging.configure() + log = logging.getLogger(__name__) + import pandas as pd + import sys + print("ROOT DIR:", ROOT_DIR) + sys.path.append(str(ROOT_DIR)) + import config + cfg = config.get_config() + path = cfg.datafile_path("mnist_train.csv.zip") + if os.path.exists(path): + log.warning(f"FILE EXISTS {path}") + log.warning("READ ABS") + mnist_df = pd.read_csv(os.path.abspath(path)) + log.warning("READ NORMAL") + mnist_df = pd.read_csv(path) + notebook_path = NOTEBOOKS_DIR / notebook nb = execute_notebook(notebook_path) diff --git a/src/sensai/util/version.py b/src/sensai/util/version.py index ef800b10..01c376df 100644 --- a/src/sensai/util/version.py +++ b/src/sensai/util/version.py @@ -1,6 +1,7 @@ +from .string import ToStringMixin -class Version: +class Version(ToStringMixin): """ Assists in checking the version of a Python package based on the __version__ attribute """