diff --git a/.circleci/config.yml b/.circleci/config.yml index da7065911f..d1e1456ac0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,48 @@ +# reusable anchors +_machine_defaults: &machine_defaults + environment: + TZ: "/usr/share/zoneinfo/America/Los_Angeles" + SCRATCH: "/scratch" + machine: + image: ubuntu-2204:current + docker_layer_caching: true + working_directory: /tmp/src/sdcflows + resource_class: large + +_python_defaults: &python_defaults + docker: + - image: cimg/python:3.10.9 + working_directory: /tmp/src/sdcflows + +_docker_auth: &docker_auth + name: Docker authentication + command: | + if [[ -n $DOCKER_PAT ]]; then + echo "$DOCKER_PAT" | docker login -u $DOCKER_USER --password-stdin + fi + +_setup_docker_registry: &setup_docker_registry + name: Set up Docker registry + command: | + if [[ -f /tmp/images/registry.tar.gz ]]; then + echo "Loading saved registry image" + docker load < /tmp/images/registry.tar.gz + else + echo "Pulling registry image from DockerHub" + docker pull registry:2 + fi + docker run -d -p 5000:5000 --restart=always --name=registry \ + -v /tmp/docker:/var/lib/registry registry:2 + +_pull_from_registry: &pull_from_registry + name: Pull and tag image from local registry + command: | + docker pull localhost:5000/sdcflows + docker tag localhost:5000/sdcflows nipreps/sdcflows:latest + version: 2.1 orbs: - docker: circleci/docker@1.6.0 + docker: circleci/docker@2.1.4 jobs: cache_test_data: @@ -89,8 +131,8 @@ jobs: - restore_cache: keys: - - freesurfer-v1-{{ .BuildNum }} - - freesurfer-v1- + - freesurfer-v0-{{ .BuildNum }} + - freesurfer-v0- - run: name: Pull FreeSurfer down command: | @@ -114,23 +156,19 @@ jobs: --exclude='freesurfer/subjects/sample-*.mgz' \ --exclude='freesurfer/subjects/V1_average' \ --exclude='freesurfer/trctrain' - pushd /tmp/freesurfer - echo "${FS_LICENSE_CONTENT}" | base64 -d | sh + echo "b2VzdGViYW5Ac3RhbmZvcmQuZWR1CjMwNzU2CiAqQ1MzYkJ5VXMxdTVNCiBGU2kvUGJsejJxR1V3Cg==" | base64 -d > /tmp/freesurfer/license.txt else echo "FreeSurfer was cached." circleci step halt fi - save_cache: - key: freesurfer-v1-{{ .BuildNum }} + key: freesurfer-v0-{{ .BuildNum }} paths: - /tmp/freesurfer build_n_pytest: - machine: - image: ubuntu-2004:202107-02 + <<: *machine_defaults working_directory: /tmp/tests - environment: - TZ: "/usr/share/zoneinfo/America/Los_Angeles" steps: - restore_cache: keys: @@ -140,45 +178,53 @@ jobs: - build-v2- paths: - /tmp/docker + - docker/install-docker-credential-helper + - run: *docker_auth + - run: *setup_docker_registry - run: - name: Set-up a Docker registry + name: Pull Ubuntu/jammy image command: | - docker run -d -p 5000:5000 --restart=always --name=registry \ - -v /tmp/docker:/var/lib/registry registry:2 + set +e + docker pull localhost:5000/ubuntu + success=$? + set -e + if [[ "$success" = "0" ]]; then + echo "Pulling from local registry" + docker tag localhost:5000/ubuntu ubuntu:jammy + else + echo "Pulling from Docker Hub" + docker pull ubuntu:jammy + docker tag ubuntu:jammy localhost:5000/ubuntu + docker push localhost:5000/ubuntu + fi - run: - name: Pull images + name: Pull SDCFlows Docker image command: | set +e - docker pull localhost:5000/ubuntu + docker pull localhost:5000/sdcflows success=$? set -e if [[ "$success" = "0" ]]; then echo "Pulling from local registry" - docker tag localhost:5000/ubuntu ubuntu:xenial-20201030 - docker pull localhost:5000/sdcflows docker tag localhost:5000/sdcflows nipreps/sdcflows:latest docker tag localhost:5000/sdcflows nipreps/sdcflows else echo "Pulling from Docker Hub" - docker pull ubuntu:xenial-20201030 - docker tag ubuntu:xenial-20201030 localhost:5000/ubuntu - docker push localhost:5000/ubuntu docker pull nipreps/sdcflows:latest fi - checkout: path: /tmp/src/sdcflows - run: name: Build Docker image + working_directory: /tmp/src/sdcflows no_output_timeout: 60m command: | - cd /tmp/src/sdcflows - export PY3=$(pyenv versions | grep '3\.' | - sed -e 's/.* 3\./3./' -e 's/ .*//') + export PY3=$( pyenv versions | awk '/^\* 3/ { print $2 }' ) pyenv local $PY3 - python3 -m pip install "setuptools ~= 45.0" "setuptools_scm >= 6.2" "pip>=10.0.1" + python3 -m pip install --upgrade setuptools setuptools_scm pip # Get version, update files. - THISVERSION=$( python3 setup.py --version ) + THISVERSION=$( python3 -m setuptools_scm ) if [[ ${THISVERSION:0:1} == "0" ]] ; then echo "WARNING: latest git tag could not be found" echo "Please, make sure you fetch all tags from upstream with" @@ -194,6 +240,20 @@ jobs: --build-arg VERSION="${CIRCLE_TAG:-$THISVERSION}" . \ | tee build-output.log echo "${CIRCLE_TAG:-$THISVERSION}" >> /tmp/.local-version.txt + - run: + name: Check Docker image + working_directory: /tmp/src/sdcflows + command: | + export PY3=$( pyenv versions | awk '/^\* 3/ { print $2 }' ) + pyenv local $PY3 + # Get version, update files. + THISVERSION=$( python3 -m setuptools_scm ) + BUILT_VERSION=$( docker run --rm --entrypoint=python nipreps/sdcflows:latest -c "import sdcflows; print(sdcflows.__version__)" ) + BUILT_VERSION=${BUILT_VERSION%$'\r'} + echo "VERSION: \"$THISVERSION\"" + echo "BUILT: \"$BUILT_VERSION\"" + set -e + test "$BUILT_VERSION" = "$THISVERSION" - run: name: Docker push to local registry no_output_timeout: 40m @@ -209,21 +269,11 @@ jobs: key: build-v2-{{ .Branch }}-{{ epoch }} paths: - /tmp/docker - - run: - name: Check version packaged in Docker image - command: | - docker run --rm -v /tmp:/tmp -v /tmp/src/sdcflows/.circleci/version.py:/usr/share/version.py \ - --entrypoint=python nipreps/sdcflows /usr/share/version.py - THISVERSION=$( head -n1 /tmp/.local-version.txt ) - INSTALLED_VERSION=$( head -n1 /tmp/.docker-version.txt ) - echo "VERSION: \"${THISVERSION}\"" - echo "INSTALLED: \"${INSTALLED_VERSION}\"" - test "${INSTALLED_VERSION}" = "${THISVERSION}" - restore_cache: keys: - - freesurfer-v1-{{ .BuildNum }} - - freesurfer-v1- + - freesurfer-v0-{{ .BuildNum }} + - freesurfer-v0- - restore_cache: keys: - data-v6-{{ .Branch }}-{{ .Revision }} @@ -239,8 +289,8 @@ jobs: - workdir-v2- - run: name: Refreshing cached intermediate results + working_directory: /tmp/src/sdcflows command: | - cd /tmp/src/sdcflows COMMIT_MSG=$( git log --format=oneline -n 1 $CIRCLE_SHA1 ) set +e do_refresh="$( echo "${COMMIT_MSG}" | grep -i -E '\[refresh[ _]?cache\]' )" @@ -295,12 +345,11 @@ jobs: - run: name: Submit unit test coverage + working_directory: /tmp/src/sdcflows command: | - export PY3=$(pyenv versions | grep '3\.' | - sed -e 's/.* 3\./3./' -e 's/ .*//') + export PY3=$( pyenv versions | awk '/^\* 3/ { print $2 }' ) pyenv local $PY3 python3 -m pip install codecov - cd /tmp/src/sdcflows python3 -m codecov --file /tmp/tests/unittests.xml \ --flags unittests -e CIRCLE_JOB @@ -326,8 +375,7 @@ jobs: command: | python -m venv /tmp/venv source /tmp/venv/bin/activate - python -m pip install -U build "setuptools >= 45" wheel "setuptools_scm >= 6.2" \ - setuptools_scm_git_archive pip twine docutils + python -m pip install -U pip setuptools_scm pip install --no-cache-dir -r docs/requirements.txt - run: name: Build only this commit @@ -340,9 +388,7 @@ jobs: path: ~/docs/ deploy_docker: - machine: - image: ubuntu-2004:202107-02 - working_directory: /tmp/src/ + <<: *machine_defaults steps: - restore_cache: keys: @@ -352,31 +398,22 @@ jobs: - build-v2- paths: - /tmp/docker - - run: - name: Set-up a Docker registry - command: | - docker run -d -p 5000:5000 --restart=always --name=registry \ - -v /tmp/docker:/var/lib/registry registry:2 - - run: - name: Pull images from local registry - command: | - docker pull localhost:5000/sdcflows - docker tag localhost:5000/sdcflows nipreps/sdcflows:latest + - docker/install-docker-credential-helper + - run: *docker_auth + - run: *setup_docker_registry + - run: *pull_from_registry - run: name: Deploy to Docker Hub no_output_timeout: 40m command: | if [[ -n "$DOCKER_PAT" ]]; then - docker login -u $DOCKER_USER -p $DOCKER_PAT docker push nipreps/sdcflows:latest docker tag nipreps/sdcflows nipreps/sdcflows:$CIRCLE_TAG docker push nipreps/sdcflows:$CIRCLE_TAG fi test_package: - docker: - - image: cimg/python:3.8.5 - working_directory: /tmp/src/sdcflows + <<: *python_defaults steps: - checkout - run: @@ -384,8 +421,7 @@ jobs: command: | python -m venv /tmp/buildenv source /tmp/buildenv/bin/activate - python -m pip install -U build "setuptools >= 45" wheel "setuptools_scm >= 6.2" \ - setuptools_scm_git_archive pip twine docutils + python -m pip install -U build twine setuptools_scm python -m build -s -w python -m twine check dist/* - store_artifacts: @@ -399,24 +435,20 @@ jobs: command: | source /tmp/buildenv/bin/activate THISVERSION=$( python -m setuptools_scm ) - python -m pip install dist/*.tar.gz + python -m pip install dist/*.whl mkdir empty cd empty INSTALLED=$( python -c 'import sdcflows; print(sdcflows.__version__)' ) test "${CIRCLE_TAG:-$THISVERSION}" == "$INSTALLED" deploy_pypi: - docker: - - image: cimg/python:3.8.5 - working_directory: /tmp/src/sdcflows + <<: *python_defaults steps: - attach_workspace: at: /tmp/src/sdcflows - run: name: Upload to Pypi command: | - python -m venv /tmp/upload - source /tmp/upload/bin/activate python -m pip install twine python -m twine check dist/* python -m twine upload dist/* --non-interactive diff --git a/.dockerignore b/.dockerignore index 89ba00d7e2..fd68abaa11 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,47 +8,18 @@ build/**/* build dist/**/* dist -docs/**/* -docs sdcflows.egg-info/**/* sdcflows.egg-info .eggs/**/* .eggs -# housekeeping tools -get_version.py -update_changes.sh -tox.ini - # pip installs src/**/* src/ -# git -.gitattributes -.gitignore -.github/**/* -.github -.git/**/* -.git - # other work/**/* work out/**/* out/ -tools/**/* -tools .DS_Store - -# CI, etc. -.circleci/**/* -.circleci -.codecov.yml -.coveragerc -.pep8speaks.yml -.readthedocs.yml -.travis.yml -.zenodo.json -CONTRIBUTING.md - diff --git a/Dockerfile b/Dockerfile index 01c0953068..24a52a1319 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,8 +22,15 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -# Use Ubuntu 20.04 LTS -FROM ubuntu:focal-20210416 +FROM python:slim AS src +RUN pip install build +RUN apt-get update && \ + apt-get install -y --no-install-recommends git +COPY . /src/sdcflows +RUN python -m build /src/sdcflows + +# Use Ubuntu 22.04 LTS +FROM ubuntu:jammy-20221130 # Prepare environment RUN apt-get update && \ @@ -35,8 +42,10 @@ RUN apt-get update && \ ca-certificates \ curl \ git \ + gnupg \ libtool \ lsb-release \ + netbase \ pkg-config \ unzip \ xvfb && \ @@ -138,10 +147,12 @@ RUN echo "Downloading Convert3D ..." \ ENV C3DPATH="/opt/convert3d-1.0.0" \ PATH="/opt/convert3d-1.0.0/bin:$PATH" +# Configure PPA for libpng12 +RUN GNUPGHOME=/tmp gpg --keyserver hkps://keyserver.ubuntu.com --no-default-keyring --keyring /usr/share/keyrings/linuxuprising.gpg --recv 0xEA8CACC073C3DB2A \ + && echo "deb [signed-by=/usr/share/keyrings/linuxuprising.gpg] https://ppa.launchpadcontent.net/linuxuprising/libpng12/ubuntu jammy main" > /etc/apt/sources.list.d/linuxuprising.list # AFNI latest (neurodocker build) RUN apt-get update -qq \ && apt-get install -y -q --no-install-recommends \ - apt-utils \ ed \ gsl-bin \ libglib2.0-0 \ @@ -149,6 +160,7 @@ RUN apt-get update -qq \ libglw1-mesa \ libgomp1 \ libjpeg62 \ + libpng12-0 \ libxm4 \ netpbm \ tcsh \ @@ -156,15 +168,12 @@ RUN apt-get update -qq \ xvfb \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ - && curl -sSL --retry 5 -o /tmp/multiarch.deb http://archive.ubuntu.com/ubuntu/pool/main/g/glibc/multiarch-support_2.27-3ubuntu1.2_amd64.deb \ + && curl -sSL --retry 5 -o /tmp/multiarch.deb http://archive.ubuntu.com/ubuntu/pool/main/g/glibc/multiarch-support_2.27-3ubuntu1.5_amd64.deb \ && dpkg -i /tmp/multiarch.deb \ && rm /tmp/multiarch.deb \ && curl -sSL --retry 5 -o /tmp/libxp6.deb http://mirrors.kernel.org/debian/pool/main/libx/libxp/libxp6_1.0.2-2_amd64.deb \ && dpkg -i /tmp/libxp6.deb \ && rm /tmp/libxp6.deb \ - && curl -sSL --retry 5 -o /tmp/libpng.deb http://snapshot.debian.org/archive/debian-security/20160113T213056Z/pool/updates/main/libp/libpng/libpng12-0_1.2.49-1%2Bdeb7u2_amd64.deb \ - && dpkg -i /tmp/libpng.deb \ - && rm /tmp/libpng.deb \ && apt-get install -f \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ @@ -242,13 +251,8 @@ COPY .docker/files/nipype.cfg $HOME/.nipype/nipype.cfg WORKDIR /src/sdcflows # Installing SDCFlows -COPY . /src/sdcflows -# Force static versioning within container -ARG VERSION -ENV SETUPTOOLS_SCM_PRETEND_VERSION=${VERSION} -RUN sed -i "s/fallback_version\s=\s\"0\.0\"/fallback_version = \"${VERSION}\"/g" pyproject.toml && \ - pip install --no-cache-dir .[all] && \ - rm -rf $HOME/.cache/pip +COPY --from=src /src/sdcflows/dist/*.whl . +RUN /opt/conda/bin/python -m pip install --no-cache-dir $( ls *.whl )[all] RUN find $HOME -type d -exec chmod go=u {} + && \ find $HOME -type f -exec chmod go=u {} + diff --git a/sdcflows/data/affine.json b/sdcflows/data/affine.json index ed2c37fc6d..4cace3284d 100644 --- a/sdcflows/data/affine.json +++ b/sdcflows/data/affine.json @@ -16,7 +16,6 @@ "smoothing_sigmas": [ [ 6, 0 ], [ 2, 1 ] ], "transform_parameters": [ [ 2.0 ], [ 1.0 ] ], "transforms": [ "Rigid", "Affine" ], - "use_estimate_learning_rate_once": [ false, false ], "use_histogram_matching": [ true, true ], "winsorize_lower_quantile": 0.001, "winsorize_upper_quantile": 0.998, diff --git a/sdcflows/data/fmap-any_registration.json b/sdcflows/data/fmap-any_registration.json index 0f34a54ffc..b33e73da94 100644 --- a/sdcflows/data/fmap-any_registration.json +++ b/sdcflows/data/fmap-any_registration.json @@ -16,7 +16,6 @@ "smoothing_sigmas": [ [ 8.0 ], [ 2.0 ] ], "transform_parameters": [ [ 0.1 ], [ 0.1 ] ], "transforms": [ "Rigid", "Rigid" ], - "use_estimate_learning_rate_once": [ true, false ], "use_histogram_matching": [ true, true ], "winsorize_lower_quantile": 0.001, "winsorize_upper_quantile": 0.999 diff --git a/sdcflows/data/fmap-any_registration_testing.json b/sdcflows/data/fmap-any_registration_testing.json index eb632ee273..c37f03853e 100644 --- a/sdcflows/data/fmap-any_registration_testing.json +++ b/sdcflows/data/fmap-any_registration_testing.json @@ -16,7 +16,6 @@ "smoothing_sigmas": [ [ 8.0 ], [ 2.0 ] ], "transform_parameters": [ [ 1.0 ], [ 0.5 ] ], "transforms": [ "Rigid", "Rigid" ], - "use_estimate_learning_rate_once": [ true, true ], "use_histogram_matching": [ true, true ], "winsorize_lower_quantile": 0.005, "winsorize_upper_quantile": 0.998 diff --git a/sdcflows/data/sd_syn.json b/sdcflows/data/sd_syn.json index f4bc76dc07..57b28ed082 100644 --- a/sdcflows/data/sd_syn.json +++ b/sdcflows/data/sd_syn.json @@ -16,7 +16,6 @@ "smoothing_sigmas": [ [ 2, 0 ], [ 0 ] ], "transform_parameters": [ [ 0.8, 6.0, 3.0 ], [ 0.8, 2.0, 1.0 ] ], "transforms": [ "SyN", "SyN" ], - "use_estimate_learning_rate_once": [ true, true ], "use_histogram_matching": [ true, true ], "winsorize_lower_quantile": 0.001, "winsorize_upper_quantile": 0.998, diff --git a/sdcflows/data/sd_syn_sloppy.json b/sdcflows/data/sd_syn_sloppy.json index cfecf850be..9560c48fd1 100644 --- a/sdcflows/data/sd_syn_sloppy.json +++ b/sdcflows/data/sd_syn_sloppy.json @@ -16,7 +16,6 @@ "smoothing_sigmas": [ [ 2, 0 ], [ 0 ] ], "transform_parameters": [ [ 0.8, 6.0, 3.0 ], [ 0.8, 2.0, 1.0 ] ], "transforms": [ "SyN", "SyN" ], - "use_estimate_learning_rate_once": [ true, true ], "use_histogram_matching": [ true, true ], "verbose": true, "winsorize_lower_quantile": 0.001, diff --git a/sdcflows/data/translation_rigid.json b/sdcflows/data/translation_rigid.json index 6e14b0225b..09e6ec51e9 100644 --- a/sdcflows/data/translation_rigid.json +++ b/sdcflows/data/translation_rigid.json @@ -6,7 +6,6 @@ "collapse_output_transforms": true, "write_composite_transform": true, "use_histogram_matching": [ true, true ], - "use_estimate_learning_rate_once": [ true, true ], "transforms": [ "Translation", "Rigid" ], "number_of_iterations": [ [ 500 ], [ 200 ] ], "transform_parameters": [ [ 0.05 ], [ 0.01 ] ], diff --git a/sdcflows/workflows/apply/tests/test_correction.py b/sdcflows/workflows/apply/tests/test_correction.py index d61860ffc6..04ba92d37b 100644 --- a/sdcflows/workflows/apply/tests/test_correction.py +++ b/sdcflows/workflows/apply/tests/test_correction.py @@ -23,6 +23,7 @@ """Test unwarp.""" from pathlib import Path from nipype.pipeline import engine as pe +from nipype.interfaces import utility as niu from ...fit.fieldmap import init_magnitude_wf from ..correction import init_unwarp_wf @@ -83,6 +84,8 @@ def test_unwarp_wf(tmpdir, datadir, workdir, outdir): from ...outputs import DerivativesDataSink from ....interfaces.reportlets import FieldmapReportlet + squeeze = pe.Node(niu.Function(function=_squeeze), name="squeeze") + report = pe.Node( SimpleBeforeAfter( before_label="Distorted", @@ -122,8 +125,9 @@ def test_unwarp_wf(tmpdir, datadir, workdir, outdir): # fmt: off workflow.connect([ (epi_ref_wf, report, [("outputnode.fmap_ref", "before")]), - (unwarp_wf, report, [("outputnode.corrected", "after"), - ("outputnode.corrected_mask", "wm_seg")]), + (unwarp_wf, squeeze, [("outputnode.corrected", "in_file")]), + (unwarp_wf, report, [("outputnode.corrected_mask", "wm_seg")]), + (squeeze, report, [("out", "after")]), (report, ds_report, [("out_report", "in_file")]), (epi_ref_wf, rep, [("outputnode.fmap_ref", "reference"), ("outputnode.fmap_mask", "mask")]), @@ -135,3 +139,18 @@ def test_unwarp_wf(tmpdir, datadir, workdir, outdir): if workdir: workflow.base_dir = str(workdir) workflow.run(plugin="Linear") + + +def _squeeze(in_file): + from pathlib import Path + import nibabel as nb + + img = nb.load(in_file) + squeezed = nb.squeeze_image(img) + + if squeezed.shape == img.shape: + return in_file + + out_fname = Path.cwd() / Path(in_file).name + squeezed.to_filename(out_fname) + return str(out_fname)