Merge pull request #203 from desihub/pure-MPI

pure-MPI implementation in a Podman container
desihub · Dec 31, 2024 · 42bf09d · 42bf09d
2 parents c59a6d2 + 8e508e5
commit 42bf09d
Show file tree

Hide file tree

Showing 17 changed files with 1,205 additions and 627 deletions.
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
@@ -31,6 +31,8 @@ jobs:
                 python-version: ${{ matrix.python-version }}
             - name: Install Python dependencies
               run: |
+                sudo apt-get update
+                sudo apt install libbz2-dev
                 python -m pip install --upgrade pip setuptools wheel
                 python -m pip install pytest
                 python -m pip install git+https://github.com/desihub/desiutil.git@${DESIUTIL_VERSION}#egg=desiutil
@@ -62,6 +64,8 @@ jobs:
                 python-version: ${{ matrix.python-version }}
             - name: Install Python dependencies
               run: |
+                sudo apt-get update
+                sudo apt install libbz2-dev
                 python -m pip install --upgrade pip setuptools wheel
                 python -m pip install pytest pytest-cov coveralls
                 python -m pip install git+https://github.com/desihub/desiutil.git@${DESIUTIL_VERSION}#egg=desiutil

diff --git a/bin/fastspec b/bin/fastspec
@@ -7,4 +7,7 @@ Example call:
 """
 if __name__ == '__main__':
     from fastspecfit.fastspecfit import fastspec
+    ## https://docs.nersc.gov/development/languages/python/parallel-python/#use-the-spawn-start-method
+    #import multiprocessing
+    #multiprocessing.set_start_method('spawn')
     fastspec(comm=None)
diff --git a/bin/mpi-fastspecfit b/bin/mpi-fastspecfit
diff --git a/doc/changes.rst b/doc/changes.rst
@@ -5,6 +5,7 @@ Change Log
 3.1.1 (not released yet)
 ------------------------
 
+* Pure-MPI implementation; new Podman container; bug fixes [`PR #203`_].
 * Updated algorithm for updating QSO redshifts [`PR #201`_].
 * Progress toward pure-MPI production code [`PR #200`_].
 * Fix <1% bias in fluxes and EWs of tied and free doublet ratios [`PR #198`_].
@@ -14,6 +15,7 @@ Change Log
 .. _`PR #198`: https://github.com/desihub/fastspecfit/pull/198
 .. _`PR #200`: https://github.com/desihub/fastspecfit/pull/200
 .. _`PR #201`: https://github.com/desihub/fastspecfit/pull/201
+.. _`PR #203`: https://github.com/desihub/fastspecfit/pull/203
 
 3.1.0 (2024-11-21)
 ------------------

diff --git a/podman/Containerfile b/podman/Containerfile
@@ -0,0 +1,131 @@
+FROM ubuntu:24.04
+
+WORKDIR /src
+
+RUN mkdir -p /src
+RUN apt-get -y clean && apt -y update && apt install -y apt-utils && apt -y upgrade
+
+RUN DEBIAN_FRONTEND=noninteractive \
+    apt install -y --no-install-recommends \
+    build-essential \
+    ca-certificates \
+    gfortran \
+    wget \
+    git \
+    libbz2-dev \
+    libgsl-dev \
+    libssl-dev \
+    libcfitsio-dev \
+    libcfitsio-bin \
+    # libhdf5 needed by h5py
+    libhdf5-dev \
+    && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+ARG mpich=3.4.3
+ARG mpich_prefix=mpich-$mpich
+
+ENV FFLAGS="-fallow-argument-mismatch" \
+    FCFLAGS="-fallow-argument-mismatch"
+
+RUN wget --no-check-certificate -nv https://www.mpich.org/static/downloads/$mpich/$mpich_prefix.tar.gz \
+    && tar xvzf $mpich_prefix.tar.gz \
+    && cd $mpich_prefix \
+    && ./configure --with-device=ch4:ofi \
+    && make -j 16 \
+    && make install \
+    && make clean \
+    && cd .. \
+    && rm -rf $mpich_prefix \
+    && rm -f $mpich_prefix.tar.gz
+
+RUN /sbin/ldconfig
+
+## Try to prevent MKL from throttling AMD
+## https://gitlab.com/NERSC/python-benchmark/-/tree/main/amd
+#COPY fakeintel.c /src/fakeintel.c
+#RUN gcc -shared -fPIC -o /usr/local/lib/libfakeintel.so /src/fakeintel.c
+#ENV LD_PRELOAD=/usr/local/lib/libfakeintel.so
+
+# Install Miniconda
+ENV CONDA_DIR=/opt/miniconda
+
+ENV MINICONDA_VERSION=Miniforge3-Linux-x86_64.sh
+ENV MINICONDA=https://github.com/conda-forge/miniforge/releases/latest/download/$MINICONDA_VERSION
+
+RUN wget --no-check-certificate $MINICONDA \
+    && bash $MINICONDA_VERSION -b -p $CONDA_DIR \
+    && rm -f $MINICONDA_VERSION
+
+# Update PATH environment variable
+ENV PATH="$CONDA_DIR/bin:$PATH"
+
+# Verify Miniconda installation and initialize environment
+RUN conda init bash && conda update -n base -c defaults conda -y
+
+# Set default shell to bash
+SHELL ["/bin/bash", "-c"]
+
+# Install all our dependencies
+RUN conda install -y -c conda-forge \
+    "python<3.13" \
+    wheel \
+    setuptools \
+    pytest \
+    "numpy<2.0" \
+    scipy \
+    astropy \
+    healpy \
+    fitsio \
+    numba \
+    seaborn \
+    matplotlib \
+    ipython \
+    ipykernel \
+    h5py
+
+RUN conda install -y -c conda-forge \
+    libblas=*=*mkl \
+    # also tried mkl_fft<1.3.9
+    # mkl_fft \
+    intel-cmplr-lib-rt \
+    && conda clean --all -y
+
+# Need to install mpi4py from source to link it properly to MPICH.
+ENV PIP_ROOT_USER_ACTION=ignore
+RUN pip install --no-cache-dir --no-binary=mpi4py mpi4py
+
+ENV DESIUTIL_VERSION=3.4.3
+ENV DESIMODEL_VERSION=0.19.2
+ENV DESITARGET_VERSION=2.8.0
+ENV DESISPEC_VERSION=0.68.1
+ENV SPECLITE_VERSION=v0.20
+ENV FASTSPECFIT_VERSION=main
+#ENV FASTSPECFIT_VERSION=3.1.1
+
+RUN pip install git+https://github.com/desihub/desiutil.git@${DESIUTIL_VERSION}#egg=desiutil
+RUN pip install git+https://github.com/desihub/desimodel.git@${DESIMODEL_VERSION}#egg=desimodel
+RUN pip install git+https://github.com/desihub/desitarget.git@${DESITARGET_VERSION}#egg=desitarget
+RUN pip install git+https://github.com/desihub/desispec.git@${DESISPEC_VERSION}#egg=desispec
+RUN pip install git+https://github.com/desihub/speclite.git@${SPECLITE_VERSION}#egg=speclite
+RUN pip install git+https://github.com/desihub/fastspecfit.git@${FASTSPECFIT_VERSION}#egg=fastspecfit
+
+ENV DESI_SPECTRO_REDUX=/dvs_ro/cfs/cdirs/desi/spectro/redux
+ENV DUST_DIR=/dvs_ro/cfs/cdirs/cosmo/data/dust/v0_1
+ENV FPHOTO_DIR=/dvs_ro/cfs/cdirs/desi/external/legacysurvey/dr9
+ENV FTEMPLATES_DIR=/dvs_ro/cfs/cdirs/desi/public/external/templates/fastspecfit
+
+# Some environment variables to ensure good performance with MKL.
+# https://www.diracprogram.org/doc/master/installation/mkl.html
+ENV MKL_NUM_THREADS=1
+ENV MKL_DYNAMIC="FALSE"
+ENV OMP_NUM_THREADS=1
+
+# Create the numba cache.
+ENV HOME=/homedir
+ENV NUMBA_CACHE_DIR=$HOME/numba_cache
+
+RUN mkdir -p /homedir/numba_cache \
+    && chmod -R 777 /homedir*
+RUN pytest /opt/miniconda/lib/python3.12/site-packages/fastspecfit/test/test_fastspecfit.py
+
+LABEL Maintainer="John Moustakas [email protected]"
diff --git a/podman/Containerfile-mkl b/podman/Containerfile-mkl
@@ -0,0 +1,131 @@
+FROM ubuntu:24.04
+
+WORKDIR /src
+
+RUN mkdir -p /src
+RUN apt-get -y clean && apt -y update && apt install -y apt-utils && apt -y upgrade
+
+RUN DEBIAN_FRONTEND=noninteractive \
+    apt install -y --no-install-recommends \
+    build-essential \
+    ca-certificates \
+    gfortran \
+    wget \
+    git \
+    libbz2-dev \
+    libgsl-dev \
+    libssl-dev \
+    libcfitsio-dev \
+    libcfitsio-bin \
+    # libhdf5 needed by h5py
+    libhdf5-dev \
+    && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+ARG mpich=3.4.3
+ARG mpich_prefix=mpich-$mpich
+
+ENV FFLAGS="-fallow-argument-mismatch" \
+    FCFLAGS="-fallow-argument-mismatch"
+
+RUN wget --no-check-certificate -nv https://www.mpich.org/static/downloads/$mpich/$mpich_prefix.tar.gz \
+    && tar xvzf $mpich_prefix.tar.gz \
+    && cd $mpich_prefix \
+    && ./configure --with-device=ch4:ofi \
+    && make -j 16 \
+    && make install \
+    && make clean \
+    && cd .. \
+    && rm -rf $mpich_prefix \
+    && rm -f $mpich_prefix.tar.gz
+
+RUN /sbin/ldconfig
+
+# Try to prevent MKL from throttling AMD
+# https://gitlab.com/NERSC/python-benchmark/-/tree/main/amd
+COPY fakeintel.c /src/fakeintel.c
+RUN gcc -shared -fPIC -o /usr/local/lib/libfakeintel.so /src/fakeintel.c
+ENV LD_PRELOAD=/usr/local/lib/libfakeintel.so
+
+# Install Miniconda
+ENV CONDA_DIR=/opt/miniconda
+
+ENV MINICONDA_VERSION=Miniforge3-Linux-x86_64.sh
+ENV MINICONDA=https://github.com/conda-forge/miniforge/releases/latest/download/$MINICONDA_VERSION
+
+RUN wget --no-check-certificate $MINICONDA \
+    && bash $MINICONDA_VERSION -b -p $CONDA_DIR \
+    && rm -f $MINICONDA_VERSION
+
+# Update PATH environment variable
+ENV PATH="$CONDA_DIR/bin:$PATH"
+
+# Verify Miniconda installation and initialize environment
+RUN conda init bash && conda update -n base -c defaults conda -y
+
+# Set default shell to bash
+SHELL ["/bin/bash", "-c"]
+
+# Install all our dependencies
+RUN conda install -y -c conda-forge \
+    "python<3.13" \
+    wheel \
+    setuptools \
+    pytest \
+    "numpy<2.0" \
+    scipy \
+    astropy \
+    healpy \
+    fitsio \
+    numba \
+    seaborn \
+    matplotlib \
+    ipython \
+    ipykernel \
+    h5py
+
+RUN conda install -y -c conda-forge \
+    libblas=*=*mkl \
+    # also tried mkl_fft<1.3.9
+    mkl_fft \
+    intel-cmplr-lib-rt \
+    && conda clean --all -y
+
+# Need to install mpi4py from source to link it properly to MPICH.
+ENV PIP_ROOT_USER_ACTION=ignore
+RUN pip install --no-cache-dir --no-binary=mpi4py mpi4py
+
+ENV DESIUTIL_VERSION=3.4.3
+ENV DESIMODEL_VERSION=0.19.2
+ENV DESITARGET_VERSION=2.8.0
+ENV DESISPEC_VERSION=0.68.1
+ENV SPECLITE_VERSION=v0.20
+ENV FASTSPECFIT_VERSION=main
+#ENV FASTSPECFIT_VERSION=3.1.1
+
+RUN pip install git+https://github.com/desihub/desiutil.git@${DESIUTIL_VERSION}#egg=desiutil
+RUN pip install git+https://github.com/desihub/desimodel.git@${DESIMODEL_VERSION}#egg=desimodel
+RUN pip install git+https://github.com/desihub/desitarget.git@${DESITARGET_VERSION}#egg=desitarget
+RUN pip install git+https://github.com/desihub/desispec.git@${DESISPEC_VERSION}#egg=desispec
+RUN pip install git+https://github.com/desihub/speclite.git@${SPECLITE_VERSION}#egg=speclite
+RUN pip install git+https://github.com/desihub/fastspecfit.git@${FASTSPECFIT_VERSION}#egg=fastspecfit
+
+ENV DESI_SPECTRO_REDUX=/dvs_ro/cfs/cdirs/desi/spectro/redux
+ENV DUST_DIR=/dvs_ro/cfs/cdirs/cosmo/data/dust/v0_1
+ENV FPHOTO_DIR=/dvs_ro/cfs/cdirs/desi/external/legacysurvey/dr9
+ENV FTEMPLATES_DIR=/dvs_ro/cfs/cdirs/desi/public/external/templates/fastspecfit
+
+# Some environment variables to ensure good performance with MKL.
+# https://www.diracprogram.org/doc/master/installation/mkl.html
+ENV MKL_NUM_THREADS=1
+ENV MKL_DYNAMIC="FALSE"
+ENV OMP_NUM_THREADS=1
+
+# Create the numba cache.
+ENV HOME=/homedir
+ENV NUMBA_CACHE_DIR=$HOME/numba_cache
+
+RUN mkdir -p /homedir/numba_cache \
+    && chmod -R 777 /homedir*
+RUN pytest /opt/miniconda/lib/python3.12/site-packages/fastspecfit/test/test_fastspecfit.py
+
+LABEL Maintainer="John Moustakas [email protected]"