...

pdidev · Feb 22, 2024 · f259bdb · f259bdb
1 parent 0fd123b
commit f259bdb
Show file tree

Hide file tree

Showing 9 changed files with 79 additions and 104 deletions.
diff --git a/spack/1b-spackbase/Dockerfile b/spack/1b-spackbase/Dockerfile
@@ -40,7 +40,6 @@ RUN set -ex \
  && apt-get autoclean -y \
  && locale-gen ${LC_ALL} \
  && rm -rf /var/lib/apt/lists/* /run/nologin \
- && pip3 --no-cache-dir install boto3 \
  && ln -s ${SPACK_ROOT}/share/spack/docker/entrypoint.bash /usr/local/bin/docker-shell \
  && ln -s ${SPACK_ROOT}/share/spack/docker/entrypoint.bash /usr/local/bin/interactive-shell \
  && ln -s ${SPACK_ROOT}/share/spack/docker/entrypoint.bash /usr/local/bin/spack-env
diff --git a/spack/2a-precomp/Dockerfile b/spack/2a-precomp/Dockerfile
@@ -19,18 +19,23 @@ ENTRYPOINT ["/bin/bash", "/opt/spack/share/spack/docker/entrypoint.bash"]
 CMD ["interactive-shell"]
 
 
-RUN spack compiler find --scope site
-RUN spack config --scope site add 'packages:all:target:[x86_64]'
-RUN spack config --scope site add "config:install_tree:root:${SPACK_INSTALL}"
-RUN spack config --scope site add 'modules:prefix_inspections:lib64:[LD_LIBRARY_PATH]'
-RUN spack config --scope site add 'modules:prefix_inspections:lib:[LD_LIBRARY_PATH]'
-RUN spack config --scope site add "modules:default:roots:tcl:${SPACK_MODULES}"
+RUN spack compiler find --scope site \
+ && spack config --scope site add 'packages:all:target:[x86_64]' \
+ && spack config --scope site add "config:install_tree:root:${SPACK_INSTALL}" \
+ && spack config --scope site add 'modules:prefix_inspections:lib64:[LD_LIBRARY_PATH]' \
+ && spack config --scope site add 'modules:prefix_inspections:lib:[LD_LIBRARY_PATH]' \
+ && spack config --scope site add "modules:default:roots:tcl:${SPACK_MODULES}" \
+ && spack clean -a
 
 
 ARG COMPILER
 ARG SPACK_INSTALL_ARGS="--show-log-on-error --fail-fast"
 
 COPY ${COMPILER}.lst compilerpkg.lst
 RUN spack solve -I --reuse $(cat compilerpkg.lst) target=x86_64 \
+ && ( spack fetch -Dm $(cat compilerpkg.lst) \
+   || spack fetch -Dm $(cat compilerpkg.lst) \
+   || spack fetch -Dm $(cat compilerpkg.lst) \
+   || spack fetch -Dm $(cat compilerpkg.lst) ) \
  && spack install ${SPACK_INSTALL_ARGS} --reuse --only dependencies $(cat compilerpkg.lst) target=x86_64 \
  && spack clean -a
diff --git a/spack/2b-compiler/Dockerfile b/spack/2b-compiler/Dockerfile
@@ -10,6 +10,10 @@ LABEL "org.opencontainers.image.source"="https://github.com/pdidev/test_env"
 ARG SPACK_INSTALL_ARGS="--show-log-on-error --fail-fast"
 
 RUN spack solve -I --reuse $(cat compilerpkg.lst) target=x86_64 \
+ && ( spack fetch -Dm $(cat compilerpkg.lst) \
+   || spack fetch -Dm $(cat compilerpkg.lst) \
+   || spack fetch -Dm $(cat compilerpkg.lst) \
+   || spack fetch -Dm $(cat compilerpkg.lst) ) \
  && spack install ${SPACK_INSTALL_ARGS} --reuse $(cat compilerpkg.lst) target=x86_64 \
  && spack clean -a
 
@@ -24,27 +28,17 @@ RUN spack find --no-groups  "$(cat compilerpkg.lst)" > compilerpkg.lst \
 
 # switch to our new, just built, compiler
 RUN spack compiler remove --scope site -a gcc || true \
- && spack clean -a
-RUN spack compiler remove --scope site -a clang || true \
- && spack clean -a
-RUN spack compiler remove -a clang || true \
- && spack clean -a
-RUN spack compiler remove -a gcc || true \
- && spack clean -a
-RUN spack compiler find --scope site \
+ && spack compiler remove --scope site -a clang || true \
+ && spack compiler remove -a clang || true \
+ && spack compiler remove -a gcc || true \
+ && spack compiler find --scope site \
           "$(spack find --format '{name}' --paths "$(cat compilerpkg.lst)" | awk '{print $2}')" \
- && spack clean -a
-RUN spack compiler remove --scope defaults -a gcc || true \
- && spack clean -a
-RUN spack compiler remove --scope defaults -a clang || true \
- && spack clean -a
-RUN spack compiler remove --scope system -a gcc || true \
- && spack clean -a
-RUN spack compiler remove --scope system -a clang || true \
- && spack clean -a
-RUN spack compiler remove --scope user -a gcc || true \
- && spack clean -a
-RUN spack compiler remove --scope user -a clang || true \
+ && spack compiler remove --scope defaults -a gcc || true \
+ && spack compiler remove --scope defaults -a clang || true \
+ && spack compiler remove --scope system -a gcc || true \
+ && spack compiler remove --scope system -a clang || true \
+ && spack compiler remove --scope user -a gcc || true \
+ && spack compiler remove --scope user -a clang || true \
  && spack clean -a
 
 

diff --git a/spack/3a-precomp/Dockerfile b/spack/3a-precomp/Dockerfile
@@ -9,10 +9,8 @@ LABEL "org.opencontainers.image.source"="https://github.com/pdidev/test_env"
 
 COPY spack.yaml spack.yaml
 RUN spack env create pdienv spack.yaml \
- && spack clean -a
-RUN spack -e pdienv add -l compilerpkg $(cat compilerpkg.lst) \
- && spack clean -a
-RUN spack -e pdienv add -l compiler $(cat compiler.lst) \
+ && spack -e pdienv add -l compilerpkg $(cat compilerpkg.lst) \
+ && spack -e pdienv add -l compiler $(cat compiler.lst) \
  && spack clean -a
 
 
@@ -29,10 +27,9 @@ ARG SPACK_INSTALL_ARGS="--show-log-on-error --fail-fast"
 
 RUN spack -e pdienv concretize -fU \
  && spack clean -a
-RUN spack -e pdienv fetch $(cat compilerpkg.lst) \
- || spack -e pdienv fetch $(cat compilerpkg.lst) \
- || spack -e pdienv fetch $(cat compilerpkg.lst) \
- || spack -e pdienv fetch $(cat compilerpkg.lst) \
- && spack clean -a
-RUN spack -e pdienv install ${SPACK_INSTALL_ARGS} --only dependencies $(cat compilerpkg.lst) \
+RUN ( spack -e pdienv fetch -m \
+   || spack -e pdienv fetch -m \
+   || spack -e pdienv fetch -m \
+   || spack -e pdienv fetch -m ) \
+ && spack -e pdienv install ${SPACK_INSTALL_ARGS} --only dependencies $(cat compilerpkg.lst) \
  && spack clean -a
diff --git a/spack/3b-compiler/Dockerfile b/spack/3b-compiler/Dockerfile
@@ -11,7 +11,11 @@ ARG SPACK_INSTALL_ARGS="--show-log-on-error --fail-fast"
 
 RUN spack -e pdienv concretize -fU \
  && spack clean -a
-RUN spack -e pdienv install ${SPACK_INSTALL_ARGS} $(cat compilerpkg.lst) \
+RUN ( spack -e pdienv fetch -m \
+   || spack -e pdienv fetch -m \
+   || spack -e pdienv fetch -m \
+   || spack -e pdienv fetch -m ) \
+ && spack -e pdienv install ${SPACK_INSTALL_ARGS} $(cat compilerpkg.lst) \
  && spack mark -ia \
  && spack mark -e "/$(spack -e pdienv find --format '{name}' --no-groups -L $(cat compilerpkg.lst) | awk '{print $1}')" \
  && spack gc -y \
@@ -20,27 +24,17 @@ RUN spack -e pdienv install ${SPACK_INSTALL_ARGS} $(cat compilerpkg.lst) \
 
 # switch to our new, just built, compiler
 RUN spack compiler remove --scope site -a gcc || true \
- && spack clean -a
-RUN spack compiler remove --scope site -a clang || true \
- && spack clean -a
-RUN spack compiler remove -a clang || true \
- && spack clean -a
-RUN spack compiler remove -a gcc || true \
- && spack clean -a
-RUN spack compiler find --scope site \
+ && spack compiler remove --scope site -a clang || true \
+ && spack compiler remove -a clang || true \
+ && spack compiler remove -a gcc || true \
+ && spack compiler find --scope site \
           "$(spack find --format '{name}' --paths "$(cat compilerpkg.lst)" | awk '{print $2}')" \
- && spack clean -a
-RUN spack compiler remove --scope defaults -a gcc || true \
- && spack clean -a
-RUN spack compiler remove --scope defaults -a clang || true \
- && spack clean -a
-RUN spack compiler remove --scope system -a gcc || true \
- && spack clean -a
-RUN spack compiler remove --scope system -a clang || true \
- && spack clean -a
-RUN spack compiler remove --scope user -a gcc || true \
- && spack clean -a
-RUN spack compiler remove --scope user -a clang || true \
+ && spack compiler remove --scope defaults -a gcc || true \
+ && spack compiler remove --scope defaults -a clang || true \
+ && spack compiler remove --scope system -a gcc || true \
+ && spack compiler remove --scope system -a clang || true \
+ && spack compiler remove --scope user -a gcc || true \
+ && spack compiler remove --scope user -a clang || true \
  && spack clean -a
 
 

diff --git a/spack/4a-baselibs/Dockerfile b/spack/4a-baselibs/Dockerfile
@@ -11,10 +11,9 @@ ARG SPACK_INSTALL_ARGS="--show-log-on-error --fail-fast"
 
 RUN spack -e pdienv concretize -fU \
  && spack clean -a
-RUN spack -e pdienv fetch \
- || spack -e pdienv fetch \
- || spack -e pdienv fetch \
- || spack -e pdienv fetch \
- && spack clean -a
-RUN spack -e pdienv install ${SPACK_INSTALL_ARGS} \
+RUN ( spack -e pdienv fetch -m \
+   || spack -e pdienv fetch -m \
+   || spack -e pdienv fetch -m \
+   || spack -e pdienv fetch -m ) \
+ && spack -e pdienv install ${SPACK_INSTALL_ARGS} \
  && spack clean -a
diff --git a/spack/5a-mpi/Dockerfile b/spack/5a-mpi/Dockerfile
@@ -22,17 +22,15 @@ ARG SPACK_INSTALL_ARGS="--show-log-on-error --fail-fast"
 
 RUN spack -e pdienv concretize -fU \
  && spack clean -a
-RUN spack -e pdienv fetch \
- || spack -e pdienv fetch \
- || spack -e pdienv fetch \
- || spack -e pdienv fetch \
- && spack clean -a
 # workaround clang issue with mixed compilers where Fortran CHARACTER detection script requires CFLAGS=-fPIE
 # Still present with spack 0.18
-RUN if ( [ 'xoldest' = "x${VERSION}" ] && grep -qs clang compiler.lst ) \
+RUN ( spack -e pdienv fetch -m \
+   || spack -e pdienv fetch -m \
+   || spack -e pdienv fetch -m \
+   || spack -e pdienv fetch -m ) \
+ && if ( [ 'xoldest' = "x${VERSION}" ] && grep -qs clang compiler.lst ) \
   ; then spack -e pdienv install --only dependencies ${SPACK_INSTALL_ARGS} $(cat mpi.lst) \
       && ( export CFLAGS=-fPIE; spack -e pdienv install ${SPACK_INSTALL_ARGS} --dirty $(cat mpi.lst) ) \
   ; fi \
- && spack clean -a
-RUN spack -e pdienv install ${SPACK_INSTALL_ARGS} \
+ && spack -e pdienv install ${SPACK_INSTALL_ARGS} \
  && spack clean -a
diff --git a/spack/6-optlibs/Dockerfile b/spack/6-optlibs/Dockerfile
@@ -8,13 +8,15 @@ FROM ghcr.io/pdidev/test_env/builder:${VERSION}-${COMPILER}-${MPI}-mini-${IMAGE_
 LABEL "org.opencontainers.image.source"="https://github.com/pdidev/test_env"
 
 
-RUN git clone --depth 1 https://github.com/leobago/fti-spack ${SPACK_ROOT}/var/spack/repos/fti
-RUN rm -rf ${SPACK_ROOT}/var/spack/repos/fti/.git
-RUN spack repo add --scope site ${SPACK_ROOT}/var/spack/repos/fti
+RUN git clone --depth 1 https://github.com/leobago/fti-spack ${SPACK_ROOT}/var/spack/repos/fti \
+ && rm -rf ${SPACK_ROOT}/var/spack/repos/fti/.git \
+ && spack repo add --scope site ${SPACK_ROOT}/var/spack/repos/fti \
+ && spack clean -a
 
-RUN git clone --depth 1 https://github.com/pdidev/spack ${SPACK_ROOT}/var/spack/repos/pdi
-RUN rm -rf ${SPACK_ROOT}/var/spack/repos/pdi/.git
-RUN spack repo add --scope site ${SPACK_ROOT}/var/spack/repos/pdi
+RUN git clone --depth 1 https://github.com/pdidev/spack ${SPACK_ROOT}/var/spack/repos/pdi \
+ && rm -rf ${SPACK_ROOT}/var/spack/repos/pdi/.git \
+ && spack repo add --scope site ${SPACK_ROOT}/var/spack/repos/pdi \
+ && spack clean -a
 
 
 ARG VERSION
@@ -27,31 +29,19 @@ RUN DEFAULT_IFS="${IFS}" \
   ; set -f \
   ; for LIB in $(cat optlibs.lst)\
   ; do IFS="${DEFAULT_IFS}" spack -e pdienv add -l optlibs "${LIB}" \
-  ; done
+  ; done \
+ && spack clean -a
 
-# # some package need to use a more recent cmake than that of in old env, so remove it
-# RUN spack -e pdienv remove -l baselibs cmake
 
 
 ARG SPACK_INSTALL_ARGS="--show-log-on-error --fail-fast"
 
 # install the libs
 RUN spack -e pdienv concretize -fU \
  && spack clean -a
-RUN spack -e pdienv fetch \
- || spack -e pdienv fetch \
- || spack -e pdienv fetch \
- || spack -e pdienv fetch \
+RUN ( spack -e pdienv fetch -m \
+   || spack -e pdienv fetch -m \
+   || spack -e pdienv fetch -m \
+   || spack -e pdienv fetch -m ) \
+ && spack -e pdienv install ${SPACK_INSTALL_ARGS} \
  && spack clean -a
-RUN spack -e pdienv install ${SPACK_INSTALL_ARGS} \
- && spack clean -a
-
-# # add back the cmake version that was removed before
-# RUN spack -e pdienv add -l baselibs "$(grep '^cmake' baselibs.lst)"
-# RUN spack -e pdienv concretize --reuse
-# RUN  ( spack -e pdienv fetch \
-#     || spack -e pdienv fetch \
-#     || spack -e pdienv fetch \
-#     || spack -e pdienv fetch ) \
-#  && spack -e pdienv install ${SPACK_INSTALL_ARGS} \
-#  && spack clean -a
diff --git a/spack/7-finalize/Dockerfile b/spack/7-finalize/Dockerfile
@@ -13,15 +13,17 @@ RUN spack mark -ia \
  && for N in $(spack -e pdienv find --format '{name}' --no-groups -L | awk '{print $1}') \
   ; do spack mark -e "/$N" \
   ; done \
- && spack gc -y
+ && spack gc -y \
+ && spack clean -a
 
 RUN find -L ${SPACK_INSTALL} -type f -exec file -i '{}' '+' \
   | awk -F: '/(x-executable|x-sharedlib); charset=binary/ {print $1}' \
   | xargs -r strip -s
 
-RUN spack config --scope site add 'modules:default:tcl:all:autoload:direct'
-RUN spack -e pdienv module tcl refresh --delete-tree -y
-RUN spack -e pdienv env loads
+RUN spack config --scope site add 'modules:default:tcl:all:autoload:direct' \
+ && spack -e pdienv module tcl refresh --delete-tree -y \
+ && spack -e pdienv env loads \
+ && spack clean -a
 
 
 ARG VERSION
@@ -43,9 +45,6 @@ RUN if [[ "xall" = "x${LIBS}" ]] \
 RUN echo "MODULEPATH=${MODULEPATH}" >> /etc/pdienv.env.sh
 
 
-RUN spack clean -a
-
-
 
 FROM ghcr.io/pdidev/test_env/base:${IMAGE_TAG} AS main