Merge branch 'JCSDA:develop' into develop

configs/common/packages.yaml

@@ -160,7 +160,7 @@ packages:
   magics:
     require: "@4.15.3:"
   mapl:
-    require: '@2.46.3 ~shared ~f2py'
+    require: '@2.53.0 ~shared ~f2py'
     variants: '+pflogger'
   met:
     require: '@11.1.1 +python +grib2'

configs/containers/specs/jedi-ci.yaml

@@ -15,7 +15,7 @@
     [email protected],
     [email protected] +fckit +trans +tesselation +fftw,
     [email protected],
-    ectrans@1.2.0,
+    ectrans@1.5.0,
     [email protected],
     [email protected],
     [email protected],
@@ -47,8 +47,8 @@
     [email protected],
     [email protected],
     nco,
-    esmf@8.6.1,
-    mapl@2.46.3,
+    esmf@8.8.0,
+    mapl@2.53.0,
     zlib-ng,
     zstd,
     [email protected],

doc/source/MaintainersSection.rst

@@ -90,7 +90,7 @@ For certain limited use cases, we need to provide ``mysql`` through spack-stack.
 ..  _MaintainersSection_Texlive:
 
 ------------------------------
-Texlive (TeX/LaTeX)
+Texlive (TeX/LaTeX; optional)
 ------------------------------
 
 Building ``texlive`` isn't straightforward as it has many dependencies. Since it is only used to generated documentation for ``spack-stack`` (and other projects), i.e. not to compile any code, it makes no sense to build it with ``spack``. We therefore require ``texlive`` or any other compatible TeX/LaTeX distribution as an external package.
@@ -324,7 +324,7 @@ The newly created local mirror should be listed at the top, which means that spa
 Spack mirrors for air-gapped systems
 ------------------------------------
 
-The procedure is similar to using spack mirrors for local reuse, but a few additional steps are needed in between.
+The procedure for creating source caches is similar to using spack mirrors for local reuse, but a few additional steps are needed in between. The steps to create a mirror for bootstrapping spack on an air-gapped system are described in the next section.
 
 1. On the air-gapped system: Create an environment as usual, activate it and run the concretization step (``spack concretize``), but do not start the installation yet.
 
@@ -358,6 +358,43 @@ The procedure is similar to using spack mirrors for local reuse, but a few addit
 
 7. On the air-gapped system: Proceed with the installation as usual.
 
+----------------------------------------
+Bootstrap mirrors for air-gapped systems
+----------------------------------------
+
+To use spack on an air-gapped system, the first step is to create a mirror that allows spack to bootstrap from. The logic is similar to that for creating source caches for spack installations. On a system with access to the internet, a series of commands is run to create a bootstrap mirror. The bootstrap mirror is then copied over to the air-gapped system and registered so that spack can use it.
+
+1. On the system with access to the internet, and using the same version of spack-stack (and its spack submodule) as on the air-gapped target system:
+
+.. code-block:: console
+
+   # After running "source setup.sh" in the spack-stack top-level directory
+
+   # Create bootstrap mirror
+   spack bootstrap mirror --binary-packages ./bootstrap-mirror
+
+2. Transfer ``./bootstrap-mirror`` to the air-gapped system, preferably to the same location inside the spack-stack top-level directory (i.e. ``${SPACK_STACK_DIR}/bootstrap-mirror/``).
+
+3. On the air-gapped system, run the following series of commands:
+
+.. code-block:: console
+
+   # After running "source setup.sh" in the spack-stack top-level directory
+
+   # Register bootstrap mirror
+   spack bootstrap add --trust local-sources /absolute/path/to/spack-stack/bootstrap-mirror/metadata/sources
+   spack bootstrap add --trust local-binaries /absolute/path/to/spack-stack/bootstrap-mirror/metadata/sources
+
+   # Update the boostrap binary cache index (silent response to this command means success)
+   spack buildcache update-index /absolute/path/to/spack-stack/bootstrap-mirror/bootstrap_cache
+
+4. Bootstrap spack explicitly before creating environments and check the status. There may be error messages along the way, but as long as the command completes and ``spack bootstrap status`` shows ``PASS`` for both ``Core Functionalities`` and ``Binary packages``, it was successful.
+
+.. code-block:: console
+
+   spack bootstrap now
+   spack bootstrap status
+
 ==============================
 Testing new packages
 ==============================

util/nrl/batch_install.sh

@@ -2,17 +2,29 @@
 
 set -e
 
-# Developer switch: create buildcaches instead of deploying environments ["true"|"false"]
-# The default "false" means to deploy environments using existing buildcaches (installer mode)
+# DH* NOT YET IMPLEMENTED. LEAVING THE COMMENT IN CASE WE NEED TO IMPLEMENT LATER,
+# OTHERWISE I WILL REMOVE THE COMMENT LATER.
+# Uncomment and set this variable to full path on the system (directory must exist
+# and mut be writeable by the current user). This is the location where environments
+# will be deployed, e.g. SPACK_STACK_ENVIRONMENT_DIR=/path/to/envs means that the
+# environments a, b, and c will be deployed in /path/to/envs/{a,b,c}. Not that this
+# option only applies in "installer" mode (see next comment).
+#SPACK_STACK_ENVIRONMENT_DIR=
+# *DH
+
+# Developer switch: create buildcaches instead of deploying environments ["true"|"false"].
+# The default "false" means to deploy environments using existing buildcaches (installer mode).
 SPACK_STACK_BATCH_CREATE_BUILDCACHE="false"
 #SPACK_STACK_BATCH_CREATE_BUILDCACHE="true"
 
 # A value of SPACK_STACK_BATCH_CREATE_BUILDCACHE == "true" enters developer mode. In this
 # mode, one must choose between reusing existing buildcaches or rebuilding from scratch.
-# This variable is meaningless in installer mode (SPACK_STACK_BATCH_CREATE_BUILDCACHE== "false")
+# This variable is meaningless in installer mode (SPACK_STACK_BATCH_CREATE_BUILDCACHE== "false").
 SPACK_STACK_BATCH_REUSE_EXISTING_BUILDCACHE="true"
 #SPACK_STACK_BATCH_REUSE_EXISTING_BUILDCACHE="false"
 
+##################################################################################################
+
 # Remove domain name suffices and digits to determine hostname
 SPACK_STACK_BATCH_HOST=$(echo ${HOSTNAME} | cut -d "." -f 1)
 SPACK_STACK_BATCH_HOST=${SPACK_STACK_BATCH_HOST//[0-9]/}
@@ -22,27 +34,32 @@ case ${SPACK_STACK_BATCH_HOST} in
     SPACK_STACK_BATCH_COMPILERS=("[email protected]" "[email protected]" "[email protected]")
     SPACK_STACK_BATCH_TEMPLATES=("neptune-dev" "unified-dev" "cylc-dev")
     SPACK_STACK_MODULE_CHOICE="lmod"
+    SPACK_STACK_BOOTSTRAP_MIRROR="/neptune_diagnostics/spack-stack/bootstrap-mirror"
     ;;
   narwhal)
     SPACK_STACK_BATCH_COMPILERS=("[email protected]" "[email protected]" "[email protected]")
     SPACK_STACK_BATCH_TEMPLATES=("neptune-dev" "unified-dev" "cylc-dev")
     SPACK_STACK_MODULE_CHOICE="tcl"
+    SPACK_STACK_BOOTSTRAP_MIRROR="/p/cwfs/projects/NEPTUNE/spack-stack/bootstrap-mirror"
     ;;
   nautilus)
     SPACK_STACK_BATCH_COMPILERS=("[email protected]" "[email protected]" "[email protected]")
     SPACK_STACK_BATCH_TEMPLATES=("neptune-dev" "unified-dev" "cylc-dev")
     SPACK_STACK_MODULE_CHOICE="tcl"
+    SPACK_STACK_BOOTSTRAP_MIRROR="/p/cwfs/projects/NEPTUNE/spack-stack/bootstrap-mirror"
     ;;
   # DH*
   blackpearl)
     SPACK_STACK_BATCH_COMPILERS=("[email protected]" "[email protected]" "[email protected]")
     SPACK_STACK_BATCH_TEMPLATES=("neptune-dev" "unified-dev" "cylc-dev")
     SPACK_STACK_MODULE_CHOICE="tcl"
+    SPACK_STACK_BOOTSTRAP_MIRROR="/home/dom/prod/spack-bootstrap-mirror"
     ;;
   bounty)
     SPACK_STACK_BATCH_COMPILERS=("[email protected]" "[email protected]" "[email protected]" "[email protected]")
     SPACK_STACK_BATCH_TEMPLATES=("neptune-dev" "unified-dev" "cylc-dev")
     SPACK_STACK_MODULE_CHOICE="tcl"
+    SPACK_STACK_BOOTSTRAP_MIRROR="/home/dom/prod/spack-bootstrap-mirror"
     ;;
   # *DH
   *)
@@ -53,6 +70,56 @@ esac
 
 ##################################################################################################
 
+function fix_permissions() {
+  host=$1
+  dir=$2
+  executables=$3
+  echo "Repairing permissions for directory ${dir} on ${host} ..."
+  set +e
+  case ${host} in
+    atlantis)
+      nice -n 19 find ${dir} -type d -print0 | xargs --null chmod a+rx
+      if [[ ${executables} -eq 1 ]]; then
+        nice -n 19 find ${dir} -type f -executable -print0 | xargs --null chmod a+rx
+      fi
+      nice -n 19 find ${dir} -type f -print0 | xargs --null chmod a+r
+      ;;
+    narwhal)
+      nice -n 19 lfs find ${dir} -type d -print0 | xargs --null chmod a+rx
+      # In case the find command returns no executables
+      if [[ ${executables} -eq 1 ]]; then
+        sleep 30
+        nice -n 19 find ${dir} -type f -executable -print0 | xargs --null chmod a+rx
+        sleep 30
+      fi
+      nice -n 19 lfs find ${dir} -type f -print0 | xargs --null chmod a+r
+      ;;
+    nautilus)
+      nice -n 19 lfs find ${dir} -type d -print0 | xargs --null chmod a+rx
+      # In case the find command returns no executables
+      if [[ ${executables} -eq 1 ]]; then
+        sleep 30
+        nice -n 19 find ${dir} -type f -executable -print0 | xargs --null chmod a+rx
+        sleep 30
+      fi
+      nice -n 19 lfs find ${dir} -type f -print0 | xargs --null chmod a+r
+      ;;
+    # DH*
+    blackpearl)
+      ;;
+    bounty)
+      ;;
+    # *DH
+    *)
+      echo "ERROR, xargs-chmod command not configured for ${host}"
+      exit 1
+      ;;
+  esac
+  set -e
+}
+
+##################################################################################################
+
 echo "\n"
 echo "Welcome to NRL SPACK-STACK BATCH INSTALL"
 echo
@@ -64,6 +131,7 @@ fi
 
 host=${SPACK_STACK_BATCH_HOST}
 module_choice=${SPACK_STACK_MODULE_CHOICE}
+bootstrap_mirror_path=${SPACK_STACK_BOOTSTRAP_MIRROR}
 
 # For Cray systems, capture the default=current environment (loaded modules)
 # so that it can be restored between building stacks for different compilers
@@ -102,6 +170,7 @@ case ${SPACK_STACK_BATCH_CREATE_BUILDCACHE} in
     ;;
 esac
 
+
 # Loop through all compilers and templates for this host
 for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do
 
@@ -263,6 +332,34 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do
       cp -av configs/sites/tier1/narwhal/compilers.gcc-direct.tmp ${env_dir}/site/compilers.yaml
     fi
 
+    # Bootstrapping/bootstrap mirrors. In developer mode, create bootstrap
+    # mirror locally, then synchronize with shared bootstrap mirror. In
+    # installer mode, configure bootstrap mirror.
+    if [[ "${create_buildcache}" == "true"* ]]; then
+      tmp_bootstrap_mirror_path=${PWD}/tmp-bootstrap-mirror-${env_name}
+      echo "Creating bootstrap mirror ${tmp_bootstrap_mirror_path} ..."
+      rm -fr ${tmp_bootstrap_mirror_path}
+      if [[ -d ${tmp_bootstrap_mirror_path} ]]; then
+        echo "ERROR, directory ${tmp_bootstrap_mirror_path} already exists"
+        exit 1
+      fi
+      spack bootstrap mirror --binary-packages ${tmp_bootstrap_mirror_path} 2>&1 | tee log.bootstrap-mirror.${env_name}.001
+      rsync -av ${tmp_bootstrap_mirror_path}/ ${bootstrap_mirror_path}/
+      rm -fr ${tmp_bootstrap_mirror_path}
+      # Update buildcache index
+      spack buildcache update-index ${bootstrap_mirror_path}/bootstrap_cache
+    fi
+
+    if [[ "${create_buildcache}" == "false" || "${create_buildcache}" == "true-reuse" ]]; then
+      echo "Registering bootstrap mirror ${bootstrap_mirror_path} ..."
+      if [[ ! -d ${bootstrap_mirror_path} ]]; then
+        echo "ERROR, directory ${bootstrap_mirror_path} not found"
+        exit 1
+      fi
+      spack bootstrap add --trust local-sources ${bootstrap_mirror_path}/metadata/sources
+      spack bootstrap add --trust local-binaries ${bootstrap_mirror_path}/metadata/binaries
+    fi
+
     # Check that the site has mirrors configured for local source and binary caches,
     # and extract the local path on disk. Need to strip leading "file://" from path
     result=$(spack mirror list | grep local-source) || \
@@ -280,6 +377,10 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do
       spack config add config:install_tree:padded_length:200
     fi
 
+    # Bootstrap spack explicitly
+    echo "Bootstrapping spack ..."
+    spack bootstrap now 2>&1 | tee log.bootstrap.${env_name}.001
+
     # Concretize environment, and check that spack.lock is created
     spack concretize --force --fresh 2>&1 | tee log.concretize.${env_name}.001
     if [[ ! -e ${env_dir}/spack.lock ]]; then
@@ -296,11 +397,6 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do
       spack mirror create -a -d ${source_mirror_path}
     fi
 
-    # Update the buildcache index if it already contains packages
-    if [[ -e ${binary_mirror_path}/build_cache ]]; then
-      spack buildcache update-index local-binary
-    fi
-
     # Install the environment with the correct flags
     case ${create_buildcache} in
       "false")
@@ -358,6 +454,19 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do
       esac
     fi
 
+    # In installer mode, create a log file with a list of all installed packages
+    if [[ "${create_buildcache}" == "false" ]]; then
+      spack find 2>&1 | tee log.installed_packages.${env_name}.001
+    fi
+
+    # When creating or updating buildcaches, fix permissions for mirrors.
+    # Mirrors do not contain executables, therefore skip looking for them.
+    if [[ "${create_buildcache}" == "true"* ]]; then
+      fix_permissions ${host} ${bootstrap_mirror_path} 0
+      fix_permissions ${host} ${binary_mirror_path} 0
+      fix_permissions ${host} ${source_mirror_path} 0
+    fi
+
     # Clean up
     spack clean -a
     spack env deactivate
@@ -369,35 +478,11 @@ done
 # Remove any module snapshots
 rm -vf ${module_snapshot}
 
-# Note. Add in the xargs stuff
-# Repair permissions for environments
-case ${host} in
-  atlantis)
-    find ./ -type d -print0 | xargs --null chmod a+rx
-    find ./ -type f -executable -print0 | xargs --null chmod a+rx
-    find ./ -type f -print0 | xargs --null chmod a+r
-    ;;
-  narwhal)
-    nice -n 19 lfs find ./ -type d -print0 | xargs --null chmod a+rx
-    nice -n 19 find ./ -type f -executable -print0 | xargs --null chmod a+rx
-    nice -n 19 lfs find ./ -type f -print0 | xargs --null chmod a+r
-    ;;
-  nautilus)
-    nice -n 19 lfs find ./ -type d -print0 | xargs --null chmod a+rx
-    nice -n 19 find ./ -type f -executable -print0 | xargs --null chmod a+rx
-    nice -n 19 lfs find ./ -type f -print0 | xargs --null chmod a+r
-    ;;
-  # DH*
-  blackpearl)
-    ;;
-  bounty)
-    ;;
-  # *DH
-  *)
-    echo "ERROR, xargs-chmod command not configured for ${host}"
-    exit 1
-    ;;
-esac
+# Repair permissions for environments if in installer mode
+if [[ "${create_buildcache}" == "false" ]]; then
+  # Also search for exectuables
+  fix_permissions ${host} ${PWD} 1
+fi
 
 echo "NRL SPACK-STACK BATCH INSTALL SUCCESSFUL"