Jenkinsfile-mpi

pipeline {

agent none
options {
    // Required to clean before build
    skipDefaultCheckout( true )
}

// cron syntax: minute hour day-of-month month day-of-week
// run hourly
triggers { pollSCM 'H * * * *' }

stages {
    //======================================================================
    stage('Parallel Build') {
        matrix {
            axes {
                axis {
                    name 'maker'
                    values 'make'
                }
                axis {
                    name 'host'
                    values 'gpu_nvidia'
                }
            } // axes
            stages {
                stage('Build') {
                    agent { label "${host}" }

                    //----------------------------------------------------------
                    steps {
                        cleanWs()
                        checkout scm
                        sh '''
#!/bin/sh

set +e  # errors are not fatal (e.g., Spack sometimes has spurious failures)
set -x  # echo commands

date
hostname && pwd
export top=`pwd`

date
git submodule update --init

# Suppress echo (-x) output of commands executed with `run`. Useful for Spack.
# set +x, set -x are not echo'd.
run() {
    { set +x; } 2> /dev/null;
    $@;
    set -x
}

# Suppress echo (-x) output of `print` commands. https://superuser.com/a/1141026
# aliasing `echo` causes issues with spack_setup, so use `print` instead.
echo_and_restore() {
    builtin echo "$*"
    case "$save_flags" in
        (*x*)  set -x
    esac
}
alias print='{ save_flags="$-"; set +x; } 2> /dev/null; echo_and_restore'

date
module load python/3.9
module load gcc/7.3.0
module load intel-oneapi-mkl/2022

# hipcc needs /usr/sbin/lsmod
export PATH=${PATH}:/usr/sbin

print "========================================"
date
print "maker ${maker}"

# For simplicity, create make.inc regardless of ${maker}
export color=no
cat > make.inc << END
CXX    = mpicxx
FC     = mpif90
blas   = mkl
prefix = ${top}/install
END

print "========================================"
# Run CUDA, OpenMPI tests.
if [ "${host}" = "gpu_nvidia" ]; then
    module load openmpi/4
    export OMPI_CXX=${CXX}

    echo "CXXFLAGS  = -Werror" >> make.inc
    echo "CXXFLAGS += -Dslate_omp_default_none='default(none)'" >> make.inc
    echo "mkl_blacs = openmpi" >> make.inc
    echo "cuda_arch = sm_35"   >> make.inc  # kepler sm_35 works in CUDA 11
    echo "gpu_backend = cuda"  >> make.inc

    # Load CUDA. LD_LIBRARY_PATH set by Spack.
    module load cuda/11
    export CPATH=${CPATH}:${CUDA_HOME}/include
    export LIBRARY_PATH=${LIBRARY_PATH}:${CUDA_HOME}/lib64
fi

# Run HIP, Intel MPI tests.
if [ "${host}" = "dopamine" ]; then
    module load intel-mpi
    export FI_PROVIDER=tcp

    #echo "CXXFLAGS  = -Werror"  >> make.inc  # HIP headers have many errors; ignore.
    echo "mkl_blacs = intelmpi" >> make.inc
    echo "gpu_backend = hip"    >> make.inc

    # Load ROCm/HIP.
    export PATH=${PATH}:/opt/rocm/bin
    export CPATH=${CPATH}:/opt/rocm/include
    export LIBRARY_PATH=${LIBRARY_PATH}:/opt/rocm/lib:/opt/rocm/lib64
    export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/rocm/lib:/opt/rocm/lib64
    export ROCBLAS_TENSILE_LIBPATH=/opt/rocm/lib/rocblas/library/

    # HIP headers have many errors; reduce noise.
    perl -pi -e 's/-pedantic//' GNUmakefile

    # See the files that are opened by the executable.
    # Used for debugging the ROCm library.
    # extra_test_args=( -t \\\" strace -e trace=open ./tester \\\" )
fi

if [ "${maker}" = "make" ]; then
    print "========================================"
    make echo
fi

if [ "${maker}" = "cmake" ]; then
    print "========================================"
    module load cmake/3.18
    rm -rf build && mkdir build && cd build
    cmake -Dcolor=no -DCMAKE_CXX_FLAGS="-Werror" \
          -DCMAKE_INSTALL_PREFIX=${top}/install \
          ..
fi

print "========================================"
# Check what is loaded.
module list

which mpicxx
which mpif90
mpicxx --version
mpif90 --version

which nvcc
nvcc --version

which hipcc
hipcc --version

echo "MKLROOT ${MKLROOT}"

print "========================================"
env

print "========================================"
date
make -j8

print "========================================"
date
make -j8 install
ls -R ${top}/install

print "========================================"
ldd test/tester

print "========================================"
date
cd test
if [ "${maker}" = "cmake" ]; then
    # only sanity check with cmake build
    export tests=potrf
fi
export OMP_NUM_THREADS=4

ex="gbmm, gemmA, trsmA, gels, cholqr, gelqf, unmtr_he2hb, unmtr_he2hb, he2hb, hegv, gbnorm"
ex2=${ex//,};
eval ./run_tests.py --np 4 --origin s --target t,d --quick --ref n \
    "${extra_test_args[@]}" \
    --timeout 1200 \
    --xml ${top}/report-${maker}.xml ${tests} \
    `echo $ex2|xargs printf -- ' -x %s'` \
    2> ${top}/report-${maker}.txt
cd ..

date
'''
                    } // steps

                    //----------------------------------------------------------
                    post {
                        failure {
                            mail to: 'slate-test@icl.utk.edu',
                                subject: "${currentBuild.fullDisplayName} >> ${STAGE_NAME} >> ${host} failed",
                                body: "See more at ${env.BUILD_URL}"
                        }
                        always {
                            junit '*.xml'
                        }
                    } // post

                } // stage(Build)
            } // stages
        } // matrix
    } // stage(Parallel Build)
} // stages

} // pipeline