From 9916f441644462bbd6b6903325526c2acd4b5dae Mon Sep 17 00:00:00 2001 From: Bruce Kropp Date: Fri, 8 Nov 2024 12:07:46 -0600 Subject: [PATCH 1/3] template files for cicd pipeline builds --- .cicd/Jenkinsfile.combined | 388 ++++++++++++++++++++++++++++++++++++ .cicd/scripts/disk_usage.sh | 48 +++++ .cicd/scripts/wm_build.sh | 24 +++ .cicd/scripts/wm_init.sh | 18 ++ .cicd/scripts/wm_test.sh | 19 ++ 5 files changed, 497 insertions(+) create mode 100644 .cicd/Jenkinsfile.combined create mode 100755 .cicd/scripts/disk_usage.sh create mode 100755 .cicd/scripts/wm_build.sh create mode 100755 .cicd/scripts/wm_init.sh create mode 100755 .cicd/scripts/wm_test.sh diff --git a/.cicd/Jenkinsfile.combined b/.cicd/Jenkinsfile.combined new file mode 100644 index 0000000000..2f62ecfe24 --- /dev/null +++ b/.cicd/Jenkinsfile.combined @@ -0,0 +1,388 @@ +matchedNode = [] +generateBaselineNode = [] +for (label in pullRequest.labels) { + listOfLabelNodeNames = jenkins.model.Jenkins.instance.nodes.collect { + node -> node.getLabelString().contains(label) ? node.name : null + + if ((label.matches(node.getLabelString()+"-(.*)"))) { + matchedNode += node.getLabelString() + } + + if ((label.matches(node.getLabelString()+"(.*)-BL"))) { + generateBaselineNode += node.getLabelString() + } + } +} + +modifiedLabels = matchedNode.collect{"'" + it + "'"} +baselineLabels = generateBaselineNode.collect{"'" + it + "'"} +def generateStage(nodeLabel) { + return { + stage("Running on ${nodeLabel}") { + node(nodeLabel) { + cleanWs() + checkout scm + script { + try { + echo "Running on ${nodeLabel}" + if (baselineLabels.contains(nodeLabel)) { + sh ''' + git submodule update --init --recursive + cd tests + pwd + export BL_DATE=$(cat bl_date.conf | cut -d '=' -f2) + export machine=${NODE_NAME} + export PATH=$PATH:~/bin + echo $CHANGE_ID + export SSH_ORIGIN=$(curl --silent https://api.github.com/repos/ufs-community/ufs-weather-model/pulls/$CHANGE_ID | jq -r '.head.repo.ssh_url') + export FORK_BRANCH=$(curl --silent https://api.github.com/repos/ufs-community/ufs-weather-model/pulls/$CHANGE_ID | jq -r '.head.ref') + + if [[ $machine =~ "Jet" ]] + then + echo "Creating baselines on $machine" + export dprefix=/lfs1/NAGAPE/$ACCNR/$USER + ./rt.sh -a ${ACCNR} -c -r -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + elif [[ $machine =~ "Hercules" ]] + then + echo "Creating baselines on $machine" + export dprefix=/work2/noaa/$ACCNR/$USER + sed "s|/noaa/stmp/|/noaa/$ACCNR/stmp/|g" -i rt.sh + export ACCNR=epic + ./rt.sh -a ${ACCNR} -c -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + export DISKNM=/work/noaa/epic/hercules/UFS-WM_RT + cd ${DISKNM}/NEMSfv3gfs/ + mkdir develop-${BL_DATE} + cd /work2/noaa/epic/stmp/role-epic/stmp/role-epic/FV3_RT + rsync -a REGRESSION_TEST/ ${DISKNM}/NEMSfv3gfs/develop-${BL_DATE} + cd ${DISKNM}/NEMSfv3gfs/ + ./adjust_permissions.sh hercules develop-${BL_DATE} + chgrp noaa-hpc develop-${BL_DATE} + cd $WORKSPACE/tests + ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_hercules.log /work/noaa/epic/role-epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_hercules.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + elif [[ $machine =~ "Orion" ]] + then + cd .. + module load git/2.28.0 + git submodule update --init --recursive + cd tests + echo "Creating baselines on $machine" + export dprefix=/work2/noaa/$ACCNR/$USER + sed -i 's|/work/noaa/stmp/${USER}|/work/noaa/epic/stmp/role-epic/|g' rt.sh + export ACCNR=epic + ./rt.sh -a ${ACCNR} -c -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + export DISKNM=/work/noaa/epic/UFS-WM_RT + cd ${DISKNM}/NEMSfv3gfs/ + mkdir develop-${BL_DATE} + cd /work/noaa/epic/stmp/role-epic/stmp/role-epic/FV3_RT/ + rsync -a REGRESSION_TEST/ ${DISKNM}/NEMSfv3gfs/develop-${BL_DATE} + cd ${DISKNM}/NEMSfv3gfs/ + ./adjust_permissions.sh orion develop-${BL_DATE} + chgrp noaa-hpc develop-${BL_DATE} + cd $WORKSPACE/tests + ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_orion.log /work/noaa/epic/role-epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_orion.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + elif [[ $machine =~ "Gaea" ]] + then + echo "Creating baselines on $machine" + ./rt.sh -a ${ACCNR} -c -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + unset LD_LIBRARY_PATH + export DISKNM=/gpfs/f5/epic/world-shared/UFS-WM_RT + cd ${DISKNM}/NEMSfv3gfs/ + mkdir develop-${BL_DATE} + cd /gpfs/f5/epic/scratch/role.epic/FV3_RT + rsync -a REGRESSION_TEST/ ${DISKNM}/NEMSfv3gfs/develop-${BL_DATE} + cd ${DISKNM}/NEMSfv3gfs/ + chgrp ncep develop-${BL_DATE} + cd $WORKSPACE/tests + ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_gaea.log /gpfs/f5/epic/scratch/role.epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_gaea.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + elif [[ $machine =~ "Hera" ]] + then + echo "Creating baselines on $machine" + export ACCNR=epic + ./rt.sh -a ${ACCNR} -c -r -l rt.conf + export DISKNM=/scratch2/NAGAPE/epic/UFS-WM_RT + cd ${DISKNM}/NEMSfv3gfs/ + mkdir develop-${BL_DATE} + cd /scratch1/NCEPDEV/stmp4/role.epic/FV3_RT + rsync -a REGRESSION_TEST/ ${DISKNM}/NEMSfv3gfs/develop-${BL_DATE} + cd $WORKSPACE/tests + ./rt.sh -a ${ACCNR} -r -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_hera.log /scratch2/NAGAPE/epic/role.epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_hera.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + elif [[ $machine =~ "Derecho" ]] + then + echo "Creating baselines on $machine" + export ACCNR=nral0032 + ./rt.sh -a ${ACCNR} -c -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + export DISKNM=/glade/derecho/scratch/epicufsrt/ufs-weather-model/RT/ + cd ${DISKNM}/NEMSfv3gfs/ + mkdir develop-${BL_DATE} + cd /glade/derecho/scratch/epicufsrt/FV3_RT + rsync -a REGRESSION_TEST/ ${DISKNM}/NEMSfv3gfs/develop-${BL_DATE} + cd $WORKSPACE/tests + ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_derecho.log /glade/derecho/scratch/epicufsrt/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_derecho.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + else + echo "Creating baselines on $machine" + ./rt.sh -a ${ACCNR} -c -r -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + fi + git config user.email "ecc.platform@noaa.gov" + git config user.name "epic-cicd-jenkins" + echo "Testing concluded...removing labels for $machine from $GIT_URL" + + export machine_name_logs=$(echo $machine | awk '{ print tolower($1) }') + git remote -v | grep -w sshorigin > /dev/null 2>&1 && git remote remove sshorigin > /dev/null 2>&1 + git remote add sshorigin $SSH_ORIGIN > /dev/null 2>&1 + git add logs/RegressionTests_$machine_name_logs.log + git commit -m "[AutoRT] $machine Job Completed.\n\n\n on-behalf-of @ufs-community " + git pull sshorigin $FORK_BRANCH + git push sshorigin HEAD:$FORK_BRANCH + + tar --create --gzip --verbose --dereference --file "${machine_name_logs}.tgz" ${WORKSPACE}/tests/logs/*.log + + GIT_OWNER=$(echo $GIT_URL | cut -d '/' -f4) + GIT_REPO_NAME=$(echo $GIT_URL | cut -d '/' -f5 | cut -d '.' -f1) + + curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels/$machine-BL + ''' + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: true, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "**/*tgz*", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + + } + else { + sh ''' + git submodule update --init --recursive + pwd + cd tests + export machine=${NODE_NAME} + export PATH=$PATH:~/bin + echo $CHANGE_ID + export SSH_ORIGIN=$(curl --silent https://api.github.com/repos/ufs-community/ufs-weather-model/pulls/$CHANGE_ID | jq -r '.head.repo.ssh_url') + export FORK_BRANCH=$(curl --silent https://api.github.com/repos/ufs-community/ufs-weather-model/pulls/$CHANGE_ID | jq -r '.head.ref') + + if [[ $machine =~ "Jet" ]] + then + echo "Running regression tests on $machine" + export dprefix=/lfs1/NAGAPE/$ACCNR/$USER + ./rt.sh -a ${ACCNR} -r -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + elif [[ $machine =~ "Hercules" ]] + then + echo "Running regression tests on $machine" + export dprefix=/work2/noaa/$ACCNR/$USER + sed "s|/noaa/stmp/|/noaa/$ACCNR/stmp/|g" -i rt.sh + export ACCNR=epic + ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_hercules.log /work/noaa/epic/role-epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_hercules.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + elif [[ $machine =~ "Orion" ]] + then + echo "Running regression tests on $machine" + cd .. + module load git/2.28.0 + git submodule update --init --recursive + cd tests + export dprefix=/work2/noaa/$ACCNR/$USER + sed "s|/noaa/stmp/|/noaa/$ACCNR/stmp/|g" -i rt.sh + ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_orion.log /work/noaa/epic/role-epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_orion.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + elif [[ $machine =~ "Gaea" ]] + then + echo "Running regression tests on $machine" + ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + unset LD_LIBRARY_PATH + cd logs/ + cp RegressionTests_gaea.log /gpfs/f5/epic/scratch/role.epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_gaea.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + elif [[ $machine =~ "Hera" ]] + then + echo "Running regression tests on $machine" + export ACCNR=epic + ./rt.sh -a ${ACCNR} -r -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_hera.log /scratch2/NAGAPE/epic/role.epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_hera.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + elif [[ $machine =~ "Derecho" ]] + then + echo "Running regression tests on $machine" + export ACCNR=nral0032 + ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_derecho.log /glade/derecho/scratch/epicufsrt/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_derecho.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + else + echo "Running regression tests on $machine" + ./rt.sh -a ${ACCNR} -r -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + fi + + git config user.email "ecc.platform@noaa.gov" + git config user.name "epic-cicd-jenkins" + export machine_name_logs=$(echo $machine | awk '{ print tolower($1) }') + echo "Testing concluded...removing labels for $machine from $GIT_URL" + git remote -v | grep -w sshorigin > /dev/null 2>&1 && git remote remove sshorigin > /dev/null 2>&1 + git remote add sshorigin $SSH_ORIGIN > /dev/null 2>&1 + git add logs/RegressionTests_$machine_name_logs.log + git commit -m "[AutoRT] $machine Job Completed.\n\n\n on-behalf-of @ufs-community " + git pull sshorigin $FORK_BRANCH + git push sshorigin HEAD:$FORK_BRANCH + + tar --create --gzip --verbose --dereference --file "${machine_name_logs}.tgz" ${WORKSPACE}/tests/logs/*.log + + GIT_OWNER=$(echo $GIT_URL | cut -d '/' -f4) + GIT_REPO_NAME=$(echo $GIT_URL | cut -d '/' -f5 | cut -d '.' -f1) + + curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels/$machine-RT + + ''' + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: true, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "**/*tgz*", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + } + } + catch(err) { + sh ''' + export machine=${NODE_NAME} + export CHANGE_ID=${CHANGE_ID} + export SSH_ORIGIN=$(curl --silent https://api.github.com/repos/ufs-community/ufs-weather-model/pulls/$CHANGE_ID | jq -r '.head.repo.ssh_url') + export FORK_BRANCH=$(curl --silent https://api.github.com/repos/ufs-community/ufs-weather-model/pulls/$CHANGE_ID | jq -r '.head.ref') + cd $WORKSPACE/tests + git config user.email "ecc.platform@noaa.gov" + git config user.name "epic-cicd-jenkins" + export machine_name_logs=$(echo $machine | awk '{ print tolower($1) }') + echo "Testing concluded...removing labels for $machine from $GIT_URL" + git remote -v | grep -w sshorigin > /dev/null 2>&1 && git remote remove sshorigin > /dev/null 2>&1 + git remote add sshorigin $SSH_ORIGIN > /dev/null 2>&1 + + tar --create --gzip --verbose --dereference --file "${machine_name_logs}.tgz" ${WORKSPACE}/tests/logs/*.log + + GIT_OWNER=$(echo $GIT_URL | cut -d '/' -f4) + GIT_REPO_NAME=$(echo $GIT_URL | cut -d '/' -f5 | cut -d '.' -f1) + + curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels/{$machine-RT,$machine-BL} + ''' + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: true, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "**/*tgz*", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + currentBuild.result = 'FAILURE' + } + } + } + } + } +} + +def parallelStagesMap = modifiedLabels.collectEntries { + ["${it}" : generateStage(it)] +} + +pipeline { + agent none + environment { + ACCNR = 'epic' + AWS_PROD_ACCOUNT_ID = credentials('AWS_PROD_ACCOUNT_ID') + AWS_PROD_SNS_TOPIC = credentials('AWS_PROD_SNS_TOPIC') + GITHUB_TOKEN = credentials('GithubJenkinsNew') + GIT_URL = 'https://github.com/ufs-community/ufs-weather-model.git' + } + stages { + stage('Launch SonarQube') { + steps { + script { + echo "BRANCH_NAME=${env.CHANGE_BRANCH}" + echo "FORK_NAME=${env.CHANGE_FORK}" + echo "CHANGE_URL=${env.CHANGE_URL}" + echo "CHANGE_ID=${env.CHANGE_ID}" + build job: '/ufs-weather-model/ufs-wm-sonarqube', parameters: [ + string(name: 'BRANCH_NAME', value: env.CHANGE_BRANCH ?: 'develop'), + string(name: 'FORK_NAME', value: env.CHANGE_FORK ?: ''), + string(name: 'CHANGE_URL', value: env.CHANGE_URL ?: ''), + string(name: 'CHANGE_ID', value: env.CHANGE_ID ?: '') + ], wait: false + } + } + } + stage('Run Regression Tests in Parallel') { + steps { + script { + parallel parallelStagesMap + } + } + } + } + post { + success { + node('built-in') { + echo 'This will run only if successful.' + sh ''' + aws sns publish --topic-arn "arn:aws:sns:us-east-1:${AWS_PROD_ACCOUNT_ID}:${AWS_PROD_SNS_TOPIC}" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":sunny: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'* *succeeded*"}}' + ''' + } + } + failure { + node('built-in') { + echo 'This will run only if the run was marked as unstable.' + sh ''' + aws sns publish --topic-arn "arn:aws:sns:us-east-1:${AWS_PROD_ACCOUNT_ID}:${AWS_PROD_SNS_TOPIC}" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":warning: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'* *failed!*"}}' + ''' + } + } + } +} diff --git a/.cicd/scripts/disk_usage.sh b/.cicd/scripts/disk_usage.sh new file mode 100755 index 0000000000..22ffbeaa54 --- /dev/null +++ b/.cicd/scripts/disk_usage.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# Output a CSV report of disk usage on subdirs of some path +# Usage: +# [JOB_NAME=] [BUILD_NUMBER=] [UFS_COMPILER=] [UFS_PLATFORM=] disk_usage path depth size outfile.csv +# +# args: +# directory=$1 +# depth=$2 +# size=$3 +# outfile=$4 + +[[ -n ${WORKSPACE} ]] || WORKSPACE=$(pwd) +[[ -n ${UFS_PLATFORM} ]] || UFS_PLATFORM=$(hostname -s 2>/dev/null) || UFS_PLATFORM=$(hostname 2>/dev/null) +[[ -n ${UFS_COMPILER} ]] || UFS_COMPILER=compiler + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" + +# Get repository root from Jenkins WORKSPACE variable if set, otherwise, set +# relative to script directory. +declare workspace +if [[ -n "${WORKSPACE}/${UFS_PLATFORM}" ]]; then + workspace="${WORKSPACE}/${UFS_PLATFORM}" +else + workspace="$(cd -- "${script_dir}/../.." && pwd)" +fi + +echo "STAGE_NAME=${STAGE_NAME}" # from pipeline +outfile="${4:-${workspace}-${UFS_COMPILER}-disk-usage${STAGE_NAME}.csv}" + +function disk_usage() { + local directory=${1:-${PWD}} + local depth=${2:-1} + local size=${3:-k} + echo "Disk usage: ${JOB_NAME:-ci}/${UFS_PLATFORM}/$(basename $directory)" + ( + cd $directory || exit 1 + echo "Platform,Build,Owner,Group,Inodes,${size:-k}bytes,Access Time,Filename" + du -Px -d ${depth:-1} --inode --exclude='./workspace' | \ + while read line ; do + arr=($line); inode=${arr[0]}; filename=${arr[1]}; + echo "${UFS_PLATFORM}-${UFS_COMPILER:-compiler},${JOB_NAME:-ci}/${BUILD_NUMBER:-0},$(stat -c '%U,%G' $filename),${inode:-0},$(du -Px -s -${size:-k} --time $filename)" | tr '\t' ',' ; + done | sort -t, -k5 -n #-r + ) + echo "" +} + +disk_usage $1 $2 $3 | tee ${outfile} diff --git a/.cicd/scripts/wm_build.sh b/.cicd/scripts/wm_build.sh new file mode 100755 index 0000000000..ee81b04a5d --- /dev/null +++ b/.cicd/scripts/wm_build.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -eu + +SCRIPT_REALPATH=$(realpath "${BASH_SOURCE[0]}") +SCRIPTS_DIR=$(dirname "${SCRIPT_REALPATH}") +UFS_MODEL_DIR=$(realpath "${SCRIPTS_DIR}/../..") +readonly UFS_MODEL_DIR +echo "UFS MODEL DIR: ${UFS_MODEL_DIR}" + +export CC=${CC:-mpicc} +export CXX=${CXX:-mpicxx} +export FC=${FC:-mpif90} + +cd "${UFS_MODEL_DIR}" +pwd +ls -l ./build.sh + +BUILD_DIR=${BUILD_DIR:-${UFS_MODEL_DIR}/build} +TESTS_DIR=${TESTS_DIR:-${UFS_MODEL_DIR}/tests} +mkdir -p "${BUILD_DIR}" + +cd "${BUILD_DIR}" +pwd + diff --git a/.cicd/scripts/wm_init.sh b/.cicd/scripts/wm_init.sh new file mode 100755 index 0000000000..27db702be1 --- /dev/null +++ b/.cicd/scripts/wm_init.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -eu + +SCRIPT_REALPATH=$(realpath "${BASH_SOURCE[0]}") +SCRIPTS_DIR=$(dirname "${SCRIPT_REALPATH}") +UFS_MODEL_DIR=$(realpath "${SCRIPTS_DIR}/../..") +readonly UFS_MODEL_DIR +echo "UFS MODEL DIR: ${UFS_MODEL_DIR}" + +export CC=${CC:-mpicc} +export CXX=${CXX:-mpicxx} +export FC=${FC:-mpif90} + +BUILD_DIR=${BUILD_DIR:-${UFS_MODEL_DIR}/build} +TESTS_DIR=${TESTS_DIR:-${UFS_MODEL_DIR}/tests} + +cd "${UFS_MODEL_DIR}" +pwd diff --git a/.cicd/scripts/wm_test.sh b/.cicd/scripts/wm_test.sh new file mode 100755 index 0000000000..9ac36ac6e1 --- /dev/null +++ b/.cicd/scripts/wm_test.sh @@ -0,0 +1,19 @@ +#!/bin/bash -x +set -eu + +SCRIPT_REALPATH=$(realpath "${BASH_SOURCE[0]}") +SCRIPTS_DIR=$(dirname "${SCRIPT_REALPATH}") +UFS_MODEL_DIR=$(realpath "${SCRIPTS_DIR}/../..") +readonly UFS_MODEL_DIR +echo "UFS MODEL DIR: ${UFS_MODEL_DIR}" + +export CC=${CC:-mpicc} +export CXX=${CXX:-mpicxx} +export FC=${FC:-mpif90} + +BUILD_DIR=${BUILD_DIR:-${UFS_MODEL_DIR}/build} +TESTS_DIR=${TESTS_DIR:-${UFS_MODEL_DIR}/tests} + +cd "${TESTS_DIR}" +pwd +ls -al ./rt.sh From a8c0c621dd1cb7dc97bff275609cfc74fae83543 Mon Sep 17 00:00:00 2001 From: Bruce Kropp Date: Fri, 8 Nov 2024 12:44:52 -0600 Subject: [PATCH 2/3] start with a Jenkinsfile that doesnt push logs to the repo --- .cicd/Jenkinsfile.pipeline | 388 +++++++++++++++++++++++++++++++++++++ 1 file changed, 388 insertions(+) create mode 100644 .cicd/Jenkinsfile.pipeline diff --git a/.cicd/Jenkinsfile.pipeline b/.cicd/Jenkinsfile.pipeline new file mode 100644 index 0000000000..58f1d239b4 --- /dev/null +++ b/.cicd/Jenkinsfile.pipeline @@ -0,0 +1,388 @@ +matchedNode = [] +generateBaselineNode = [] +for (label in pullRequest.labels) { + listOfLabelNodeNames = jenkins.model.Jenkins.instance.nodes.collect { + node -> node.getLabelString().contains(label) ? node.name : null + + if ((label.matches(node.getLabelString()+"-(.*)"))) { + matchedNode += node.getLabelString() + } + + if ((label.matches(node.getLabelString()+"(.*)-BL"))) { + generateBaselineNode += node.getLabelString() + } + } +} + +modifiedLabels = matchedNode.collect{"'" + it + "'"} +baselineLabels = generateBaselineNode.collect{"'" + it + "'"} +def generateStage(nodeLabel) { + return { + stage("Running on ${nodeLabel}") { + node(nodeLabel) { + cleanWs() + checkout scm + script { + try { + echo "Running on ${nodeLabel}" + if (baselineLabels.contains(nodeLabel)) { + sh ''' + git submodule update --init --recursive + cd tests + pwd + export BL_DATE=$(cat bl_date.conf | cut -d '=' -f2) + export machine=${NODE_NAME} + export PATH=$PATH:~/bin + echo $CHANGE_ID + export SSH_ORIGIN=$(curl --silent https://api.github.com/repos/ufs-community/ufs-weather-model/pulls/$CHANGE_ID | jq -r '.head.repo.ssh_url') + export FORK_BRANCH=$(curl --silent https://api.github.com/repos/ufs-community/ufs-weather-model/pulls/$CHANGE_ID | jq -r '.head.ref') + + if [[ $machine =~ "Jet" ]] + then + echo "Creating baselines on $machine" + export dprefix=/lfs1/NAGAPE/$ACCNR/$USER + ./rt.sh -a ${ACCNR} -c -r -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + elif [[ $machine =~ "Hercules" ]] + then + echo "Creating baselines on $machine" + export dprefix=/work2/noaa/$ACCNR/$USER + sed "s|/noaa/stmp/|/noaa/$ACCNR/stmp/|g" -i rt.sh + export ACCNR=epic + ./rt.sh -a ${ACCNR} -c -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + export DISKNM=/work/noaa/epic/hercules/UFS-WM_RT + cd ${DISKNM}/NEMSfv3gfs/ + mkdir develop-${BL_DATE} + cd /work2/noaa/epic/stmp/role-epic/stmp/role-epic/FV3_RT + rsync -a REGRESSION_TEST/ ${DISKNM}/NEMSfv3gfs/develop-${BL_DATE} + cd ${DISKNM}/NEMSfv3gfs/ + ./adjust_permissions.sh hercules develop-${BL_DATE} + chgrp noaa-hpc develop-${BL_DATE} + cd $WORKSPACE/tests + ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_hercules.log /work/noaa/epic/role-epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_hercules.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + elif [[ $machine =~ "Orion" ]] + then + cd .. + module load git/2.28.0 + git submodule update --init --recursive + cd tests + echo "Creating baselines on $machine" + export dprefix=/work2/noaa/$ACCNR/$USER + sed -i 's|/work/noaa/stmp/${USER}|/work/noaa/epic/stmp/role-epic/|g' rt.sh + export ACCNR=epic + ./rt.sh -a ${ACCNR} -c -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + export DISKNM=/work/noaa/epic/UFS-WM_RT + cd ${DISKNM}/NEMSfv3gfs/ + mkdir develop-${BL_DATE} + cd /work/noaa/epic/stmp/role-epic/stmp/role-epic/FV3_RT/ + rsync -a REGRESSION_TEST/ ${DISKNM}/NEMSfv3gfs/develop-${BL_DATE} + cd ${DISKNM}/NEMSfv3gfs/ + ./adjust_permissions.sh orion develop-${BL_DATE} + chgrp noaa-hpc develop-${BL_DATE} + cd $WORKSPACE/tests + ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_orion.log /work/noaa/epic/role-epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_orion.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + elif [[ $machine =~ "Gaea" ]] + then + echo "Creating baselines on $machine" + ./rt.sh -a ${ACCNR} -c -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + unset LD_LIBRARY_PATH + export DISKNM=/gpfs/f5/epic/world-shared/UFS-WM_RT + cd ${DISKNM}/NEMSfv3gfs/ + mkdir develop-${BL_DATE} + cd /gpfs/f5/epic/scratch/role.epic/FV3_RT + rsync -a REGRESSION_TEST/ ${DISKNM}/NEMSfv3gfs/develop-${BL_DATE} + cd ${DISKNM}/NEMSfv3gfs/ + chgrp ncep develop-${BL_DATE} + cd $WORKSPACE/tests + ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_gaea.log /gpfs/f5/epic/scratch/role.epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_gaea.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + elif [[ $machine =~ "Hera" ]] + then + echo "Creating baselines on $machine" + export ACCNR=epic + ./rt.sh -a ${ACCNR} -c -r -l rt.conf + export DISKNM=/scratch2/NAGAPE/epic/UFS-WM_RT + cd ${DISKNM}/NEMSfv3gfs/ + mkdir develop-${BL_DATE} + cd /scratch1/NCEPDEV/stmp4/role.epic/FV3_RT + rsync -a REGRESSION_TEST/ ${DISKNM}/NEMSfv3gfs/develop-${BL_DATE} + cd $WORKSPACE/tests + ./rt.sh -a ${ACCNR} -r -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_hera.log /scratch2/NAGAPE/epic/role.epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_hera.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + elif [[ $machine =~ "Derecho" ]] + then + echo "Creating baselines on $machine" + export ACCNR=nral0032 + ./rt.sh -a ${ACCNR} -c -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + export DISKNM=/glade/derecho/scratch/epicufsrt/ufs-weather-model/RT/ + cd ${DISKNM}/NEMSfv3gfs/ + mkdir develop-${BL_DATE} + cd /glade/derecho/scratch/epicufsrt/FV3_RT + rsync -a REGRESSION_TEST/ ${DISKNM}/NEMSfv3gfs/develop-${BL_DATE} + cd $WORKSPACE/tests + ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_derecho.log /glade/derecho/scratch/epicufsrt/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_derecho.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + else + echo "Creating baselines on $machine" + ./rt.sh -a ${ACCNR} -c -r -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + fi + git config user.email "ecc.platform@noaa.gov" + git config user.name "epic-cicd-jenkins" + echo "Testing concluded...removing labels for $machine from $GIT_URL" + + export machine_name_logs=$(echo $machine | awk '{ print tolower($1) }') + #git remote -v | grep -w sshorigin > /dev/null 2>&1 && git remote remove sshorigin > /dev/null 2>&1 + #git remote add sshorigin $SSH_ORIGIN > /dev/null 2>&1 + #git add logs/RegressionTests_$machine_name_logs.log + #git commit -m "[AutoRT] $machine Job Completed.\n\n\n on-behalf-of @ufs-community " + #git pull sshorigin $FORK_BRANCH + #git push sshorigin HEAD:$FORK_BRANCH + + tar --create --gzip --verbose --dereference --file "${machine_name_logs}.tgz" ${WORKSPACE}/tests/logs/*.log + + GIT_OWNER=$(echo $GIT_URL | cut -d '/' -f4) + GIT_REPO_NAME=$(echo $GIT_URL | cut -d '/' -f5 | cut -d '.' -f1) + + curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels/$machine-BL + ''' + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: true, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "**/*tgz*", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + + } + else { + sh ''' + git submodule update --init --recursive + pwd + cd tests + export machine=${NODE_NAME} + export PATH=$PATH:~/bin + echo $CHANGE_ID + export SSH_ORIGIN=$(curl --silent https://api.github.com/repos/ufs-community/ufs-weather-model/pulls/$CHANGE_ID | jq -r '.head.repo.ssh_url') + export FORK_BRANCH=$(curl --silent https://api.github.com/repos/ufs-community/ufs-weather-model/pulls/$CHANGE_ID | jq -r '.head.ref') + + if [[ $machine =~ "Jet" ]] + then + echo "Running regression tests on $machine" + export dprefix=/lfs1/NAGAPE/$ACCNR/$USER + ./rt.sh -a ${ACCNR} -r -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + elif [[ $machine =~ "Hercules" ]] + then + echo "Running regression tests on $machine" + export dprefix=/work2/noaa/$ACCNR/$USER + sed "s|/noaa/stmp/|/noaa/$ACCNR/stmp/|g" -i rt.sh + export ACCNR=epic + ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_hercules.log /work/noaa/epic/role-epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_hercules.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + elif [[ $machine =~ "Orion" ]] + then + echo "Running regression tests on $machine" + cd .. + module load git/2.28.0 + git submodule update --init --recursive + cd tests + export dprefix=/work2/noaa/$ACCNR/$USER + sed "s|/noaa/stmp/|/noaa/$ACCNR/stmp/|g" -i rt.sh + ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_orion.log /work/noaa/epic/role-epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_orion.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + elif [[ $machine =~ "Gaea" ]] + then + echo "Running regression tests on $machine" + ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + unset LD_LIBRARY_PATH + cd logs/ + cp RegressionTests_gaea.log /gpfs/f5/epic/scratch/role.epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_gaea.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + elif [[ $machine =~ "Hera" ]] + then + echo "Running regression tests on $machine" + export ACCNR=epic + ./rt.sh -a ${ACCNR} -r -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_hera.log /scratch2/NAGAPE/epic/role.epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_hera.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + elif [[ $machine =~ "Derecho" ]] + then + echo "Running regression tests on $machine" + export ACCNR=nral0032 + ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_derecho.log /glade/derecho/scratch/epicufsrt/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_derecho.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + else + echo "Running regression tests on $machine" + ./rt.sh -a ${ACCNR} -r -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + fi + + git config user.email "ecc.platform@noaa.gov" + git config user.name "epic-cicd-jenkins" + export machine_name_logs=$(echo $machine | awk '{ print tolower($1) }') + echo "Testing concluded...removing labels for $machine from $GIT_URL" + #git remote -v | grep -w sshorigin > /dev/null 2>&1 && git remote remove sshorigin > /dev/null 2>&1 + #git remote add sshorigin $SSH_ORIGIN > /dev/null 2>&1 + #git add logs/RegressionTests_$machine_name_logs.log + #git commit -m "[AutoRT] $machine Job Completed.\n\n\n on-behalf-of @ufs-community " + #git pull sshorigin $FORK_BRANCH + #git push sshorigin HEAD:$FORK_BRANCH + + tar --create --gzip --verbose --dereference --file "${machine_name_logs}.tgz" ${WORKSPACE}/tests/logs/*.log + + GIT_OWNER=$(echo $GIT_URL | cut -d '/' -f4) + GIT_REPO_NAME=$(echo $GIT_URL | cut -d '/' -f5 | cut -d '.' -f1) + + curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels/$machine-RT + + ''' + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: true, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "**/*tgz*", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + } + } + catch(err) { + sh ''' + export machine=${NODE_NAME} + export CHANGE_ID=${CHANGE_ID} + export SSH_ORIGIN=$(curl --silent https://api.github.com/repos/ufs-community/ufs-weather-model/pulls/$CHANGE_ID | jq -r '.head.repo.ssh_url') + export FORK_BRANCH=$(curl --silent https://api.github.com/repos/ufs-community/ufs-weather-model/pulls/$CHANGE_ID | jq -r '.head.ref') + cd $WORKSPACE/tests + git config user.email "ecc.platform@noaa.gov" + git config user.name "epic-cicd-jenkins" + export machine_name_logs=$(echo $machine | awk '{ print tolower($1) }') + echo "Testing concluded...removing labels for $machine from $GIT_URL" + #git remote -v | grep -w sshorigin > /dev/null 2>&1 && git remote remove sshorigin > /dev/null 2>&1 + #git remote add sshorigin $SSH_ORIGIN > /dev/null 2>&1 + + tar --create --gzip --verbose --dereference --file "${machine_name_logs}.tgz" ${WORKSPACE}/tests/logs/*.log + + GIT_OWNER=$(echo $GIT_URL | cut -d '/' -f4) + GIT_REPO_NAME=$(echo $GIT_URL | cut -d '/' -f5 | cut -d '.' -f1) + + curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels/{$machine-RT,$machine-BL} + ''' + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: true, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "**/*tgz*", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + currentBuild.result = 'FAILURE' + } + } + } + } + } +} + +def parallelStagesMap = modifiedLabels.collectEntries { + ["${it}" : generateStage(it)] +} + +pipeline { + agent none + environment { + ACCNR = 'epic' + AWS_PROD_ACCOUNT_ID = credentials('AWS_PROD_ACCOUNT_ID') + AWS_PROD_SNS_TOPIC = credentials('AWS_PROD_SNS_TOPIC') + GITHUB_TOKEN = credentials('GithubJenkinsNew') + GIT_URL = 'https://github.com/ufs-community/ufs-weather-model.git' + } + stages { + stage('Launch SonarQube') { + steps { + script { + echo "BRANCH_NAME=${env.CHANGE_BRANCH}" + echo "FORK_NAME=${env.CHANGE_FORK}" + echo "CHANGE_URL=${env.CHANGE_URL}" + echo "CHANGE_ID=${env.CHANGE_ID}" + build job: '/ufs-weather-model/ufs-wm-sonarqube', parameters: [ + string(name: 'BRANCH_NAME', value: env.CHANGE_BRANCH ?: 'develop'), + string(name: 'FORK_NAME', value: env.CHANGE_FORK ?: ''), + string(name: 'CHANGE_URL', value: env.CHANGE_URL ?: ''), + string(name: 'CHANGE_ID', value: env.CHANGE_ID ?: '') + ], wait: false + } + } + } + stage('Run Regression Tests in Parallel') { + steps { + script { + parallel parallelStagesMap + } + } + } + } + post { + success { + node('built-in') { + echo 'This will run only if successful.' + sh ''' + aws sns publish --topic-arn "arn:aws:sns:us-east-1:${AWS_PROD_ACCOUNT_ID}:${AWS_PROD_SNS_TOPIC}" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":sunny: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'* *succeeded*"}}' + ''' + } + } + failure { + node('built-in') { + echo 'This will run only if the run was marked as unstable.' + sh ''' + aws sns publish --topic-arn "arn:aws:sns:us-east-1:${AWS_PROD_ACCOUNT_ID}:${AWS_PROD_SNS_TOPIC}" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":warning: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'* *failed!*"}}' + ''' + } + } + } +} From 8839312fb4e274154e6113fd9dec592c37042708 Mon Sep 17 00:00:00 2001 From: Bruce Kropp - Raytheon <104453151+BruceKropp-Raytheon@users.noreply.github.com> Date: Wed, 13 Nov 2024 10:42:16 -0800 Subject: [PATCH 3/3] Update Jenkinsfile.pipeline list .cicd/ --- .cicd/Jenkinsfile.pipeline | 1 + 1 file changed, 1 insertion(+) diff --git a/.cicd/Jenkinsfile.pipeline b/.cicd/Jenkinsfile.pipeline index 58f1d239b4..57d9bc62fb 100644 --- a/.cicd/Jenkinsfile.pipeline +++ b/.cicd/Jenkinsfile.pipeline @@ -28,6 +28,7 @@ def generateStage(nodeLabel) { if (baselineLabels.contains(nodeLabel)) { sh ''' git submodule update --init --recursive + ls -al .cicd/* cd tests pwd export BL_DATE=$(cat bl_date.conf | cut -d '=' -f2)