From 06cdf991472de25444999c52a97070b49296a7ba Mon Sep 17 00:00:00 2001 From: Ross Ceballo Date: Sat, 18 Jan 2025 22:06:53 -0500 Subject: [PATCH] Prep for prod release --- etc/scipipe/build_matrix.yaml | 2 +- pipelines/lib/util.groovy | 14 +- pipelines/release/docker/build_stack.groovy | 44 +++--- pipelines/release/nightly_release.groovy | 141 ++++++++++---------- pipelines/release/run_publish.groovy | 3 +- pipelines/release/run_rebuild.groovy | 5 +- 6 files changed, 102 insertions(+), 107 deletions(-) diff --git a/etc/scipipe/build_matrix.yaml b/etc/scipipe/build_matrix.yaml index 366826bf..f5f2bd13 100644 --- a/etc/scipipe/build_matrix.yaml +++ b/etc/scipipe/build_matrix.yaml @@ -173,5 +173,5 @@ newinstall: github_repo: lsst/lsst git_ref: tickets/DM-46554 eups: - base_url: https://eups.lsst.codes/stack_test + base_url: https://eups.lsst.codes/stack s3_bucket: eups.lsst.codes diff --git a/pipelines/lib/util.groovy b/pipelines/lib/util.groovy index 046825d7..f24fc5f5 100644 --- a/pipelines/lib/util.groovy +++ b/pipelines/lib/util.groovy @@ -687,18 +687,17 @@ def void createDirs(List eds) { */ def void getManifest(String rebuildId, String filename) { def buildJob = 'release/run-rebuild' - - step([$class: 'CopyArtifact', + def manifest_artifact = 'lsstsw/build/manifest.txt' + step([$class: 'CopyArtifact', projectName: buildJob, - filter: "**/lsstsw/build/manifest.txt", - target: 'buildmanifest', + filter: manifest_artifact, selector: [ $class: 'SpecificBuildSelector', buildNumber: rebuildId // wants a string ], ]) - def manifest = readFile('buildmanifest/lsstsw/build/manifest.txt') - writeFile(file: filename, text: manifest) + def manifest = readFile manifest_artifact + writeFile(file: filename, text: manifest) } // getManifest /** @@ -1385,7 +1384,7 @@ def String runRebuild(Map p) { wait: true, ) nodeTiny { - manifestArtifact = 'lsstsw/build/manifest.txt' + manifestArtifact = 'lsstsw/build/manifest.txt' step([$class: 'CopyArtifact', projectName: useP.job, @@ -1397,7 +1396,6 @@ def String runRebuild(Map p) { ]) def manifestId = parseManifestId(readFile(manifestArtifact)) - // echo sh(returnStdout: true, script: 'env|sort') echo "parsed manifest id: ${manifestId}" return manifestId } // nodeTiny diff --git a/pipelines/release/docker/build_stack.groovy b/pipelines/release/docker/build_stack.groovy index b2b139e6..10c65892 100644 --- a/pipelines/release/docker/build_stack.groovy +++ b/pipelines/release/docker/build_stack.groovy @@ -153,27 +153,27 @@ notify.wrap { run() } } finally { - stage('archive') { - def resultsFile = 'results.json' - - util.dumpJson(resultsFile, [ - base_image: baseImage ?: null, - image: "${dockerRepo}:${dockerTag}", - docker_registry: [ - repo: dockerRepo, - tag: dockerTag - ], - ]) - - archiveArtifacts([ - artifacts: resultsFile, - fingerprint: true - ]) - } // stage - } // try - } // util.nodeWrap - } -} -parallel matrix + stage('archive') { + def resultsFile = 'results.json' + + util.dumpJson(resultsFile, [ + base_image: baseImage ?: null, + image: "${dockerRepo}:${dockerTag}", + docker_registry: [ + repo: dockerRepo, + tag: dockerTag + ], + ]) + + archiveArtifacts([ + artifacts: resultsFile, + fingerprint: true + ]) + } // stage + } // try + } // util.nodeWrap + } + } + parallel matrix } // notify.wrap diff --git a/pipelines/release/nightly_release.groovy b/pipelines/release/nightly_release.groovy index 62b2862c..1e7e3612 100644 --- a/pipelines/release/nightly_release.groovy +++ b/pipelines/release/nightly_release.groovy @@ -35,8 +35,7 @@ notify.wrap { def run = { stage('format nightly tag') { - //gitTag = "d.${year}.${month}.${day}" - gitTag = "test.${year}.${month}" + gitTag = "d.${year}.${month}.${day}" eupsTag = util.sanitizeEupsTag(gitTag) echo "generated [git] tag: ${gitTag}" echo "generated [eups] tag: ${eupsTag}" @@ -47,7 +46,7 @@ notify.wrap { manifestId = util.runRebuild( parameters: [ PRODUCTS: products, - BUILD_DOCS: false, + BUILD_DOCS: true, ], ) } // retry @@ -57,7 +56,7 @@ notify.wrap { def pub = [:] [eupsTag, - //'d_latest' + 'd_latest' ].each { tagName -> pub[tagName] = { retry(retries) { @@ -138,7 +137,7 @@ notify.wrap { EUPS_TAG: eupsTag, DOCKER_TAGS: extraDockerTags, MANIFEST_ID: manifestId, - LSST_COMPILER: lsstswConfig.compiler[0], + LSST_COMPILER: lsstswConfig.compiler, ], ) } // retry @@ -146,72 +145,72 @@ notify.wrap { def triggerMe = [:] - // triggerMe['build Science Platform Notebook Aspect Lab image'] = { - // retry(retries) { - // // based on lsstsqre/stack image - // build( - // job: 'sqre/infra/build-sciplatlab', - // parameters: [ - // string(name: 'TAG', value: eupsTag), - // ], - // wait: false, - // ) - // } // retry - // } - - // triggerMe['verify_drp_metrics'] = { - // retry(1) { - // // based on lsstsqre/stack image - // build( - // job: 'sqre/verify_drp_metrics', - // parameters: [ - // string(name: 'DOCKER_IMAGE', value: stackResults.image), - // booleanParam( - // name: 'NO_PUSH', - // value: scipipe.release.step.verify_drp_metrics.no_push, - // ), - // booleanParam(name: 'WIPEOUT', value: false), - // string(name: 'GIT_REF', value: 'main'), - // ], - // wait: false, - // ) - // } // retry - // } - - // triggerMe['doc build'] = { - // retry(retries) { - // build( - // job: 'sqre/infra/documenteer', - // parameters: [ - // string(name: 'EUPS_TAG', value: eupsTag), - // string(name: 'LTD_SLUG', value: eupsTag), - // string(name: 'RELEASE_IMAGE', value: stackResults.image), - // booleanParam( - // name: 'PUBLISH', - // value: scipipe.release.step.documenteer.publish, - // ), - // ], - // wait: false, - // ) - // } // retry - // } - - // triggerMe['ap_verify'] = { - // retry(retries) { - // build( - // job: 'scipipe/ap_verify', - // parameters: [ - // string(name: 'DOCKER_IMAGE', value: stackResults.image), - // booleanParam( - // name: 'NO_PUSH', - // value: scipipe.release.step.ap_verify.no_push, - // ), - // booleanParam(name: 'WIPEOUT', value: false), - // ], - // wait: false, - // ) - // } // retry - // } + triggerMe['build Science Platform Notebook Aspect Lab image'] = { + retry(retries) { + // based on lsstsqre/stack image + build( + job: 'sqre/infra/build-sciplatlab', + parameters: [ + string(name: 'TAG', value: eupsTag), + ], + wait: false, + ) + } // retry + } + + triggerMe['verify_drp_metrics'] = { + retry(1) { + // based on lsstsqre/stack image + build( + job: 'sqre/verify_drp_metrics', + parameters: [ + string(name: 'DOCKER_IMAGE', value: stackResults.image), + booleanParam( + name: 'NO_PUSH', + value: scipipe.release.step.verify_drp_metrics.no_push, + ), + booleanParam(name: 'WIPEOUT', value: false), + string(name: 'GIT_REF', value: 'main'), + ], + wait: false, + ) + } // retry + } + + triggerMe['doc build'] = { + retry(retries) { + build( + job: 'sqre/infra/documenteer', + parameters: [ + string(name: 'EUPS_TAG', value: eupsTag), + string(name: 'LTD_SLUG', value: eupsTag), + string(name: 'RELEASE_IMAGE', value: stackResults.image), + booleanParam( + name: 'PUBLISH', + value: scipipe.release.step.documenteer.publish, + ), + ], + wait: false, + ) + } // retry + } + + triggerMe['ap_verify'] = { + retry(retries) { + build( + job: 'scipipe/ap_verify', + parameters: [ + string(name: 'DOCKER_IMAGE', value: stackResults.image), + booleanParam( + name: 'NO_PUSH', + value: scipipe.release.step.ap_verify.no_push, + ), + booleanParam(name: 'WIPEOUT', value: false), + ], + wait: false, + ) + } // retry + } stage('triggered jobs') { parallel triggerMe diff --git a/pipelines/release/run_publish.groovy b/pipelines/release/run_publish.groovy index 93173204..13943d40 100644 --- a/pipelines/release/run_publish.groovy +++ b/pipelines/release/run_publish.groovy @@ -45,8 +45,7 @@ notify.wrap { } def slug = util.lsstswConfigSlug(lsstswConfig) def run = { - def workingDir = canonical.workspace - ws(workingDir) { + ws(canonical.workspace) { def cwd = pwd() def pkgroot = "${cwd}/distrib" diff --git a/pipelines/release/run_rebuild.groovy b/pipelines/release/run_rebuild.groovy index 6dd75ab2..9503e545 100644 --- a/pipelines/release/run_rebuild.groovy +++ b/pipelines/release/run_rebuild.groovy @@ -49,8 +49,7 @@ notify.wrap { def slug = util.lsstswConfigSlug(lsstswConfig) def run = { - def workingDir = canonical.workspace - ws(workingDir) { + ws(canonical.workspace) { def cwd = pwd() splenvRef = lsstswConfig.splenv_ref if (params.SPLENV_REF) { @@ -69,7 +68,7 @@ notify.wrap { LSST_PYTHON_VERSION: lsstswConfig.python, LSST_SPLENV_REF: splenvRef, LSST_REFS: refs, - VERSIONDB_PUSH: versiondbPush, + VERSIONDB_PUSH: versiondbPush, VERSIONDB_REPO: versiondbRepo, ]