From 88aadd2b402a7f40c677725348d00e25134a88fe Mon Sep 17 00:00:00 2001 From: Kian-Tat Lim Date: Fri, 23 Apr 2021 09:01:10 -0700 Subject: [PATCH 1/6] Fix typo (test pipeline). --- pipelines/test_stack_os_matrix.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/test_stack_os_matrix.groovy b/pipelines/test_stack_os_matrix.groovy index 950e6b90..a90badd7 100644 --- a/pipelines/test_stack_os_matrix.groovy +++ b/pipelines/test_stack_os_matrix.groovy @@ -39,7 +39,7 @@ notify.wrap { if (params.RUBINENV_ORG_FORK != null) { buildParams['RUBINENV_ORG_FORK'] = params.RUBINENV_ORG_FORK } - if (params.RUBINENV_BRANCHi != null) { + if (params.RUBINENV_BRANCH != null) { buildParams['RUBINENV_BRANCH'] = params.RUBINENV_BRANCH } From 9e3dddaeaa47a4d427eefbb2ca743539fc3903f3 Mon Sep 17 00:00:00 2001 From: Kian-Tat Lim Date: Fri, 23 Apr 2021 09:02:40 -0700 Subject: [PATCH 2/6] Update macOS versions (test pipeline). --- jobs/test_tarball.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/test_tarball.groovy b/jobs/test_tarball.groovy index e148a188..fa54b8c5 100644 --- a/jobs/test_tarball.groovy +++ b/jobs/test_tarball.groovy @@ -20,7 +20,7 @@ p.pipeline().with { booleanParam('WIPEOUT', false, 'Completely wipe out workspace(s) before starting build.') stringParam('TIMEOUT', '8', 'build timeout in hours') stringParam('IMAGE', null, 'Pipeline base docker image (e.g. docker.io/lsstdm/scipipe-base:8 or empty for macOS)') - choiceParam('LABEL', ['centos-8-conda', 'centos-7-conda', 'osx-10.11', 'osx-10.12', 'osx-10.14'], 'Jenkins build agent label') + choiceParam('LABEL', ['centos-8-conda', 'centos-7-conda', 'osx-10.13', 'osx-10.14'], 'Jenkins build agent label') stringParam('COMPILER', 'conda-system', 'compiler version string') choiceParam('PYTHON_VERSION', ['3'], 'Python major version') stringParam('MINIVER', scipipe.template.tarball_defaults.miniver, 'Miniconda installer version') From b9af6d92bd08a95e984c2f1ed4c58e8f3294c2d4 Mon Sep 17 00:00:00 2001 From: Kian-Tat Lim Date: Fri, 23 Apr 2021 09:03:53 -0700 Subject: [PATCH 3/6] Add doc build/upload to stack-os-matrix. --- jobs/stack_os_matrix.groovy | 5 ++-- pipelines/lib/util.groovy | 40 ++++++++++++++++++++++++++++++++ pipelines/stack_os_matrix.groovy | 2 ++ 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/jobs/stack_os_matrix.groovy b/jobs/stack_os_matrix.groovy index c9930b52..913f584b 100644 --- a/jobs/stack_os_matrix.groovy +++ b/jobs/stack_os_matrix.groovy @@ -11,16 +11,15 @@ p.pipeline().with { stringParam('REFS', null, 'Whitespace delimited list of "refs" to attempt to build. Priority is highest -> lowest from left to right. "master" or branch from repos.yaml is implicitly appended to the right side of the list, if not specified; do not specify "master" explicitly.') stringParam('PRODUCTS', scipipe.canonical.products, 'Whitespace delimited list of EUPS products to build.') + booleanParam('BUILD_DOCS', true, 'Build pipelines.lsst.io site') + booleanParam('PUBLISH_DOCS', true, 'Publish pipelines.lsst.io/v edition') stringParam('SPLENV_REF', scipipe.template.splenv_ref, 'conda env ref') - // XXX testing only - //booleanParam('NO_FETCH', false, 'Do not pull from git remote if branch is already the current ref. (This should generally be false outside of testing the CI system)') } concurrentBuild(true) environmentVariables( BUILD_CONFIG: 'scipipe-lsstsw-matrix', - BUILD_DOCS: false, WIPEOUT: false, ) } diff --git a/pipelines/lib/util.groovy b/pipelines/lib/util.groovy index c5302a8b..a2e5fe2f 100644 --- a/pipelines/lib/util.groovy +++ b/pipelines/lib/util.groovy @@ -339,8 +339,48 @@ def lsstswBuild( LSST_SPLENV_REF: lsstswConfig.splenv_ref ] + buildParams + if (lsstswConfig.build_docs && buildParams['LSST_BUILD_DOCS'] == "true") { + buildParams['LSST_PRODUCTS'] += " pipelines_lsst_io" + // suppress old-style doc build + buildParams['LSST_BUILD_DOCS'] = "false" + } else { + // don't publish docs if we didn't build them + buildParams['LSST_PUBLISH_DOCS'] = "false" + } + def run = { jenkinsWrapper(buildParams) + + if (buildParams['LSST_PUBLISH_DOCS'] == "true") { + withEnv(["LSST_REFS=${buildParams['LSST_REFS']}"]) { + withCredentials([[ + $class: 'UsernamePasswordMultiBinding', + credentialsId: 'ltd-upload', + usernameVariable: 'LTD_USERNAME', + passwordVariable: 'LTD_PASSWORD', + ]]) { + bash ''' + set +o xtrace + cd lsstsw + source bin/envconfig + ( + conda activate ltd && + conda install -y -c conda-forge ltd-conveyor + ) || ( + conda create -y -n ltd -c conda-forge ltd-conveyor && + conda activate ltd + ) + set -o xtrace + GIT_REF=${LSST_REFS// /-} + ln -s ../../_doxygen/html/cpp-api build/pipelines_lsst_io/_build/html/cpp-api + ltd upload --product pipelines --dir build/pipelines_lsst_io/_build/html --git-ref "$GIT_REF" + set +o xtrace + conda deactivate + set -o xtrace + ''' + } // withCredentials + } // withEnv + } // if LSST_PUBLISH_DOCS } // run def runDocker = { diff --git a/pipelines/stack_os_matrix.groovy b/pipelines/stack_os_matrix.groovy index 0bfde471..84bec282 100644 --- a/pipelines/stack_os_matrix.groovy +++ b/pipelines/stack_os_matrix.groovy @@ -23,6 +23,7 @@ notify.wrap { 'BUILD_CONFIG', 'PRODUCTS', 'BUILD_DOCS', + 'PUBLISH_DOCS', 'WIPEOUT', ]) @@ -30,6 +31,7 @@ notify.wrap { LSST_REFS: REFS, LSST_PRODUCTS: PRODUCTS, LSST_BUILD_DOCS: BUILD_DOCS, + LSST_PUBLISH_DOCS: PUBLISH_DOCS, ] // override conda env ref from build_matrix.yaml From 3fc234404ff5a0557568c0ea24da41ab56203a21 Mon Sep 17 00:00:00 2001 From: Kian-Tat Lim Date: Fri, 23 Apr 2021 09:02:14 -0700 Subject: [PATCH 4/6] Define platforms for doc build. --- etc/scipipe/build_matrix.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/etc/scipipe/build_matrix.yaml b/etc/scipipe/build_matrix.yaml index e8a3f669..0a203444 100644 --- a/etc/scipipe/build_matrix.yaml +++ b/etc/scipipe/build_matrix.yaml @@ -99,9 +99,11 @@ scipipe-lsstsw-matrix: label: osx-10.13||osx-10.14 display_name: macos display_compiler: clang + build_docs: true scipipe-lsstsw-lsst_distrib: - <<: *el7-conda - <<: *catalina-conda + build_docs: true - <<: *bigsur-conda scipipe-lsstsw-ci_hsc: - <<: *el7-conda @@ -119,6 +121,7 @@ canonical: <<: *el7-conda label: snowflake-0 display_name: centos-7 + build_docs: true workspace: snowflake/release # # eups distrib tarball configuration -- used by From 78d49dd1e287a535c475e9790a57348f51e850ef Mon Sep 17 00:00:00 2001 From: Kian-Tat Lim Date: Fri, 23 Apr 2021 09:54:32 -0700 Subject: [PATCH 5/6] Use group command instead of subshell. --- pipelines/lib/util.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pipelines/lib/util.groovy b/pipelines/lib/util.groovy index a2e5fe2f..81919ed5 100644 --- a/pipelines/lib/util.groovy +++ b/pipelines/lib/util.groovy @@ -363,13 +363,13 @@ def lsstswBuild( set +o xtrace cd lsstsw source bin/envconfig - ( + { conda activate ltd && conda install -y -c conda-forge ltd-conveyor - ) || ( + } || { conda create -y -n ltd -c conda-forge ltd-conveyor && conda activate ltd - ) + } set -o xtrace GIT_REF=${LSST_REFS// /-} ln -s ../../_doxygen/html/cpp-api build/pipelines_lsst_io/_build/html/cpp-api From 71f446498a4173e7aaf78ed58f872b0d4f12d002 Mon Sep 17 00:00:00 2001 From: Kian-Tat Lim Date: Tue, 27 Apr 2021 17:54:07 -0700 Subject: [PATCH 6/6] Remove unneeded symlink. --- pipelines/lib/util.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/pipelines/lib/util.groovy b/pipelines/lib/util.groovy index 81919ed5..40ad6cb0 100644 --- a/pipelines/lib/util.groovy +++ b/pipelines/lib/util.groovy @@ -372,7 +372,6 @@ def lsstswBuild( } set -o xtrace GIT_REF=${LSST_REFS// /-} - ln -s ../../_doxygen/html/cpp-api build/pipelines_lsst_io/_build/html/cpp-api ltd upload --product pipelines --dir build/pipelines_lsst_io/_build/html --git-ref "$GIT_REF" set +o xtrace conda deactivate