From 8d9a764e7915c863d151e28304ebaceecb743a97 Mon Sep 17 00:00:00 2001 From: Dennis Bell Date: Mon, 7 Mar 2022 00:32:14 -0800 Subject: [PATCH] [ci] refactor CI to match bosh pipeline --- ci/{scripts => }/cats.yml | 0 ci/envs/ci-baseline.yml | 20 - ci/envs/ci-cats.yml | 28 -- ci/envs/ci-gcp-baseline.yml | 16 + ci/envs/ci-gcp-cats.yml | 17 + ci/envs/ci-gcp-upgrade.yml | 16 + ci/envs/ci-gcp.yml | 6 + ci/envs/ci-vsphere-baseline.yml | 16 + ci/envs/ci-vsphere-cats.yml | 17 + ci/envs/ci-vsphere-upgrade.yml | 16 + ci/envs/ci-vsphere.yml | 6 + ci/envs/ci.yml | 5 + ci/pipeline/base.yml | 80 ++-- ci/pipeline/jobs-upstream.yml | 177 -------- ci/pipeline/jobs.yml | 340 ---------------- ci/pipeline/jobs/acceptance-tests.yml | 135 ++++++ ci/pipeline/jobs/build-kit.yml | 36 ++ ci/pipeline/jobs/deploy.yml | 29 ++ ci/pipeline/jobs/prepare.yml | 34 ++ ci/pipeline/jobs/ship-prerelease.yml | 40 ++ ci/pipeline/jobs/ship-release.yml | 65 +++ ci/pipeline/jobs/spec-check.yml | 25 ++ ci/pipeline/jobs/spec-tests.yml | 23 ++ ci/pipeline/jobs/upgrade.yml | 40 ++ ci/pipeline/jobs/upstream-sync.yml | 35 ++ ci/pipeline/jobs/version-major.yml | 21 + ci/pipeline/jobs/version-minor.yml | 21 + ci/pipeline/jobs/version-patch.yml | 21 + ci/pipeline/resources-upstream.yml | 14 - ci/pipeline/resources.yml | 69 ---- ci/pipeline/resources/build.yml | 9 + ci/pipeline/resources/cats.yml | 7 + ci/pipeline/resources/git-ci.yml | 10 + ci/pipeline/resources/git-latest-tag.yml | 10 + ci/pipeline/resources/git-main.yml | 8 + ci/pipeline/resources/git.yml | 13 + ci/pipeline/resources/github-prerelease.yml | 9 + ci/pipeline/resources/github.yml | 7 + ci/pipeline/resources/notify.yml | 15 + ci/pipeline/resources/release-notes.yml | 10 + ci/pipeline/resources/spec-check.yml | 9 + ci/pipeline/resources/upstream.yml | 8 + ci/pipeline/resources/version.yml | 11 + ci/repipe | 242 +++++++++-- ci/scripts/build-kit | 52 +++ ci/scripts/build-upstream-list | 27 -- ci/scripts/cats | 109 +++-- ci/scripts/check-sha1s | 21 + .../compare-release-specs} | 61 ++- ci/scripts/deploy | 96 +++++ ci/scripts/generate-release-notes | 40 ++ ci/scripts/get-latest-upstream | 114 ++++++ ci/scripts/pull-upstream | 2 +- ci/scripts/release | 113 ++++++ ci/scripts/release-notes | 383 ++++++++++++++---- ci/scripts/shipit | 93 ----- ci/scripts/spec-check | 23 ++ ci/scripts/test | 80 ---- ci/scripts/test-deployment | 216 ++++++++++ ci/settings.yml | 9 +- ci/tasks/build-kit.yml | 21 + ci/tasks/cats.yml | 26 ++ ci/tasks/cats/task | 16 - ci/tasks/cats/task.yml | 38 -- .../task.yml => deploy-stable.yml} | 15 +- ci/tasks/deploy.yml | 28 ++ ci/tasks/generate-release-notes.yml | 23 ++ ci/tasks/generate-release-notes/task | 64 --- ci/tasks/generate-release-notes/task.yml | 18 - ci/tasks/get-latest-cf-deployment/task | 38 -- .../task.yml => get-latest-upstream.yml} | 6 +- ci/tasks/prerelease.yml | 30 ++ ci/tasks/{shipit/task.yml => release.yml} | 17 +- ci/tasks/shipit/task | 81 ---- ci/tasks/spec-check.yml | 21 + ci/tasks/spec-check/task | 9 - ci/tasks/spec-check/task.yml | 15 - .../{spec-tests/task.yml => spec-tests.yml} | 6 +- ci/tasks/spec-tests/task | 5 - ci/tasks/test-flight/task | 53 --- .../{test-flight/task.yml => upgrade.yml} | 16 +- ci/upstream.yml | 73 ---- 82 files changed, 2274 insertions(+), 1520 deletions(-) rename ci/{scripts => }/cats.yml (100%) delete mode 100644 ci/envs/ci-baseline.yml delete mode 100644 ci/envs/ci-cats.yml create mode 100644 ci/envs/ci-gcp-baseline.yml create mode 100644 ci/envs/ci-gcp-cats.yml create mode 100644 ci/envs/ci-gcp-upgrade.yml create mode 100644 ci/envs/ci-gcp.yml create mode 100644 ci/envs/ci-vsphere-baseline.yml create mode 100644 ci/envs/ci-vsphere-cats.yml create mode 100644 ci/envs/ci-vsphere-upgrade.yml create mode 100644 ci/envs/ci-vsphere.yml delete mode 100644 ci/pipeline/jobs-upstream.yml delete mode 100644 ci/pipeline/jobs.yml create mode 100644 ci/pipeline/jobs/acceptance-tests.yml create mode 100644 ci/pipeline/jobs/build-kit.yml create mode 100644 ci/pipeline/jobs/deploy.yml create mode 100644 ci/pipeline/jobs/prepare.yml create mode 100644 ci/pipeline/jobs/ship-prerelease.yml create mode 100644 ci/pipeline/jobs/ship-release.yml create mode 100644 ci/pipeline/jobs/spec-check.yml create mode 100644 ci/pipeline/jobs/spec-tests.yml create mode 100644 ci/pipeline/jobs/upgrade.yml create mode 100644 ci/pipeline/jobs/upstream-sync.yml create mode 100644 ci/pipeline/jobs/version-major.yml create mode 100644 ci/pipeline/jobs/version-minor.yml create mode 100644 ci/pipeline/jobs/version-patch.yml delete mode 100644 ci/pipeline/resources-upstream.yml delete mode 100644 ci/pipeline/resources.yml create mode 100644 ci/pipeline/resources/build.yml create mode 100644 ci/pipeline/resources/cats.yml create mode 100644 ci/pipeline/resources/git-ci.yml create mode 100644 ci/pipeline/resources/git-latest-tag.yml create mode 100644 ci/pipeline/resources/git-main.yml create mode 100644 ci/pipeline/resources/git.yml create mode 100644 ci/pipeline/resources/github-prerelease.yml create mode 100644 ci/pipeline/resources/github.yml create mode 100644 ci/pipeline/resources/notify.yml create mode 100644 ci/pipeline/resources/release-notes.yml create mode 100644 ci/pipeline/resources/spec-check.yml create mode 100644 ci/pipeline/resources/upstream.yml create mode 100644 ci/pipeline/resources/version.yml create mode 100755 ci/scripts/build-kit delete mode 100755 ci/scripts/build-upstream-list create mode 100755 ci/scripts/check-sha1s rename ci/{tasks/spec-check/spec-check => scripts/compare-release-specs} (82%) create mode 100755 ci/scripts/deploy create mode 100755 ci/scripts/generate-release-notes create mode 100755 ci/scripts/get-latest-upstream create mode 100755 ci/scripts/release delete mode 100755 ci/scripts/shipit create mode 100755 ci/scripts/spec-check delete mode 100755 ci/scripts/test create mode 100755 ci/scripts/test-deployment create mode 100644 ci/tasks/build-kit.yml create mode 100644 ci/tasks/cats.yml delete mode 100755 ci/tasks/cats/task delete mode 100644 ci/tasks/cats/task.yml rename ci/tasks/{deploy-stable/task.yml => deploy-stable.yml} (52%) create mode 100644 ci/tasks/deploy.yml create mode 100644 ci/tasks/generate-release-notes.yml delete mode 100755 ci/tasks/generate-release-notes/task delete mode 100644 ci/tasks/generate-release-notes/task.yml delete mode 100755 ci/tasks/get-latest-cf-deployment/task rename ci/tasks/{get-latest-cf-deployment/task.yml => get-latest-upstream.yml} (64%) create mode 100644 ci/tasks/prerelease.yml rename ci/tasks/{shipit/task.yml => release.yml} (59%) delete mode 100755 ci/tasks/shipit/task create mode 100644 ci/tasks/spec-check.yml delete mode 100755 ci/tasks/spec-check/task delete mode 100644 ci/tasks/spec-check/task.yml rename ci/tasks/{spec-tests/task.yml => spec-tests.yml} (66%) delete mode 100755 ci/tasks/spec-tests/task delete mode 100755 ci/tasks/test-flight/task rename ci/tasks/{test-flight/task.yml => upgrade.yml} (60%) delete mode 100644 ci/upstream.yml diff --git a/ci/scripts/cats.yml b/ci/cats.yml similarity index 100% rename from ci/scripts/cats.yml rename to ci/cats.yml diff --git a/ci/envs/ci-baseline.yml b/ci/envs/ci-baseline.yml deleted file mode 100644 index 9ae0198c..00000000 --- a/ci/envs/ci-baseline.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -kit: - features: - - haproxy - - tls - - self-signed - - small-footprint - - cf-deployment/operations/enable-service-discovery - -genesis: - env: ci-baseline - bosh_env: snw-genesis-ci - -params: - disk_type: default - base_domain: 10.128.9.10.netip.cc - postgres_vip: 10.128.9.9 - availability_zones: [z1] - haproxy_ips: - - 10.128.9.10 diff --git a/ci/envs/ci-cats.yml b/ci/envs/ci-cats.yml deleted file mode 100644 index 52fde55e..00000000 --- a/ci/envs/ci-cats.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -kit: - features: - - haproxy - - tls - - self-signed - # - small-footprint - - cf-deployment/operations/enable-service-discovery - -genesis: - env: ci-cats - bosh_env: snw-genesis-ci - -params: - disk_type: default - base_domain: 10.128.8.192.netip.cc - postgres_vip: 10.128.8.193 - # availability_zones: [z1] - haproxy_instances: 1 - haproxy_ips: - - 10.128.8.192 - - cf_core_network: default - cf_edge_network: default - cf_runtime_network: default - cf_db_network: default - - diego_cell_instances: 4 diff --git a/ci/envs/ci-gcp-baseline.yml b/ci/envs/ci-gcp-baseline.yml new file mode 100644 index 00000000..43e3fcc0 --- /dev/null +++ b/ci/envs/ci-gcp-baseline.yml @@ -0,0 +1,16 @@ +--- +kit: + features: + - ((append)) + - small-footprint + +genesis: + env: ci-vsphere-baseline + +params: + disk_type: default + base_domain: (( concat meta._ip_prefix "9.10.netip.cc" )) + postgres_vip: (( concat meta._ip_prefix "9.9" )) + availability_zones: [z1] + haproxy_ips: [ (( concat meta._ip_prefix "9.10" )) ] + haproxy_ips: [10.128.9.10] diff --git a/ci/envs/ci-gcp-cats.yml b/ci/envs/ci-gcp-cats.yml new file mode 100644 index 00000000..edae2a39 --- /dev/null +++ b/ci/envs/ci-gcp-cats.yml @@ -0,0 +1,17 @@ +--- +genesis: + env: ci-vsphere-cats + +params: + disk_type: default + base_domain: (( concat meta._ip_prefix "8.192.netip.cc" )) + postgres_vip: (( concat meta._ip_prefix "8.193" )) + haproxy_instances: 1 + haproxy_ips: [ (( concat meta._ip_prefix "8.192" )) ] + diego_cell_instances: 4 + + cf_core_network: default + cf_edge_network: default + cf_runtime_network: default + cf_db_network: default + diff --git a/ci/envs/ci-gcp-upgrade.yml b/ci/envs/ci-gcp-upgrade.yml new file mode 100644 index 00000000..43e3fcc0 --- /dev/null +++ b/ci/envs/ci-gcp-upgrade.yml @@ -0,0 +1,16 @@ +--- +kit: + features: + - ((append)) + - small-footprint + +genesis: + env: ci-vsphere-baseline + +params: + disk_type: default + base_domain: (( concat meta._ip_prefix "9.10.netip.cc" )) + postgres_vip: (( concat meta._ip_prefix "9.9" )) + availability_zones: [z1] + haproxy_ips: [ (( concat meta._ip_prefix "9.10" )) ] + haproxy_ips: [10.128.9.10] diff --git a/ci/envs/ci-gcp.yml b/ci/envs/ci-gcp.yml new file mode 100644 index 00000000..b8f4c051 --- /dev/null +++ b/ci/envs/ci-gcp.yml @@ -0,0 +1,6 @@ +--- +genesis: + bosh_env: gcp-uswest2-genesis-ci + +meta: + _ip_prefix: "10.4." diff --git a/ci/envs/ci-vsphere-baseline.yml b/ci/envs/ci-vsphere-baseline.yml new file mode 100644 index 00000000..43e3fcc0 --- /dev/null +++ b/ci/envs/ci-vsphere-baseline.yml @@ -0,0 +1,16 @@ +--- +kit: + features: + - ((append)) + - small-footprint + +genesis: + env: ci-vsphere-baseline + +params: + disk_type: default + base_domain: (( concat meta._ip_prefix "9.10.netip.cc" )) + postgres_vip: (( concat meta._ip_prefix "9.9" )) + availability_zones: [z1] + haproxy_ips: [ (( concat meta._ip_prefix "9.10" )) ] + haproxy_ips: [10.128.9.10] diff --git a/ci/envs/ci-vsphere-cats.yml b/ci/envs/ci-vsphere-cats.yml new file mode 100644 index 00000000..edae2a39 --- /dev/null +++ b/ci/envs/ci-vsphere-cats.yml @@ -0,0 +1,17 @@ +--- +genesis: + env: ci-vsphere-cats + +params: + disk_type: default + base_domain: (( concat meta._ip_prefix "8.192.netip.cc" )) + postgres_vip: (( concat meta._ip_prefix "8.193" )) + haproxy_instances: 1 + haproxy_ips: [ (( concat meta._ip_prefix "8.192" )) ] + diego_cell_instances: 4 + + cf_core_network: default + cf_edge_network: default + cf_runtime_network: default + cf_db_network: default + diff --git a/ci/envs/ci-vsphere-upgrade.yml b/ci/envs/ci-vsphere-upgrade.yml new file mode 100644 index 00000000..43e3fcc0 --- /dev/null +++ b/ci/envs/ci-vsphere-upgrade.yml @@ -0,0 +1,16 @@ +--- +kit: + features: + - ((append)) + - small-footprint + +genesis: + env: ci-vsphere-baseline + +params: + disk_type: default + base_domain: (( concat meta._ip_prefix "9.10.netip.cc" )) + postgres_vip: (( concat meta._ip_prefix "9.9" )) + availability_zones: [z1] + haproxy_ips: [ (( concat meta._ip_prefix "9.10" )) ] + haproxy_ips: [10.128.9.10] diff --git a/ci/envs/ci-vsphere.yml b/ci/envs/ci-vsphere.yml new file mode 100644 index 00000000..954d8ca7 --- /dev/null +++ b/ci/envs/ci-vsphere.yml @@ -0,0 +1,6 @@ +--- +genesis: + bosh_env: snw-genesis-ci + +meta: + _ip_prefix: "10.128." diff --git a/ci/envs/ci.yml b/ci/envs/ci.yml index 245a91d3..64784a55 100644 --- a/ci/envs/ci.yml +++ b/ci/envs/ci.yml @@ -2,3 +2,8 @@ kit: name: dev version: latest + features: + - haproxy + - tls + - self-signed + - cf-deployment/operations/enable-service-discovery diff --git a/ci/pipeline/base.yml b/ci/pipeline/base.yml index bdc3a9a4..606723e5 100644 --- a/ci/pipeline/base.yml +++ b/ci/pipeline/base.yml @@ -19,6 +19,13 @@ meta: url: (( param "Please specify the full url of the target Concourse CI" )) pipeline: (( grab meta.name )) + upstream: + package: example # dummy value for things that don't use upstream + path: (( grab meta.upstream.package )) + org: cloudfoundry + repo: (( concat meta.upstream.org "/" meta.upstream.package )) + url: (( concat "https://github.com/" meta.upstream.repo ".git" )) + vault: url: (( param "Please provide the address of your Vault" )) token: (( param "Please provide a Vault Token" )) @@ -44,17 +51,12 @@ meta: branch: master private_key: (( param "Please generate an SSH Deployment Key for this repo and specify it here" )) access_token: (( param "Please generate a Personal Access Token to be used for creating github releases (do you have a ci-bot?)" )) - - credhub: - url: (( param "Please provide a credhub url" )) - username: (( param "Please provide a Credhub username" )) - password: (( param "Please provide a Credhub password" )) - - bosh: - uri: (( param "Please specify the Genesis CI BOSH Director URI" )) - cacert: (( param "Please specify the Genesis CI BOSH Director Root CA cert" )) - username: admin - password: (( param "Please specify the Genesis CI BOSH Director admin password" )) + release_notes: + repo: "ci-release-notes" + branch: "main" + file: (( concat meta.name "-release-notes.md" )) + uri: (( concat "git@github.com:" meta.github.owner "/" meta.github.release_notes.repo )) + edit: (( concat "https://github.com/" meta.github.owner "/" meta.github.release_notes.repo "/edit/" meta.github.release_notes.branch "/" meta.github.release_notes.file )) shout: topic: (( concat meta.name "-pipeline" )) @@ -71,47 +73,41 @@ meta: pipeline: (( concat meta.url "/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME" )) build: (( concat meta.shout.links.pipeline "/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME" )) - jobs: - cats: - params: - GENESIS_HONOR_ENV: 1 - GIT_NAME: (( grab meta.git.name )) - GIT_EMAIL: (( grab meta.git.email )) - VAULT_URI: (( grab meta.vault.url )) - VAULT_TOKEN: (( grab meta.vault.token )) - KIT_SHORTNAME: (( grab meta.kit )) - BOSH_ENVIRONMENT: (( grab meta.bosh.uri )) - BOSH_CA_CERT: (( grab meta.bosh.cacert )) - BOSH_CLIENT: (( grab meta.bosh.username )) - BOSH_CLIENT_SECRET: (( grab meta.bosh.password )) - CREDHUB_URL: (( grab meta.credhub.url)) - CREDHUB_USER: (( grab meta.credhub.username)) - CREDHUB_PASSWORD: (( grab meta.credhub.password)) - DEPLOYMENT_NAMES: ci-cats - SKIP_CATS: false - FAILFAST: true - VERBOSE: false - CATS_NODES: 6 - - + task_connectivity_params: + GIT_NAME: (( grab meta.git.name )) + GIT_EMAIL: (( grab meta.git.email )) + VAULT_URI: (( grab meta.vault.url )) + VAULT_TOKEN: (( grab meta.vault.token )) + INFRASTRUCTURE: (( grab meta.iaas )) + + cats_params: + .: (( inject meta.task_connectivity_params )) + DEPLOY_ENV: (( concat "ci-" meta.iaas "-cats" )) + KIT_SHORTNAME: (( grab meta.kit )) + SKIP_CATS: false + FAILFAST: true + VERBOSE: false + CATS_NODES: 6 groups: - name: (( grab meta.pipeline )) jobs: + - build-kit - spec-tests - spec-check - - testflight - - upgrade-test - - cats-tests - - pre - - rc - - shipit + - ship-prerelease + - deploy + - upgrade + - acceptance-tests + - prepare + - ship-release - name: upstream jobs: - - cf-deployment + - upstream-sync + - name: versions jobs: - major - minor - - patch \ No newline at end of file + - patch diff --git a/ci/pipeline/jobs-upstream.yml b/ci/pipeline/jobs-upstream.yml deleted file mode 100644 index e4418451..00000000 --- a/ci/pipeline/jobs-upstream.yml +++ /dev/null @@ -1,177 +0,0 @@ -jobs: -- name: cf-deployment - public: true - serial: true - plan: - - do: - - in_parallel: - - { get: git, trigger: false, passed: [spec-tests] } - - { get: git-ci, trigger: false} - - { get: cf-deployment, trigger: true } - - task: update-cf-deployment - file: git-ci/ci/tasks/get-latest-cf-deployment/task.yml - - put: git - params: - merge: true - repository: git - on_success: - put: notify - params: - topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) - message: tests job '$BUILD_JOB_NAME' succeeded. - ok: yes - link: (( grab meta.shout.links.build )) - on_failure: - put: notify - params: - topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) - message: tests job '$BUILD_JOB_NAME' failed. - ok: no - link: (( grab meta.shout.links.build )) - -- name: cats-tests - public: true - serial: true - serial_groups: [genesis-ci] - plan: - - do: - - in_parallel: - - { get: git, trigger: true, passed: [spec-tests] } - - { get: git-ci } - - { get: cats } - - task: testflights - file: git-ci/ci/tasks/test-flight/task.yml - params: - GENESIS_HONOR_ENV: "1" - GIT_NAME: (( grab meta.git.name )) - GIT_EMAIL: (( grab meta.git.email )) - VAULT_URI: (( grab meta.vault.url )) - VAULT_TOKEN: (( grab meta.vault.token )) - KIT_SHORTNAME: (( grab meta.kit )) - BOSH_ENVIRONMENT: (( grab meta.bosh.uri )) - BOSH_CA_CERT: (( grab meta.bosh.cacert )) - BOSH_CLIENT: (( grab meta.bosh.username )) - BOSH_CLIENT_SECRET: (( grab meta.bosh.password )) - CREDHUB_URL: (( grab meta.credhub.url)) - CREDHUB_USER: (( grab meta.credhub.username)) - CREDHUB_PASSWORD: (( grab meta.credhub.password)) - DEPLOYMENT_NAMES: ci-cats - SKIP_FRESH: true - SKIP_CLEAN: true - SKIP_SMOKE_TESTS: false - - do: - - in_parallel: - fail_fast: true - limit: 1 - steps: - - task: cats-apps - file: git-ci/ci/tasks/cats/task.yml - attempts: 5 - params: - .: (( inject meta.jobs.cats.params )) - RUN_CATS: include_apps - - task: cats-detect - file: git-ci/ci/tasks/cats/task.yml - attempts: 5 - params: - .: (( inject meta.jobs.cats.params )) - RUN_CATS: include_detect - - task: cats-routing - file: git-ci/ci/tasks/cats/task.yml - attempts: 5 - params: - .: (( inject meta.jobs.cats.params )) - RUN_CATS: include_routing - - task: cats-backend-compability - file: git-ci/ci/tasks/cats/task.yml - attempts: 5 - params: - .: (( inject meta.jobs.cats.params )) - RUN_CATS: include_backend_compatibility - - task: cats-internet-dependent - file: git-ci/ci/tasks/cats/task.yml - attempts: 5 - params: - .: (( inject meta.jobs.cats.params )) - RUN_CATS: include_internet_dependent - # - task: cats-route-services - # file: git-ci/ci/tasks/cats/task.yml - # attempts: 5 - # params: - # .: (( inject meta.jobs.cats.params )) - # RUN_CATS: include_route_services - - task: cats-sec-grps - file: git-ci/ci/tasks/cats/task.yml - attempts: 5 - params: - .: (( inject meta.jobs.cats.params )) - RUN_CATS: include_security_groups - - task: cats-services - file: git-ci/ci/tasks/cats/task.yml - attempts: 5 - params: - .: (( inject meta.jobs.cats.params )) - RUN_CATS: include_services - - task: cats-ssh - file: git-ci/ci/tasks/cats/task.yml - attempts: 5 - params: - .: (( inject meta.jobs.cats.params )) - RUN_CATS: include_ssh - - task: cats-sso - file: git-ci/ci/tasks/cats/task.yml - attempts: 5 - params: - .: (( inject meta.jobs.cats.params )) - RUN_CATS: include_sso - - task: cats-tasks - file: git-ci/ci/tasks/cats/task.yml - attempts: 5 - params: - .: (( inject meta.jobs.cats.params )) - RUN_CATS: include_tasks - - task: cats-v3 - file: git-ci/ci/tasks/cats/task.yml - attempts: 5 - params: - .: (( inject meta.jobs.cats.params )) - RUN_CATS: include_v3 - - task: cats-deployments - file: git-ci/ci/tasks/cats/task.yml - attempts: 5 - params: - .: (( inject meta.jobs.cats.params )) - RUN_CATS: include_deployments - - task: cats-service_discovery - file: git-ci/ci/tasks/cats/task.yml - attempts: 5 - params: - .: (( inject meta.jobs.cats.params )) - RUN_CATS: include_service_discovery - - task: cats-capi_no_bridge - file: git-ci/ci/tasks/cats/task.yml - attempts: 5 - params: - .: (( inject meta.jobs.cats.params )) - RUN_CATS: include_capi_no_bridge - - task: cleanup-enviorment - file: git-ci/ci/tasks/test-flight/task.yml - params: - GENESIS_HONOR_ENV: "1" - GIT_NAME: (( grab meta.git.name )) - GIT_EMAIL: (( grab meta.git.email )) - VAULT_URI: (( grab meta.vault.url )) - VAULT_TOKEN: (( grab meta.vault.token )) - KIT_SHORTNAME: (( grab meta.kit )) - BOSH_ENVIRONMENT: (( grab meta.bosh.uri )) - BOSH_CA_CERT: (( grab meta.bosh.cacert )) - BOSH_CLIENT: (( grab meta.bosh.username )) - BOSH_CLIENT_SECRET: (( grab meta.bosh.password )) - CREDHUB_URL: (( grab meta.credhub.url)) - CREDHUB_USER: (( grab meta.credhub.username)) - CREDHUB_PASSWORD: (( grab meta.credhub.password)) - DEPLOYMENT_NAMES: ci-cats - SKIP_FRESH: true - SKIP_DEPLOY: true - SKIP_SMOKE_TESTS: true - SKIP_CLEAN: false \ No newline at end of file diff --git a/ci/pipeline/jobs.yml b/ci/pipeline/jobs.yml deleted file mode 100644 index e40f64c1..00000000 --- a/ci/pipeline/jobs.yml +++ /dev/null @@ -1,340 +0,0 @@ -jobs: -- name: spec-tests - public: true - serial: true - serial_groups: [genesis-ci] - plan: - - do: - - in_parallel: - - { get: git, trigger: true } - - { get: git-ci } - - task: spec-testing - file: git-ci/ci/tasks/spec-tests/task.yml - attempts: 2 - on_success: - put: notify - params: - topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) - message: tests job '$BUILD_JOB_NAME' succeeded. - ok: yes - link: (( grab meta.shout.links.build )) - on_failure: - put: notify - params: - topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) - message: tests job '$BUILD_JOB_NAME' failed. - ok: no - link: (( grab meta.shout.links.build )) - -- name: spec-check - public: true - serial: true - serial_groups: [genesis-ci] - plan: - - do: - - in_parallel: - - { get: git, trigger: true } - - { get: git-ci } - - { get: git-latest-tag } - - task: spec-testing - file: git-ci/ci/tasks/spec-check/task.yml - on_success: - put: notify - params: - topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) - message: tests job '$BUILD_JOB_NAME' succeeded. - ok: yes - link: (( grab meta.shout.links.build )) - on_failure: - put: notify - params: - topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) - message: tests job '$BUILD_JOB_NAME' failed. - ok: no - link: (( grab meta.shout.links.build )) - -- name: testflight - public: true - serial: true - serial_groups: [genesis-ci] - plan: - - do: - - in_parallel: - - { get: git, trigger: true, passed: [spec-tests] } - - { get: git-ci } - - task: testflights - file: git-ci/ci/tasks/test-flight/task.yml - params: - GENESIS_HONOR_ENV: "1" - GIT_NAME: (( grab meta.git.name )) - GIT_EMAIL: (( grab meta.git.email )) - VAULT_URI: (( grab meta.vault.url )) - VAULT_TOKEN: (( grab meta.vault.token )) - KIT_SHORTNAME: (( grab meta.kit )) - BOSH_ENVIRONMENT: (( grab meta.bosh.uri )) - BOSH_CA_CERT: (( grab meta.bosh.cacert )) - BOSH_CLIENT: (( grab meta.bosh.username )) - BOSH_CLIENT_SECRET: (( grab meta.bosh.password )) - CREDHUB_URL: (( grab meta.credhub.url)) - CREDHUB_USER: (( grab meta.credhub.username)) - CREDHUB_PASSWORD: (( grab meta.credhub.password)) - SKIP_FRESH: false - SKIP_CLEAN: false - SKIP_SMOKE_TESTS: false - on_success: - put: notify - params: - topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) - message: tests job '$BUILD_JOB_NAME' succeeded. - ok: yes - link: (( grab meta.shout.links.build )) - on_failure: - put: notify - params: - topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) - message: tests job '$BUILD_JOB_NAME' failed. - ok: no - link: (( grab meta.shout.links.build )) - -- name: upgrade-test - public: true - serial: true - serial_groups: [genesis-ci] - plan: - - do: - - in_parallel: - - { get: git, trigger: true, passed: [spec-tests] } - - { get: git-ci } - - { get: git-latest-tag } - - task: deploy-stable - file: git-ci/ci/tasks/deploy-stable/task.yml - params: - GENESIS_HONOR_ENV: "1" - GIT_NAME: (( grab meta.git.name )) - GIT_EMAIL: (( grab meta.git.email )) - VAULT_URI: (( grab meta.vault.url )) - VAULT_TOKEN: (( grab meta.vault.token )) - KIT_SHORTNAME: (( grab meta.kit )) - BOSH_ENVIRONMENT: (( grab meta.bosh.uri )) - BOSH_CA_CERT: (( grab meta.bosh.cacert )) - BOSH_CLIENT: (( grab meta.bosh.username )) - BOSH_CLIENT_SECRET: (( grab meta.bosh.password )) - CREDHUB_URL: (( grab meta.credhub.url)) - CREDHUB_USER: (( grab meta.credhub.username)) - CREDHUB_PASSWORD: (( grab meta.credhub.password)) - SKIP_FRESH: false - SKIP_CLEAN: true - SKIP_SMOKE_TESTS: true - - task: upgrade - file: git-ci/ci/tasks/test-flight/task.yml - params: - GENESIS_HONOR_ENV: "1" - GIT_NAME: (( grab meta.git.name )) - GIT_EMAIL: (( grab meta.git.email )) - VAULT_URI: (( grab meta.vault.url )) - VAULT_TOKEN: (( grab meta.vault.token )) - KIT_SHORTNAME: (( grab meta.kit )) - BOSH_ENVIRONMENT: (( grab meta.bosh.uri )) - BOSH_CA_CERT: (( grab meta.bosh.cacert )) - BOSH_CLIENT: (( grab meta.bosh.username )) - BOSH_CLIENT_SECRET: (( grab meta.bosh.password )) - CREDHUB_URL: (( grab meta.credhub.url)) - CREDHUB_USER: (( grab meta.credhub.username)) - CREDHUB_PASSWORD: (( grab meta.credhub.password)) - SKIP_FRESH: true - SKIP_CLEAN: false - SKIP_SMOKE_TESTS: false - on_success: - put: notify - params: - topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) - message: tests job '$BUILD_JOB_NAME' succeeded. - ok: yes - link: (( grab meta.shout.links.build )) - on_failure: - put: notify - params: - topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) - message: tests job '$BUILD_JOB_NAME' failed. - ok: no - link: (( grab meta.shout.links.build )) - -- name: pre - public: true - serial: true - plan: - - do: - - in_parallel: - - { get: git, trigger: true, passed: [testflight] } - - { get: version, trigger: true } - - { get: git-ci } - - { get: git-latest-tag } - - task: generate-release-notes - file: git-ci/ci/tasks/generate-release-notes/task.yml - on_success: - put: notify - params: - topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) - message: release candidate job 'pre' succeeded. - ok: yes - link: (( grab meta.shout.links.build )) - on_failure: - put: notify - params: - topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) - message: release candidate job 'pre' failed (which is unusual). - ok: no - link: (( grab meta.shout.links.build )) - -- name: rc - public: true - serial: true - plan: - - do: - - in_parallel: - - { get: git, trigger: true, passed: [pre] } - - { get: version, trigger: false, passed: [pre], params: {pre: rc} } - - put: version - params: {file: version/number} - on_success: - put: notify - params: - topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) - message: release candidate job 'rc' succeeded. - ok: yes - link: (( grab meta.shout.links.build )) - on_failure: - put: notify - params: - topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) - message: release candidate job 'rc' failed (which is unusual). - ok: no - link: (( grab meta.shout.links.build )) - -- name: patch - public: true - plan: - - do: - - { get: version, trigger: false, params: {bump: patch} } - - { put: version, params: {file: version/number} } - on_success: - put: notify - params: - topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) - message: minor version bump job 'minor' succeeded. - ok: yes - link: (( grab meta.shout.links.build )) - on_failure: - put: notify - params: - topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) - message: minor version bump job 'minor' failed (which is unusual). - ok: no - link: (( grab meta.shout.links.build )) - -- name: minor - public: true - plan: - - do: - - { get: version, trigger: false, params: {bump: minor} } - - { put: version, params: {file: version/number} } - on_success: - put: notify - params: - topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) - message: minor version bump job 'minor' succeeded. - ok: yes - link: (( grab meta.shout.links.build )) - on_failure: - put: notify - params: - topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) - message: minor version bump job 'minor' failed (which is unusual). - ok: no - link: (( grab meta.shout.links.build )) - - -- name: major - public: true - plan: - - do: - - { get: version, trigger: false, params: {bump: major} } - - { put: version, params: {file: version/number} } - on_success: - put: notify - params: - topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) - message: major version bump job '$BUILD_JOB_NAME' succeeded. - ok: no - link: (( grab meta.shout.links.build )) - on_failure: - put: notify - params: - topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) - message: major version bump job '$BUILD_JOB_NAME' failed (which is unusual). - ok: no - link: (( grab meta.shout.links.build )) - -- name: shipit - public: true - serial: true - plan: - - do: - - in_parallel: - - { get: version, resource: version, passed: [rc], params: {bump: final} } - - { get: git, passed: [rc] } - - { get: git-ci } - - { get: git-main } - - { get: git-latest-tag } - - task: generate-release-notes - file: git-ci/ci/tasks/generate-release-notes/task.yml - - task: release - file: git-ci/ci/tasks/shipit/task.yml - params: - REPO_ROOT: git - VERSION_FROM: version/number - RELEASE_ROOT: gh - NOTIFICATION_OUT: notifications - GITHUB_OWNER: (( grab meta.github.owner )) - GIT_EMAIL: (( grab meta.git.email )) - GIT_NAME: (( grab meta.git.name )) - KIT_SHORTNAME: (( grab meta.kit )) - - put: git - params: - merge: true - repository: git-ci - - put: git-main - params: - merge: true - tag: gh/tag - repository: git-main - - put: github - params: - name: gh/name - tag: gh/tag - body: release-notes/notes.md - globs: [gh/artifacts/*] - - put: version - params: - bump: patch - - in_parallel: - steps: - - put: notify - params: - method: announce - file: notifications/message - link: (( concat meta.github.uri "/releases" )) - on_success: - put: notify - params: - topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) - message: release job '$BUILD_JOB_NAME' succeeded. - ok: yes - link: (( grab meta.shout.links.build )) - on_failure: - put: notify - params: - topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) - message: release job '$BUILD_JOB_NAME' failed. - ok: no - link: (( grab meta.shout.links.build )) \ No newline at end of file diff --git a/ci/pipeline/jobs/acceptance-tests.yml b/ci/pipeline/jobs/acceptance-tests.yml new file mode 100644 index 00000000..b65e284c --- /dev/null +++ b/ci/pipeline/jobs/acceptance-tests.yml @@ -0,0 +1,135 @@ +jobs: +- name: acceptance-tests + public: true + serial: true + serial_groups: [testing] + plan: + - do: + - in_parallel: + - { get: version, passed: [spec-check, spec-tests]} + - { get: build, passed: [spec-check, spec-tests], trigger: true } + - { get: spec-check, passed: [spec-check]} + - { get: git, passed: [spec-check, spec-tests]} + - { get: git-ci } + - { get: cats } + - task: testflights + file: git-ci/ci/tasks/deploy.yml + params: + .: (( inject meta.task_connectivity_params )) + DEPLOY_ENV: (( concat "ci-" meta.iaas "-cats" )) + KIT_SHORTNAME: (( grab meta.kit )) + SKIP_FRESH: false + SKIP_CLEAN: true + SKIP_SMOKE_TESTS: false + - do: + - in_parallel: + fail_fast: true + limit: 1 + steps: + - task: cats-apps + file: git-ci/ci/tasks/cats.yml + attempts: 5 + params: + .: (( inject meta.cats_params )) + RUN_CATS: include_apps + - task: cats-detect + file: git-ci/ci/tasks/cats.yml + attempts: 5 + params: + .: (( inject meta.cats_params )) + RUN_CATS: include_detect + - task: cats-routing + file: git-ci/ci/tasks/cats.yml + attempts: 5 + params: + .: (( inject meta.cats_params )) + RUN_CATS: include_routing + - task: cats-backend-compability + file: git-ci/ci/tasks/cats.yml + attempts: 5 + params: + .: (( inject meta.cats_params )) + RUN_CATS: include_backend_compatibility + - task: cats-internet-dependent + file: git-ci/ci/tasks/cats.yml + attempts: 5 + params: + .: (( inject meta.cats_params )) + RUN_CATS: include_internet_dependent + # - task: cats-route-services + # file: git-ci/ci/tasks/cats.yml + # attempts: 5 + # params: + # .: (( inject meta.cats_params )) + # RUN_CATS: include_route_services + - task: cats-sec-grps + file: git-ci/ci/tasks/cats.yml + attempts: 5 + params: + .: (( inject meta.cats_params )) + RUN_CATS: include_security_groups + - task: cats-services + file: git-ci/ci/tasks/cats.yml + attempts: 5 + params: + .: (( inject meta.cats_params )) + RUN_CATS: include_services + - task: cats-ssh + file: git-ci/ci/tasks/cats.yml + attempts: 5 + params: + .: (( inject meta.cats_params )) + RUN_CATS: include_ssh + - task: cats-sso + file: git-ci/ci/tasks/cats.yml + attempts: 5 + params: + .: (( inject meta.cats_params )) + RUN_CATS: include_sso + - task: cats-tasks + file: git-ci/ci/tasks/cats.yml + attempts: 5 + params: + .: (( inject meta.cats_params )) + RUN_CATS: include_tasks + - task: cats-v3 + file: git-ci/ci/tasks/cats.yml + attempts: 5 + params: + .: (( inject meta.cats_params )) + RUN_CATS: include_v3 + - task: cats-deployments + file: git-ci/ci/tasks/cats.yml + attempts: 5 + params: + .: (( inject meta.cats_params )) + RUN_CATS: include_deployments + - task: cats-service_discovery + file: git-ci/ci/tasks/cats.yml + attempts: 5 + params: + .: (( inject meta.cats_params )) + RUN_CATS: include_service_discovery + - task: cats-capi_no_bridge + file: git-ci/ci/tasks/cats.yml + attempts: 5 + params: + .: (( inject meta.cats_params )) + RUN_CATS: include_capi_no_bridge + - task: cleanup-enviorment + file: git-ci/ci/tasks/deploy.yml + params: + .: (( inject meta.task_connectivity_params )) + DEPLOY_ENV: (( concat "ci-" meta.iaas "-cats" )) + KIT_SHORTNAME: (( grab meta.kit )) + SKIP_FRESH: true + SKIP_DEPLOY: true + SKIP_SMOKE_TESTS: true + SKIP_CLEAN: false + on_failure: + put: notify + params: + topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) + message: release candidate job 'pre' failed (which is unusual). + ok: no + link: (( grab meta.shout.links.build )) diff --git a/ci/pipeline/jobs/build-kit.yml b/ci/pipeline/jobs/build-kit.yml new file mode 100644 index 00000000..939ed66d --- /dev/null +++ b/ci/pipeline/jobs/build-kit.yml @@ -0,0 +1,36 @@ +jobs: +- name: build-kit + public: true + serial: false + plan: + - do: + - in_parallel: + - { get: version, params: {pre: rc} } + - { get: git, trigger: true } + - { get: git-ci } + - task: build-kit + file: git-ci/ci/tasks/build-kit.yml + params: + KIT_SHORTNAME: (( grab meta.kit )) + VAULT_URI: (( grab meta.vault.url )) + VAULT_TOKEN: (( grab meta.vault.token )) + - put: build + params: + file: build/*.tar.gz + acl: public-read + - put: version + params: {file: version/number} + on_success: + put: notify + params: + topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) + message: prerelease build for '$BUILD_JOB_NAME' succeeded. + ok: yes + link: (( grab meta.shout.links.build )) + on_failure: + put: notify + params: + topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) + message: prerelease build for '$BUILD_JOB_NAME' failed. + ok: no + link: (( grab meta.shout.links.build )) diff --git a/ci/pipeline/jobs/deploy.yml b/ci/pipeline/jobs/deploy.yml new file mode 100644 index 00000000..0aa435ee --- /dev/null +++ b/ci/pipeline/jobs/deploy.yml @@ -0,0 +1,29 @@ +jobs: +- name: deploy + public: true + serial_groups: ['deployment'] # Remove if 'upgrade' has a different network + plan: + - do: + - in_parallel: + - { get: version, passed: [spec-check, spec-tests]} + - { get: build, passed: [spec-check, spec-tests], trigger: true } + - { get: spec-check, passed: [spec-check]} + - { get: git, passed: [spec-check, spec-tests]} + - { get: git-ci } + - task: test-deploy + file: git-ci/ci/tasks/deploy.yml + params: + .: (( inject meta.task_connectivity_params )) + DEPLOY_ENV: (( concat "ci-" meta.iaas "-baseline" )) + KIT_SHORTNAME: (( grab meta.kit )) + SKIP_FRESH: false + SKIP_REPLACE_SECRETS: false + SKIP_SMOKE_TESTS: false + SKIP_CLEAN: false + on_failure: + put: notify + params: + topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) + message: tests job '$BUILD_JOB_NAME' failed. + ok: no + link: (( grab meta.shout.links.build )) diff --git a/ci/pipeline/jobs/prepare.yml b/ci/pipeline/jobs/prepare.yml new file mode 100644 index 00000000..f1b6be18 --- /dev/null +++ b/ci/pipeline/jobs/prepare.yml @@ -0,0 +1,34 @@ +jobs: +- name: prepare + public: true + serial: true + plan: + - do: + - in_parallel: + - { get: version, passed: [deploy,upgrade], params: {bump: final} } + - { get: spec-check, passed: [deploy,upgrade] } + - { get: git, passed: [deploy,upgrade], trigger: true } + - { get: git-ci } + - { get: git-latest-tag } + - { get: release-notes } + - task: generate-release-notes + file: git-ci/ci/tasks/generate-release-notes.yml + params: + RELEASE_NOTES_WEB_URL: (( grab meta.github.release_notes.edit )) + RELEASE_NOTES_FILE: (( grab meta.github.release_notes.file )) + GIT_NAME: (( grab meta.git.name )) + GIT_EMAIL: (( grab meta.git.email )) + VAULT_URI: (( grab meta.vault.url )) + VAULT_TOKEN: (( grab meta.vault.token )) + KIT_SHORTNAME: (( grab meta.kit )) + - put: release-notes + params: + rebase: true + repository: release-notes + on_failure: + put: notify + params: + topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) + message: release candidate job 'pre' failed (which is unusual). + ok: no + link: (( grab meta.shout.links.build )) diff --git a/ci/pipeline/jobs/ship-prerelease.yml b/ci/pipeline/jobs/ship-prerelease.yml new file mode 100644 index 00000000..f217484f --- /dev/null +++ b/ci/pipeline/jobs/ship-prerelease.yml @@ -0,0 +1,40 @@ +jobs: +- name: ship-prerelease + public: true + serial: false + plan: + - do: + - in_parallel: + - { get: build, passed: [build-kit]} + - { get: version, passed: [build-kit]} + - { get: git, passed: [build-kit]} + - { get: git-ci } + + - task: ship-prerelease + file: git-ci/ci/tasks/prerelease.yml + params: + PRERELEASE: 1 + KIT_SHORTNAME: (( grab meta.kit )) + DEVELOP_BRANCH: (( grab meta.github.branch )) + RELEASE_BRANCH: (( grab meta.github.branch )) # TODO: main-branch )) + RELEASE_ROOT: gh + RELEASE_NOTES: (( grab meta.github.release_notes.file )) + NOTIFICATION_OUT: notifications + GITHUB_OWNER: (( grab meta.github.owner )) + GIT_EMAIL: (( grab meta.git.email )) + GIT_NAME: (( grab meta.git.name )) + + - put: github-prerelease + params: + name: gh/name + tag: gh/tag + body: gh/notes.md + globs: [gh/artifacts/*] + + on_failure: + put: notify + params: + topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) + message: tests job '$BUILD_JOB_NAME' failed. + ok: no + link: (( grab meta.shout.links.build )) diff --git a/ci/pipeline/jobs/ship-release.yml b/ci/pipeline/jobs/ship-release.yml new file mode 100644 index 00000000..fc3942f1 --- /dev/null +++ b/ci/pipeline/jobs/ship-release.yml @@ -0,0 +1,65 @@ +jobs: +- name: ship-release + public: true + serial: true + plan: + - do: + - in_parallel: + - { get: version, resource: version, passed: [prepare], params: {bump: final} } + - { get: git, passed: [prepare] } + - { get: spec-check, trigger: false, passed: [prepare] } + - { get: release-notes} + - { get: git-ci } + - { get: git-main } + - { get: git-latest-tag } + - task: build-kit + file: git-ci/ci/tasks/build-kit.yml + params: + KIT_SHORTNAME: (( grab meta.kit )) + VAULT_URI: (( grab meta.vault.url )) + VAULT_TOKEN: (( grab meta.vault.token )) + - task: release + file: git-ci/ci/tasks/release.yml + params: + KIT_SHORTNAME: (( grab meta.kit )) + DEVELOP_BRANCH: (( grab meta.github.branch )) + RELEASE_BRANCH: (( grab meta.github.main-branch )) + RELEASE_ROOT: gh + RELEASE_NOTES: (( grab meta.github.release_notes.file )) + NOTIFICATION_OUT: notifications + GITHUB_OWNER: (( grab meta.github.owner )) + GIT_EMAIL: (( grab meta.git.email )) + GIT_NAME: (( grab meta.git.name )) + - put: git-main + params: + merge: false + tag: gh/tag + repository: git-main + - put: github + params: + name: gh/name + tag: gh/tag + body: (( concat "release-notes/" meta.github.release_notes.file )) + globs: [gh/artifacts/*] + - put: version + params: + bump: patch + - put: notify + params: + method: announce + file: notifications/message + link: (( concat meta.github.uri "/releases" )) + on_success: + put: notify + params: + topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) + message: release job '$BUILD_JOB_NAME' succeeded. + ok: yes + link: (( grab meta.shout.links.build )) + on_failure: + put: notify + params: + topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) + message: release job '$BUILD_JOB_NAME' failed. + ok: no + link: (( grab meta.shout.links.build )) diff --git a/ci/pipeline/jobs/spec-check.yml b/ci/pipeline/jobs/spec-check.yml new file mode 100644 index 00000000..9b381047 --- /dev/null +++ b/ci/pipeline/jobs/spec-check.yml @@ -0,0 +1,25 @@ +jobs: +- name: spec-check + public: true + serial: false + plan: + - do: + - in_parallel: + - { get: build, passed: [build-kit], trigger: true} + - { get: version, passed: [build-kit]} + - { get: git, passed: [build-kit]} + - { get: git-ci } + - { get: git-latest-tag } + - task: spec-check + file: git-ci/ci/tasks/spec-check.yml + - put: spec-check + params: + file: spec-check/diff-* + acl: public-read + on_failure: + put: notify + params: + topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) + message: tests job '$BUILD_JOB_NAME' failed. + ok: no + link: (( grab meta.shout.links.build )) diff --git a/ci/pipeline/jobs/spec-tests.yml b/ci/pipeline/jobs/spec-tests.yml new file mode 100644 index 00000000..fb005c91 --- /dev/null +++ b/ci/pipeline/jobs/spec-tests.yml @@ -0,0 +1,23 @@ +jobs: +- name: spec-tests + public: true + serial: false + plan: + - do: + - in_parallel: + - { get: build, passed: [build-kit], trigger: true} + - { get: version, passed: [build-kit]} + - { get: git, passed: [build-kit]} + - { get: git-ci } + - { get: git-latest-tag } + - task: spec-tests + file: git-ci/ci/tasks/spec-tests.yml + attempts: 2 + on_failure: + put: notify + params: + topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) + message: tests job '$BUILD_JOB_NAME' failed. + ok: no + link: (( grab meta.shout.links.build )) + diff --git a/ci/pipeline/jobs/upgrade.yml b/ci/pipeline/jobs/upgrade.yml new file mode 100644 index 00000000..7cd50bd0 --- /dev/null +++ b/ci/pipeline/jobs/upgrade.yml @@ -0,0 +1,40 @@ +jobs: +- name: upgrade + public: true + serial_groups: ['deployment'] + plan: + - do: + - in_parallel: + - { get: version, passed: [spec-check, spec-tests]} + - { get: build, passed: [spec-check, spec-tests], trigger: true } + - { get: spec-check, passed: [spec-check]} + - { get: git, passed: [spec-check, spec-tests]} + - { get: git-ci } + - { get: git-latest-tag } + - task: deploy-stable + file: git-ci/ci/tasks/deploy-stable.yml + params: + .: (( inject meta.task_connectivity_params )) + DEPLOY_ENV: (( concat "ci-" meta.iaas "-upgrade" )) + KIT_SHORTNAME: (( grab meta.kit )) + SKIP_FRESH: false + SKIP_REPLACE_SECRETS: false + SKIP_SMOKE_TESTS: true + SKIP_CLEAN: true + - task: upgrade + file: git-ci/ci/tasks/upgrade.yml + params: + .: (( inject meta.task_connectivity_params )) + DEPLOY_ENV: (( concat "ci-" meta.iaas "-upgrade" )) + KIT_SHORTNAME: (( grab meta.kit )) + SKIP_FRESH: true + SKIP_REPLACE_SECRETS: true + SKIP_SMOKE_TESTS: false + SKIP_CLEAN: false + on_failure: + put: notify + params: + topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) + message: tests job '$BUILD_JOB_NAME' failed. + ok: no + link: (( grab meta.shout.links.build )) diff --git a/ci/pipeline/jobs/upstream-sync.yml b/ci/pipeline/jobs/upstream-sync.yml new file mode 100644 index 00000000..e75f1404 --- /dev/null +++ b/ci/pipeline/jobs/upstream-sync.yml @@ -0,0 +1,35 @@ +jobs: +- name: upstream-sync + public: true + serial: true + plan: + - do: + - in_parallel: + - { get: git, trigger: false, passed: [spec-tests] } + - { get: git-ci, trigger: false} + - { get: upstream, trigger: true } + - task: upstream-sync + file: git-ci/ci/tasks/get-latest-upstream.yml + params: + GIT_EMAIL: (( grab meta.git.email )) + GIT_NAME: (( grab meta.git.name )) + UPSTREAM_PATH: (( grab meta.upstream.path )) + UPSTREAM_REPO: (( grab meta.upstream.repo )) + - put: git + params: + merge: true + repository: git + on_success: + put: notify + params: + topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) + message: tests job '$BUILD_JOB_NAME' succeeded. + ok: yes + link: (( grab meta.shout.links.build )) + on_failure: + put: notify + params: + topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) + message: tests job '$BUILD_JOB_NAME' failed. + ok: no + link: (( grab meta.shout.links.build )) diff --git a/ci/pipeline/jobs/version-major.yml b/ci/pipeline/jobs/version-major.yml new file mode 100644 index 00000000..07b95516 --- /dev/null +++ b/ci/pipeline/jobs/version-major.yml @@ -0,0 +1,21 @@ +jobs: +- name: major + public: true + plan: + - do: + - { get: version, trigger: false, params: {bump: major} } + - { put: version, params: {file: version/number} } + on_success: + put: notify + params: + topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) + message: major version bump job '$BUILD_JOB_NAME' succeeded. + ok: yes + link: (( grab meta.shout.links.build )) + on_failure: + put: notify + params: + topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) + message: major version bump job '$BUILD_JOB_NAME' failed (which is unusual). + ok: no + link: (( grab meta.shout.links.build )) diff --git a/ci/pipeline/jobs/version-minor.yml b/ci/pipeline/jobs/version-minor.yml new file mode 100644 index 00000000..f9d3a97e --- /dev/null +++ b/ci/pipeline/jobs/version-minor.yml @@ -0,0 +1,21 @@ +jobs: +- name: minor + public: true + plan: + - do: + - { get: version, trigger: false, params: {bump: minor} } + - { put: version, params: {file: version/number} } + on_success: + put: notify + params: + topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) + message: minor version bump job '$BUILD_JOB_NAME' succeeded. + ok: yes + link: (( grab meta.shout.links.build )) + on_failure: + put: notify + params: + topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) + message: minor version bump job '$BUILD_JOB_NAME' failed (which is unusual). + ok: no + link: (( grab meta.shout.links.build )) diff --git a/ci/pipeline/jobs/version-patch.yml b/ci/pipeline/jobs/version-patch.yml new file mode 100644 index 00000000..f5d11aa5 --- /dev/null +++ b/ci/pipeline/jobs/version-patch.yml @@ -0,0 +1,21 @@ +jobs: +- name: patch + public: true + plan: + - do: + - { get: version, trigger: false, params: {bump: patch} } + - { put: version, params: {file: version/number} } + on_success: + put: notify + params: + topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) + message: patch version bump job '$BUILD_JOB_NAME' succeeded. + ok: yes + link: (( grab meta.shout.links.build )) + on_failure: + put: notify + params: + topic: (( concat meta.shout.topic "-$BUILD_JOB_NAME" )) + message: patch version bump job '$BUILD_JOB_NAME' failed (which is unusual). + ok: no + link: (( grab meta.shout.links.build )) diff --git a/ci/pipeline/resources-upstream.yml b/ci/pipeline/resources-upstream.yml deleted file mode 100644 index 0585e08d..00000000 --- a/ci/pipeline/resources-upstream.yml +++ /dev/null @@ -1,14 +0,0 @@ -resources: - - name: cf-deployment - type: git - check_every: 1h - source: - uri: https://github.com/cloudfoundry/cf-deployment.git - tag_filter: "v*" - - - name: cats - type: git - check_every: 1h - source: - uri: https://github.com/cloudfoundry/cf-acceptance-tests - tag_filter: "v*" diff --git a/ci/pipeline/resources.yml b/ci/pipeline/resources.yml deleted file mode 100644 index 7dcc40b2..00000000 --- a/ci/pipeline/resources.yml +++ /dev/null @@ -1,69 +0,0 @@ -resource_types: - - name: shout-notification - type: docker-image - source: - repository: huntprod/shout-resource - -resources: - - name: git - type: git - check_every: 1h - source: - uri: (( grab meta.github.uri )) - branch: (( grab meta.github.branch )) - private_key: (( grab meta.github.private_key )) - ignore_paths: [ci/*] - - - name: git-main - type: git - check_every: 1h - source: - uri: (( grab meta.github.uri )) - branch: (( grab meta.github.main-branch )) - private_key: (( grab meta.github.private_key )) - - - name: git-ci - type: git - check_every: 1h - source: - uri: (( grab meta.github.uri )) - branch: (( grab meta.github.branch )) - private_key: (( grab meta.github.private_key )) - paths: [ci/*] - disable_ci_skip: true - - - name: git-latest-tag - type: git - check_every: 1h - source: - uri: (( grab meta.github.uri )) - branch: (( grab meta.github.branch )) - private_key: (( grab meta.github.private_key )) - tag_filter: "v*" - disable_ci_skip: true - - - name: version - type: semver - source : - driver: s3 - bucket: (( grab meta.aws.bucket )) - region_name: (( grab meta.aws.region_name )) - key: (( concat meta.name "/" meta.version_file )) - access_key_id: (( grab meta.aws.access_key )) - secret_access_key: (( grab meta.aws.secret_key )) - initial_version: (( grab meta.initial_version || "0.0.1" )) - - - name: notify - type: shout-notification - source: - topic: (( grab meta.shout.topic )) - url: (( grab meta.shout.url )) - username: (( grab meta.shout.username )) - password: (( grab meta.shout.password )) - - - name: github - type: github-release - source: - user: (( grab meta.github.owner )) - repository: (( grab meta.github.repo )) - access_token: (( grab meta.github.access_token )) diff --git a/ci/pipeline/resources/build.yml b/ci/pipeline/resources/build.yml new file mode 100644 index 00000000..4c206928 --- /dev/null +++ b/ci/pipeline/resources/build.yml @@ -0,0 +1,9 @@ +resources: + - name: build + type: s3 + source: + bucket: (( grab meta.aws.bucket )) + region_name: (( grab meta.aws.region_name )) + regexp: (( concat meta.name "/build/(.*)\.tar.gz" )) + access_key_id: (( grab meta.aws.access_key )) + secret_access_key: (( grab meta.aws.secret_key )) diff --git a/ci/pipeline/resources/cats.yml b/ci/pipeline/resources/cats.yml new file mode 100644 index 00000000..27672b8c --- /dev/null +++ b/ci/pipeline/resources/cats.yml @@ -0,0 +1,7 @@ +resources: + - name: cats + type: git + check_every: 24h + source: + uri: https://github.com/cloudfoundry/cf-acceptance-tests + tag_filter: "v*" diff --git a/ci/pipeline/resources/git-ci.yml b/ci/pipeline/resources/git-ci.yml new file mode 100644 index 00000000..bda160a5 --- /dev/null +++ b/ci/pipeline/resources/git-ci.yml @@ -0,0 +1,10 @@ +resources: + - name: git-ci + type: git + check_every: 1h + source: + uri: (( grab meta.github.ci-uri || meta.github.uri)) + branch: (( grab meta.github.ci-branch || meta.github.branch )) + private_key: (( grab meta.github.private_key )) + paths: [ci/*] + disable_ci_skip: true diff --git a/ci/pipeline/resources/git-latest-tag.yml b/ci/pipeline/resources/git-latest-tag.yml new file mode 100644 index 00000000..00ae196b --- /dev/null +++ b/ci/pipeline/resources/git-latest-tag.yml @@ -0,0 +1,10 @@ +resources: + - name: git-latest-tag + type: git + check_every: 1h + source: + uri: (( grab meta.github.uri )) + branch: (( grab meta.github.branch )) + private_key: (( grab meta.github.private_key )) + tag_filter: "v[1-9]*" + disable_ci_skip: true diff --git a/ci/pipeline/resources/git-main.yml b/ci/pipeline/resources/git-main.yml new file mode 100644 index 00000000..20c4a3fc --- /dev/null +++ b/ci/pipeline/resources/git-main.yml @@ -0,0 +1,8 @@ +resources: + - name: git-main + type: git + check_every: 1h + source: + uri: (( grab meta.github.uri )) + branch: (( grab meta.github.main-branch )) + private_key: (( grab meta.github.private_key )) diff --git a/ci/pipeline/resources/git.yml b/ci/pipeline/resources/git.yml new file mode 100644 index 00000000..4f2d8f91 --- /dev/null +++ b/ci/pipeline/resources/git.yml @@ -0,0 +1,13 @@ +resources: + - name: git + type: git + check_every: 1h + source: + uri: (( grab meta.github.uri )) + branch: (( grab meta.github.branch )) + private_key: (( grab meta.github.private_key )) + ignore_paths: ["ci"] + commit_filter: + exclude: + - '^\[ci\] release v' + - '^WIP:' diff --git a/ci/pipeline/resources/github-prerelease.yml b/ci/pipeline/resources/github-prerelease.yml new file mode 100644 index 00000000..96c9df51 --- /dev/null +++ b/ci/pipeline/resources/github-prerelease.yml @@ -0,0 +1,9 @@ +resources: + - name: github-prerelease + type: github-release + source: + user: (( grab meta.github.owner )) + repository: (( grab meta.github.repo )) + access_token: (( grab meta.github.access_token )) + prerelease: true + release: false diff --git a/ci/pipeline/resources/github.yml b/ci/pipeline/resources/github.yml new file mode 100644 index 00000000..ab5c11e7 --- /dev/null +++ b/ci/pipeline/resources/github.yml @@ -0,0 +1,7 @@ +resources: + - name: github + type: github-release + source: + user: (( grab meta.github.owner )) + repository: (( grab meta.github.repo )) + access_token: (( grab meta.github.access_token )) diff --git a/ci/pipeline/resources/notify.yml b/ci/pipeline/resources/notify.yml new file mode 100644 index 00000000..c31ab560 --- /dev/null +++ b/ci/pipeline/resources/notify.yml @@ -0,0 +1,15 @@ +resource_types: + - name: shout-notification + type: docker-image + source: + repository: huntprod/shout-resource + +resources: + - name: notify + type: shout-notification + source: + topic: (( grab meta.shout.topic )) + url: (( grab meta.shout.url )) + username: (( grab meta.shout.username )) + password: (( grab meta.shout.password )) + diff --git a/ci/pipeline/resources/release-notes.yml b/ci/pipeline/resources/release-notes.yml new file mode 100644 index 00000000..58a8435c --- /dev/null +++ b/ci/pipeline/resources/release-notes.yml @@ -0,0 +1,10 @@ +resources: + - name: release-notes + type: git + check_every: 1h + source: + uri: (( grab meta.github.release_notes.uri )) + branch: (( grab meta.github.release_notes.branch )) + private_key: (( grab meta.github.release_notes.private_key || meta.github.private_key )) + paths: [ (( grab meta.github.release_notes.file )) ] + disable_ci_skip: true diff --git a/ci/pipeline/resources/spec-check.yml b/ci/pipeline/resources/spec-check.yml new file mode 100644 index 00000000..1704c611 --- /dev/null +++ b/ci/pipeline/resources/spec-check.yml @@ -0,0 +1,9 @@ +resources: + - name: spec-check + type: s3 + source: + bucket: (( grab meta.aws.bucket )) + region_name: (( grab meta.aws.region_name )) + regexp: (( concat meta.name "/spec-check/diff-(.*)" )) + access_key_id: (( grab meta.aws.access_key )) + secret_access_key: (( grab meta.aws.secret_key )) diff --git a/ci/pipeline/resources/upstream.yml b/ci/pipeline/resources/upstream.yml new file mode 100644 index 00000000..315f599d --- /dev/null +++ b/ci/pipeline/resources/upstream.yml @@ -0,0 +1,8 @@ +resources: + - name: upstream + type: git + check_every: 168h + source: + uri: (( grab meta.upstream.url )) + branch: master + ignore_paths: [ci/*] diff --git a/ci/pipeline/resources/version.yml b/ci/pipeline/resources/version.yml new file mode 100644 index 00000000..f0a136fd --- /dev/null +++ b/ci/pipeline/resources/version.yml @@ -0,0 +1,11 @@ +resources: + - name: version + type: semver + source : + driver: s3 + bucket: (( grab meta.aws.bucket )) + region_name: (( grab meta.aws.region_name )) + key: (( concat meta.name "/version" )) + access_key_id: (( grab meta.aws.access_key )) + secret_access_key: (( grab meta.aws.secret_key )) + initial_version: (( grab meta.initial_version || "0.0.1" )) diff --git a/ci/repipe b/ci/repipe index eef0603e..4f000949 100755 --- a/ci/repipe +++ b/ci/repipe @@ -8,11 +8,13 @@ # author: James Hunt # Dennis Bell +set -ue + need_command() { local cmd=${1:?need_command() - no command name given} local url=${2:-} - if [[ ! -x "$(command -v $cmd)" ]]; then + if [[ ! -x "$(type -p "$cmd")" ]]; then echo >&2 "${cmd} is not installed." if [[ -n "$url" ]]; then echo >&2 "Please download it from ${url}" @@ -21,54 +23,220 @@ need_command() { fi } -cd $(dirname $BASH_SOURCE[0]) -echo "Working in $(pwd)" +error() { + echo >&2 "${1:-"Error\!"}" + exit "${2:-1}" +} + +run_cmd() { + if (( DRYRUN > 0 )) ; then + (( DRYRUN >= 2 )) && echo >&2 "$@" + else + set +e + "$@" + rc="$?" + set -e + return $rc + fi +} + +open_url() { + local url + url="$1"; + if [[ -x /usr/bin/open ]]; then + exec /usr/bin/open "$url" + else + echo "Sorry, but I was not able to automagically open" + echo "your Concourse Pipeline in the browser." + echo + echo "Here's a link you can click on, though:" + echo + echo " $url" + echo + exit 0; + fi +} + +preface_cmds() { + action="$1" + echo >&2 + if (( DRYRUN >= 2 )) ; then + [[ -n "$persistent_file" ]] \ + && echo >&2 "Run these commands to $action the pipeline:" \ + || echo >&2 "These commands would be run to $action the pipeline:" + fi +} + +usage() { + rc=0 + c="" + if [[ -z "${1:-""}" ]] ; then + msg="repipe - recreate the genesis kit release engineering pipeline" + else + msg="$1" + c=";31" + rc=1 + fi + \cat >&2 <] + +OPTIONS: + -v Validate pipeline instead of uploading it to Concourse. Specify + twice to validate with strict mode. + -y Treat all prompts with affirmative action automatically. + -n No change, just a dry-run that outputs to stdout. Specify twice + to print the commands that would be run to stderr. + -P Will pause the pipeline after uploading it (default is to unpause) + -H|-X Will hide or expose the pipeline to the public (default is based on + whatever is set in 'meta.exposed' in the settings.yml file) + -o Open pipeline in browser if os supports it (mac only currently) + after applying changes. Specify twice to not do anything else. + --fly Path to fly command, otherwise will use $(type -p fly) + +EOF + exit $rc +} + +# == MAIN ===================================================================== + +settings_file="settings.yml" +PAUSE='unpause' +EXPOSE='' +DRYRUN=0 +VALIDATE=0 +OPEN=0 +OPTCOUNT=0 +YES='' +fly='' +call_dir="$(pwd)" + +# -- Parse options ------------------------------------------------------------ +while [[ -n "${1:-""}" ]] ; do + arg="$1" ; shift + case "$arg" in + --fly) + fly="${1:-}" + if [[ -z "$fly" ]] ; then + error "No argument given to --fly" + elif [[ ! -x "$fly" ]] ; then + error "File '$fly' does not seem to be executable. Cannot use it as 'fly'" + fi + fly="$(realpath "$fly")" + shift 1 ;; + -*) for (( i=0; i<${#arg}; i++ )); do + (( OPTCOUNT++ )) + opt="${arg:$i:1}" + case "$opt" in + y) YES=1;; + v) (( VALIDATE++ ));; + n) (( DRYRUN++ ));; + P) PAUSE="pause" ;; + X) EXPOSE="expose" ;; + H) EXPOSE="hide" ;; + h) usage; exit 0;; + o) (( OPEN++ )) ;; + -) ;; # ignore -- TBD: if we ever need to support --, ... + *) usage "Unknown option '$opt'";; + esac + done;; + *) usage "$0 does not take any arguments";; + esac +done + +# -- Validate options --------------------------------------------------------- +if [[ $VALIDATE -ge 1 || $DRYRUN -ge 1 ]] ; then + if [[ "$DRYRUN" -eq 1 ]] && [[ -n $EXPOSE || $PAUSE == "pause" ]] ; then + echo >&2 "Warning: -y, -P, -X, and -E options are ignored if -v or -n is specified" + fi + if (( OPEN >= 2 && OPTCOUNT > OPEN )) ; then + error "-oo cannot be used with any other options (as it only opens the web page)" + fi +fi + +# -- Validate environment ----------------------------------------------------- +[[ -z "${GENESIS_CI_DIR:-""}" ]] && GENESIS_CI_DIR="$(dirname "${BASH_SOURCE[0]}")" +cd "$GENESIS_CI_DIR" +echo >&2 "Working in $(pwd)" need_command spruce "https://github.com/geofffranks/spruce/releases" need_command jq -# Allow for target-specific settings -settings_file="$(ls -1 settings.yml ${CONCOURSE_TARGET:+"settings-${CONCOURSE_TARGET}.yml"} 2>/dev/null | tail -n1)" -if [[ -z "$settings_file" ]]; then - echo >&2 "Missing local settings in ci/settings.yml${CONCOURSE_TARGET:+" or ci/settings-${CONCOURSE_TARGET}.yml"}!" - exit 1 +if [[ -z "$fly" ]] ; then + need_command fly; + fly="$(type -p fly)" fi +# -- Get settings file -------------------------------------------------------- +# Target-based settings file cannot have / or spaces in it, convert to - and _ respectively +target_settings_file="${CONCOURSE_TARGET:+"settings-$(sed -e 's#/#-#g' -e 's/ /_/g' <<<"${CONCOURSE_TARGET}").yml"}" +[[ -n $target_settings_file && -f $target_settings_file ]] && \ + settings_file="$target_settings_file" +[[ ! -f "$settings_file" ]] && \ + error "Missing local settings in ci/settings.yml${CONCOURSE_TARGET:+" or ci/${target_settings_file}"}!" echo >&2 "Using settings found in ${settings_file}" +# -- Extract pipeline target information and compare ------------------------- set -e trap "rm -f .deploy.yml" QUIT TERM EXIT INT -spruce merge pipeline/*.yml ${settings_file} > .deploy.yml -PIPELINE=$(spruce json .deploy.yml | jq -r '.meta.pipeline // ""') -if [[ -z ${PIPELINE} ]]; then - echo >&2 "Missing pipeline name in ci/settings.yml!" - exit 1 -fi +spruce merge --fallback-append pipeline/base.yml pipeline/*/*.yml "${settings_file}" > .deploy.yml +target=$(spruce json .deploy.yml | jq -r --arg t "${CONCOURSE_TARGET:-}" '.meta.target // $t') +[[ -n "${target:-}" ]] || error "Settings file missing meta.target value" +[[ -n "${CONCOURSE_TARGET:-}" && "$CONCOURSE_TARGET" != "$target" ]] && \ + error "Target '$target' from settings file conflicts with CONCOURSE_TARGET env var ('$CONCOURSE_TARGET')" + +pipeline_url="$(spruce json .deploy.yml | jq -r '.meta.url // ""')" +target_url="$(spruce json ~/.flyrc | jq -r --arg t "$target" '.targets[$t].api')" +[[ -n "${pipeline_url:-}" && "$pipeline_url" != "$target_url" ]] && \ + error "Target '$target' does not match the url specified in the pipeline: found '$target_url', got '$pipeline_url'" + +pipeline_team="$(spruce json .deploy.yml | jq -r '.meta.team // ""')" +target_team="$(spruce json ~/.flyrc | jq -r --arg t "$target" '.targets[$t].team')" +[[ -n "${pipeline_team:-}" && "$pipeline_team" != "$target_team" ]] && \ + error "Target '$target' does not match the team specified in the pipeline: found '$target_team', got '$pipeline_team'" + +pipeline=$(spruce json .deploy.yml | jq -r '.meta.pipeline // .meta.name') +[[ -z ${pipeline:-} ]] && error "Missing pipeline name in ci/settings.yml!" +pipeline_url="$target_url/teams/$target_team/pipelines/$pipeline" + -TARGET_FROM_SETTINGS=$(spruce json .deploy.yml | jq -r '.meta.target // ""') -if [[ -z ${CONCOURSE_TARGET} ]]; then - TARGET=${TARGET_FROM_SETTINGS} -elif [[ "$CONCOURSE_TARGET" != "$TARGET_FROM_SETTINGS" ]] -then - echo >&2 "Target in {$settings_file} differs from target in \$CONCOURSE_TARGET" - echo >&2 " \$CONCOURSE_TARGET: $CONCOURSE_TARGET" - echo >&2 " Target in file: $TARGET_FROM_SETTINGS" - exit 1 -else - TARGET=${CONCOURSE_TARGET} -fi -if [[ -z ${TARGET} ]]; then - echo >&2 "Missing Concourse Target in ci/settings.yml!" - exit 1 +# -- DO THE THINGS!!! --------------------------------------------------------- +if (( OPEN >= 2 )) ; then + open_url "$pipeline_url" + exit $? fi -EXPOSED=$(spruce json .deploy.yml | jq -r '.meta.exposed // false') -EXPOSE=expose -if [[ ${EXPOSED} == "false" ]]; then - EXPOSE=hide +persistent_file="" +if (( DRYRUN > 0 )) ; then + cat .deploy.yml + if [ ! -t 1 ] ; then + pid="$$" + persistent_file="$(/usr/sbin/lsof -p "$pid" -a -d 1 | tail -n 1 | awk '{print $NF}')" + if [[ -n "${persistent_file}" && -n "$(type -p realpath || true)" ]] ; then + persistent_file="$(realpath --relative-to="$call_dir" "$persistent_file")" + fi + fi fi -set +x -fly --target ${TARGET} set-pipeline --pipeline ${PIPELINE} --config .deploy.yml -fly --target ${TARGET} unpause-pipeline --pipeline ${PIPELINE} -fly --target ${TARGET} ${EXPOSE}-pipeline --pipeline ${PIPELINE} +declare -a cmd +cmd=( "$fly" --target "$target" ) +if (( VALIDATE > 0 )) ; then + cmd+=( validate-pipeline ) + (( VALIDATE >= 2 )) && cmd+=( --strict ) + cmd+=( --config "${persistent_file:-".deploy.yml"}" ) + preface_cmds validate + run_cmd "${cmd[@]}" + exit 0 +else + [[ -z "$EXPOSE" ]] && EXPOSE=$(spruce json .deploy.yml | jq -r '.meta.exposed // false | if . then "expose" else "hide" end') + set_cmd=( "${cmd[@]}" set-pipeline --pipeline "${pipeline}" --config "${persistent_file:-".deploy.yml"}" ) + [[ -n "$YES" ]] && set_cmd+=( --non-interactive ) + preface_cmds upload + run_cmd "${set_cmd[@]}" + run_cmd "${cmd[@]}" "${PAUSE}-pipeline" --pipeline "${pipeline}" + run_cmd "${cmd[@]}" "${EXPOSE}-pipeline" --pipeline "${pipeline}" +fi +echo +(( OPEN > 0 )) && open_url "$pipeline_url" diff --git a/ci/scripts/build-kit b/ci/scripts/build-kit new file mode 100755 index 00000000..e698a692 --- /dev/null +++ b/ci/scripts/build-kit @@ -0,0 +1,52 @@ +#!/bin/bash +set -eu + +# Resource Directories +export REPO_ROOT="git" +export BUILD_ROOT="build" +export CI_ROOT="git-ci" +export VERSION_FROM="version/number" + +header() { + echo + echo "================================================================================" + echo "$1" + echo "--------------------------------------------------------------------------------" + echo +} + +bail() { + echo >&2 "$* Did you misconfigure Concourse?" + exit 2 +} +test -n "${KIT_SHORTNAME:-}" || bail "KIT_SHORTNAME must be set to the short name of this kit." +test -n "${VAULT_URI:-}" || bail "VAULT_URI must be set to an address for connecting to Vault." +test -n "${VAULT_TOKEN:-}" || bail "VAULT_TOKEN must be set to something; it will be used for connecting to Vault." + +test -f "${VERSION_FROM}" || bail "Version file (${VERSION_FROM}) not found." +VERSION=$(cat "${VERSION_FROM}") +test -n "${VERSION}" || bail "Version file (${VERSION_FROM}) was empty." + +header "Connecting to vault..." +safe target da-vault "$VAULT_URI" -k +echo "$VAULT_TOKEN" | safe auth token +safe read secret/handshake + +header "Checking SHA1s of specified components (not including bosh-deployment) ..." +pushd "$REPO_ROOT" > /dev/null +out="$(eval "spruce merge --skip-eval $( \ + grep -rl '^releases:' overlay/\ + | sed -e "s/\\(.*\\)/<(spruce json \\1 | jq -r '{releases: .releases}')/" |tr "\n" " " \ + ) | spruce json | jq -r ." )" +popd > /dev/null +echo "$out" | spruce merge | spruce json | "${CI_ROOT}/ci/scripts/check-sha1s" + +header "Building $KIT_SHORTNAME kit v$VERSION" +genesis -C "$REPO_ROOT" compile-kit -v "$VERSION" -n "$KIT_SHORTNAME" + +mv "${REPO_ROOT}/${KIT_SHORTNAME}-${VERSION}.tar.gz" "$BUILD_ROOT/" + +echo +echo "================================================================================" +echo "SUCCESS!" +exit 0 diff --git a/ci/scripts/build-upstream-list b/ci/scripts/build-upstream-list deleted file mode 100755 index 047f2f00..00000000 --- a/ci/scripts/build-upstream-list +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -dir=$(dirname $0) -( -cd $dir/../.. && -out="$(eval "spruce merge --skip-eval $( \ - grep -rl '^releases:' manifests/ \ - | sed -e "s/\\(.*\\)/<(spruce json \\1 | jq -r '{releases: .releases}')/" |tr "\n" " " \ - ) | spruce json | jq -r ." )" - -echo "$out" \ - | jq -r 'reduce .releases[] as {$name, $url, $sha1, $version} ({core: [], buildpacks: []}; - ($url - | if ($url | test("https?://bosh.io")) then - ($url | sub("^.*/d/";"https://") | sub("\\?v=.*$";"/releases/tag/${version}")) - elif ($url | test("https?://github.com")) then - ($url | sub("^.*http";"http") | sub("/download/.*$";"/tag/${version}")) - else - $url - end - ) as $notes | - if ($name | test("-buildpack$")) then - (.buildpacks += [{name: ($name|sub("-buildpack";"")),$notes}]) - else - (.core += [{$name,$notes}]) - end - )' | spruce merge --skip-eval > ci/upstream.yml -) diff --git a/ci/scripts/cats b/ci/scripts/cats index 498be273..ad749f36 100755 --- a/ci/scripts/cats +++ b/ci/scripts/cats @@ -1,17 +1,50 @@ #!/bin/bash +set -e + +export GIT_NAME="${GIT_NAME:-"Stark & Wayne CI Bot"}" +export GIT_EMAIL="${GIT_EMAIL:-"ci@starkandwayne.com"}" + +header() { + echo + echo "================================================================================" + echo "$1" + echo "--------------------------------------------------------------------------------" + echo +} + +bail() { + echo >&2 "$* Did you misconfigure Concourse?" + exit 2 +} + + +test -n "${KIT_SHORTNAME:-}" || bail "KIT_SHORTNAME must be set to the short name of this kit." +test -n "${VAULT_URI:-}" || bail "VAULT_URI must be set to the Vault connection address - suggest using ((vault.url))" +test -n "${VAULT_TOKEN:-}" || bail "VAULT_TOKEN must be set to a valid token to connect to the Vault - suggest using ((vault.token))" +test -n "${BOSH_ENVIRONMENT:-}" || bail "BOSH_ENVIRONMENT must be set to BOSH directory URL - suggest using ((bosh.uri))" +test -n "${BOSH_CA_CERT:-}" || bail "BOSH_CA_CERT must be set to the BOSH CA Cert - suggest using ((bosh.ca))" +test -n "${BOSH_CLIENT:-}" || bail "BOSH_CLIENT must be set to BOSH Client user - suggest using ((bosh.username))" +test -n "${BOSH_CLIENT_SECRET:-}" || bail "BOSH_CLIENT_SECRET must be set to BOSH Client secret - suggest using ((bosh.password))" +test -n "${CREDHUB_URL:-}" || bail "CREDHUB_URL must be set to Credhub URL - suggest using ((bosh.credhub_url))" +test -n "${CREDHUB_USER:-}" || bail "CREDHUB_USER must be set to Credhub username - suggest using ((bosh.credhub_username))" +test -n "${CREDHUB_PASSWORD:-}" || bail "CREDHUB_PASSWORD must be set to Credhub password - suggest using ((bosh.credhub_password))" + +test -f "${VERSION_FROM}" || bail "Version file (${VERSION_FROM}) not found." +VERSION=$(cat "${VERSION_FROM}") +test -n "${VERSION}" || bail "Version file (${VERSION_FROM}) was empty." function lookup(){ genesis -C "$GENESIS_ROOT" lookup "$GENESIS_ENVIRONMENT" "$@" } function run_local(){ - export tmpdir=$(mktemp -d -t cats.XXXXXXXXXX) - trap "rm -rf ${tmpdir}" EXIT + export tmpdir="$(mktemp -d -t cats.XXXXXXXXXX)" + trap "rm -rf '${tmpdir}'" EXIT echo "REPO: $CATS_GIT_SSH_URL VERSION: $VERSION" - git clone ${CATS_GIT_SSH_URL} ${tmpdir} - pushd ${tmpdir} - git checkout ${VERSION} + git clone "${CATS_GIT_SSH_URL}" "${tmpdir}" + pushd "${tmpdir}" &>/dev/null + git checkout "${VERSION}" git submodule update --init - popd + popd &>/dev/null } function run_cats(){ case $1 in @@ -22,60 +55,72 @@ function run_cats(){ RUN_CATS="" ;; "--deployment-dir") - echo "FOUND DEPL. DIR FLAG" + echo "Running CATS from deployment dir $2" export BUILD_ROOT=$2 - export GENESIS_ROOT=$BUILD_ROOT/work/cf-deployments + export GENESIS_ROOT="$BUILD_ROOT/work/cf-deployments" export GENESIS_ENVIRONMENT=ci-cats export vault=/secret/ci/cats/cf export credhub=/snw-genesis-ci-bosh/ci-cats-cf - credhub login --server $CREDHUB_URL --username $CREDHUB_USER --password $CREDHUB_PASSWORD --skip-tls-validation + credhub login --server "$CREDHUB_URL" --username "$CREDHUB_USER" --password "$CREDHUB_PASSWORD" --skip-tls-validation ;; esac echo "$GENESIS_ROOT" base=$(lookup params.base_domain) - system_domain=$(lookup params.system_domain system.$base) + system_domain="$(lookup params.system_domain system.$base)" export API_URL=api.$system_domain - export ADMIN_PASSWORD=$(credhub get -q -n $credhub/cf_admin_password) + export ADMIN_PASSWORD="$(credhub get -q -n $credhub/cf_admin_password)" export APP_DOMAINS_0=run.$base case $RUN_CATS in "include_container_networking") - sed -i "s#include_security_groups:.*#include_security_groups: true#g" ../git-ci/ci/scripts/cats.yml + sed -i "s#include_security_groups:.*#include_security_groups: true#g" ../git-ci/ci/envs/cats.yml ;; "include_deployments") - sed -i "s#include_v3:.*#include_v3: true#g" ../git-ci/ci/scripts/cats.yml + sed -i "s#include_v3:.*#include_v3: true#g" ../git-ci/ci/envs/cats.yml ;; "include_service_instance_sharing") - sed -i "s#include_services:.*#include_services: true#g" ../git-ci/ci/scripts/cats.yml + sed -i "s#include_services:.*#include_services: true#g" ../git-ci/ci/envs/cats.yml ;; "include_sso") - sed -i "s#include_services:.*#include_services: true#g" ../git-ci/ci/scripts/cats.yml + sed -i "s#include_services:.*#include_services: true#g" ../git-ci/ci/envs/cats.yml ;; "include_tasks") - sed -i "s#include_v3:.*#include_v3: true#g" ../git-ci/ci/scripts/cats.yml + sed -i "s#include_v3:.*#include_v3: true#g" ../git-ci/ci/envs/cats.yml ;; "include_zipkin") - sed -i "s#include_routing:.*#include_routing: true#g" ../git-ci/ci/scripts/cats.yml + sed -i "s#include_routing:.*#include_routing: true#g" ../git-ci/ci/envs/cats.yml ;; esac if [[ ${RUN_LOCAL} ]]; then - echo "RUNNING LOCALLY" + echo "Running CATS locally..." run_local else - sed -i "s#${RUN_CATS}:.*#${RUN_CATS}: true#g" ../git-ci/ci/scripts/cats.yml + sed -i "s#${RUN_CATS}:.*#${RUN_CATS}: true#g" ../git-ci/ci/envs/cats.yml tmpdir="$BUILD_ROOT/cats" fi export CONFIG=${tmpdir}/config.json - spruce merge ../git-ci/ci/scripts/cats.yml | spruce json | jq '.' > ${CONFIG} - pushd ${tmpdir} - FAILFAST_FLAG="" - VERBOSE_FLAG="" - if [[ ${FAILFAST}=="true" ]];then - FAILFAST_FLAG="--failFast" - fi - if [[ ${VERBOSE}=="true" ]];then - VERBOSE_FLAG="-v" - fi - echo "RUNNING CATS WITH NODES: ${CATS_NODES:-4} ${VERBOSE_FLAG} ${FAILFAST_FLAG}" - ./bin/test -nodes=${CATS_NODES:-4} ${VERBOSE_FLAG} $FAILFAST_FLAG -} \ No newline at end of file + spruce merge ../git-ci/ci/envs/cats.yml | spruce json | jq '.' > "${CONFIG}" + pushd "${tmpdir}" &>/dev/null + FAILFAST_FLAG="" + VERBOSE_FLAG="" + if [[ ${FAILFAST} == "true" ]] ; then + FAILFAST_FLAG="--failFast" + fi + if [[ ${VERBOSE} == "true" ]] ; then + VERBOSE_FLAG="-v" + fi + echo "RUNNING CATS WITH NODES: ${CATS_NODES:-4} ${VERBOSE_FLAG} ${FAILFAST_FLAG}" + ./bin/test -nodes=${CATS_NODES:-4} ${VERBOSE_FLAG} ${FAILFAST_FLAG} + popd &>/dev/null +} + +ROOT_DIR="$(pwd)" + +header "Configure Github and Vault access" +git config --global user.name "$GIT_NAME" +git config --global user.email "$GIT_EMAIL" +safe target da-vault "$VAULT_URI" -k +echo "$VAULT_TOKEN" | safe auth token +safe read secret/handshake + +run_cats --deployment-dir "$ROOT_DIR" diff --git a/ci/scripts/check-sha1s b/ci/scripts/check-sha1s new file mode 100755 index 00000000..4aa2ac98 --- /dev/null +++ b/ci/scripts/check-sha1s @@ -0,0 +1,21 @@ +#!/usr/bin/perl +use strict; +use warnings; +use JSON::PP qw/decode_json/; + +my $rc = 0; +my $data = decode_json(do { local $/; <> }); +for my $r (@{$data->{releases} || []}) { + chomp(my $sha1 = `curl -Lsk "$r->{url}" | sha1sum`); + $sha1 =~ s/ .*//; + + if ($r->{sha1} eq $sha1) { + print "[ok] $r->{name} sha1 checkums are correct.\n"; + } else { + print "[!!] $r->{name} sha1 checkums are INCORRECT.\n"; + print "[!!] the kit has '$r->{sha1}' (wrong)\n"; + print "[!!] actually got '$sha1' (correct)\n"; + $rc = 1; + } +} +exit $rc; diff --git a/ci/tasks/spec-check/spec-check b/ci/scripts/compare-release-specs similarity index 82% rename from ci/tasks/spec-check/spec-check rename to ci/scripts/compare-release-specs index eef98c35..1ba877df 100755 --- a/ci/tasks/spec-check/spec-check +++ b/ci/scripts/compare-release-specs @@ -1,43 +1,42 @@ #!/bin/bash - set -ue # What branch is the comparison of the current working branch being compared against -compare_branch="${1:-origin/master}" -check_dir="spec" +compare_branch="${1:-origin/}" +check_dirs="spec manifests manifests/releases" orig_dir="$(pwd)" # needed because of when running locally or in ci -if [ -d "${orig_dir}-ci" ]; then - ci_dir="${orig_dir}-ci" +ci_dir="../$(basename "$(pwd)")-ci" +if [[ -d "$ci_dir" ]]; then + ci_dir="$(cd "$ci_dir" && pwd)" else - ci_dir=${orig_dir} + ci_dir="$orig_dir" fi -while [[ ! -d "$check_dir/" && ! -d .git && "$(pwd)" != "/" ]] ; do - cd .. -done -if [[ "$(pwd)" == "/" ]] ; then - echo >&2 "[ERROR] No $check_dir directory found in root of git repo" - exit 1 -fi +release_files() { + for d in $check_dirs; do + grep -rl '^releases' "$d" 2>/dev/null || true + done +} + +releases() { + eval "spruce merge --skip-eval $( \ + release_files \ + | sed -e "s/\\(.*\\)/\<(spruce json \\1 | jq -r '{releases: .releases}')/" |tr "\n" " " \ + ) | spruce merge | spruce json | jq -r ." +} workdir="$(mktemp -d)" mkdir "$workdir/compare" cp -R "$(pwd)/.git" "$workdir/compare/" pushd "$workdir/compare" > /dev/null git checkout -qf --detach "$compare_branch" -prev_releases="$(eval "spruce merge --skip-eval $( \ - grep -rl '^releases:' "$check_dir/" \ - | sed -e "s/\\(.*\\)/<(spruce json \\1 | jq -r '{releases: .releases}')/" |tr "\n" " " \ - ) | spruce merge | spruce json | jq -r ." )" +prev_releases="$(releases)" popd > /dev/null rm -rf "$workdir/compare" -curr_releases="$(eval "spruce merge --skip-eval $( \ - grep -rl '^releases:' "$check_dir/" \ - | sed -e "s/\\(.*\\)/<(spruce json \\1 | jq -r '{releases: .releases}')/" |tr "\n" " " \ - ) | spruce merge | spruce json | jq -r ." )" +curr_releases="$(releases)" prev_rel_names="$(echo "$prev_releases"| jq -r '.releases[] | .name' | sort)" @@ -64,6 +63,7 @@ while IFS='' read -r rel; do fi done <<< "${curr_rel_names[@]}" +# Write diff outputs if [[ "${#removed[@]}" -gt 0 && -n "${removed[0]}" ]] ; then echo "Removed Releases:" for rel in "${removed[@]}" ; do @@ -99,14 +99,14 @@ if [[ "${#changed[@]}" -gt 0 && -n "${changed[0]}" ]] ; then echo "Fetching spec diffs..." if [ -f "${ci_dir}/ci/upstreamrepo.yml" ]; then - upstreamrepo=$(spruce json ${ci_dir}/ci/upstreamrepo.yml) + upstreamrepo=$(spruce json "${ci_dir}/ci/upstreamrepo.yml") else upstreamrepo="[]" fi repos="$( echo "$curr_releases" \ | jq --argjson gitrepos "$upstreamrepo" -r 'reduce .releases[] as {$name, $url, $sha1, $version} ({repos: []}; - ($url + ($url | if ($url | test("https?://s3.amazonaws.com")) then ($gitrepos.repos | map(select(.name == $name))[0].repo) elif ($url | test("https?://bosh.io")) then @@ -119,7 +119,7 @@ if [[ "${#changed[@]}" -gt 0 && -n "${changed[0]}" ]] ; then ) as $repo | (.repos += [{$name,$repo}]) )')" - + for info in "${changed[@]}" ; do read -r rel prev_ver curr_ver <<<"$info" repo="$(echo "$repos" | jq -r --arg r "$rel" '.repos[] | select(.name == $r) | .repo' )" @@ -130,16 +130,16 @@ if [[ "${#changed[@]}" -gt 0 && -n "${changed[0]}" ]] ; then git -C "$rel_dir" config core.sparseCheckout true >/dev/null 2>&1 echo "/jobs" > "$rel_dir/.git/info/sparse-checkout" - + set +e - reponame=$(echo ${repo} | grep github | cut -d "/" -f4-5) + reponame=$(echo "${repo}" | grep github | cut -d "/" -f4-5) set -e if [[ -n ${reponame} ]]; then - branch=$(curl --silent -L -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/${reponame} | jq -r .default_branch) + branch=$(curl --silent -L -H "Accept: application/vnd.github.v3+json" "https://api.github.com/repos/${reponame}" | jq -r .default_branch) else branch="master" fi - + git -C "$rel_dir" pull --depth 1 origin ${branch} > /dev/null 2>&1 @@ -154,7 +154,7 @@ if [[ "${#changed[@]}" -gt 0 && -n "${changed[0]}" ]] ; then [[ -d "$workdir/compare-specs/$x" ]] || mkdir -p "$workdir/compare-specs/$x" [[ -f "$x/spec" ]] && cp "$x/spec" "$workdir/compare-specs/$x/spec-$v" done) > /dev/null 2>&1 - else + else echo "[ERROR] Cannot find version $v of release $rel" fi done @@ -183,5 +183,4 @@ else echo "No Spec Changes to Consider" echo fi -[[ -n "$workdir" ]] && echo rm -rf "$workdir" -cd "$orig_dir" +[[ -n "$workdir" ]] && rm -rf "$workdir" diff --git a/ci/scripts/deploy b/ci/scripts/deploy new file mode 100755 index 00000000..ace2e8ce --- /dev/null +++ b/ci/scripts/deploy @@ -0,0 +1,96 @@ +#!/bin/bash +set -eu + +# Resource Directories +export CI_ROOT="git-ci" +export DEPLOY_ENV="${DEPLOY_ENV:-"ci-baseline"}" +export KEEP_STATE="${KEEP_STATE:-"false"}" +export VERSION_FROM="version/number" +export GIT_NAME="${GIT_NAME:-"Stark & Wayne CI Bot"}" +export GIT_EMAIL="${GIT_EMAIL:-"ci@starkandwayne.com"}" + +header() { + echo + echo "================================================================================" + echo "$1" + echo "--------------------------------------------------------------------------------" + echo +} + +bail() { + echo >&2 "$* Did you misconfigure Concourse?" + exit 2 +} +test -n "${KIT_SHORTNAME:-}" || bail "KIT_SHORTNAME must be set to the short name of this kit." +test -n "${VAULT_URI:-}" || bail "VAULT_URI must be set to an address for connecting to Vault." +test -n "${VAULT_TOKEN:-}" || bail "VAULT_TOKEN must be set to something; it will be used for connecting to Vault." + +[[ -n "${TAG_ROOT:-}" && -n "${BUILD_ROOT:-}" ]] && bail "Cannot specify both 'TAG_ROOT' and 'BUILD_ROOT'" +[[ -z "${TAG_ROOT:-}" && -z "${BUILD_ROOT:-}" ]] && bail "Must specify one of 'TAG_ROOT' or 'BUILD_ROOT'" + +WORKDIR="work/${KIT_SHORTNAME}-deployments" +VERSION= +KIT= +if [[ -n "${TAG_ROOT:-}" ]]; then + test -f "${TAG_ROOT}/.git/ref" || bail "Version reference for $TAG_ROOT repo not found." + VERSION="$(sed -e 's/^v//' < "${TAG_ROOT}/.git/ref")" + re='^[0-9]+\.[0-9]+\.[0-9]+' + [[ "${VERSION}" =~ $re ]] || bail "Version reference for $TAG_ROOT repo was not a semver value." + KIT="$KIT_SHORTNAME/$VERSION" +else + test -f "${VERSION_FROM}" || bail "Version file (${VERSION_FROM}) not found." + VERSION=$(cat "${VERSION_FROM}") + test -n "${VERSION}" || bail "Version file (${VERSION_FROM}) was empty." + KIT="$(cd "$BUILD_ROOT" && pwd)/${KIT_SHORTNAME}-${VERSION}.tar.gz" +fi +header "Setting up git..." +git config --global user.name "$GIT_NAME" +git config --global user.email "$GIT_EMAIL" + +header "Connecting to vault..." +safe target da-vault "$VAULT_URI" -k +echo "$VAULT_TOKEN" | safe auth token +safe read secret/handshake + +if [[ "${KEEP_STATE}" == "true" && -d "${WORKDIR}" ]] ; then + header "Updating Genesis deployment directory for $KIT_SHORTNAME v$VERSION..." + genesis -v + if [[ -n "${TAG_ROOT:-}" ]] ; then + genesis -C "${WORKDIR}" fetch-kit "${KIT}" + else + cp -av "$KIT" "${WORKDIR}/.genesis/kits/" + fi +else + header "Setting up Genesis deployment directory for $KIT_SHORTNAME v$VERSION..." + rm -rf work/*; mkdir -p work/ + genesis -v + genesis -C work/ init -k "$KIT" --vault da-vault +fi + +header "Copying test environment YAMLs from $CI_ROOT/ci/envs..." +CI_PATH="$(cd "${CI_ROOT}" && pwd)" +cp -av "$CI_PATH"/ci/envs/*.yml "${WORKDIR}/" +test -f "${WORKDIR}/${DEPLOY_ENV}.yml" || \ + bail "Environment $DEPLOY_ENV.yml was not found in the $CI_ROOT ci/envs/ directory" + +target="$(cat < "${WORKDIR}/ci.yml" +cat "${WORKDIR}/ci.yml" + +export PATH="$PATH:$CI_PATH/ci/scripts" + +echo $'\n'"Handing off to ${CI_ROOT}/ci/test-deployment..." +cd "${WORKDIR}" +BOSH=bosh "$CI_PATH/ci/scripts/test-deployment" + +echo +echo "SUCCESS" +exit 0 diff --git a/ci/scripts/generate-release-notes b/ci/scripts/generate-release-notes new file mode 100755 index 00000000..7e8e89b8 --- /dev/null +++ b/ci/scripts/generate-release-notes @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -e +set -o pipefail + +export VERSION_FROM="version/number" +export GIT_NAME="${GIT_NAME:-"Stark & Wayne CI Bot"}" +export GIT_EMAIL="${GIT_EMAIL:-"ci@starkandwayne.com"}" + +header() { + echo + echo "================================================================================" + echo "$1" + echo "--------------------------------------------------------------------------------" + echo +} + +bail() { + echo >&2 "$* Did you misconfigure Concourse?" + exit 2 +} +test -n "${KIT_SHORTNAME:-}" || bail "KIT_SHORTNAME must be set to the short name of this kit." +test -n "${RELEASE_NOTES_FILE:-}" || bail "RELEASE_NOTES_FILE must be set to the filename for the release notes." +test -n "${RELEASE_NOTES_WEB_URL:-}" || bail "RELEASE_NOTES_WEB_URL must be set to the release notes gist edit URL." + +test -f "${VERSION_FROM}" || bail "Version file (${VERSION_FROM}) not found." +VERSION=$(cat "${VERSION_FROM}") +test -n "${VERSION}" || bail "Version file (${VERSION_FROM}) was empty." + +git-ci/ci/scripts/release-notes "$VERSION" "git" "git-latest-tag" "release-notes/$RELEASE_NOTES_FILE" +cat "release-notes/$RELEASE_NOTES_FILE" + +header "Uploading the release notes" + +git config --global user.name "$GIT_NAME" +git config --global user.email "$GIT_EMAIL" + +git -C release-notes add "$RELEASE_NOTES_FILE" +git -C release-notes commit -m "Updated release notes for $KIT_SHORTNAME-genesis-kit v$VERSION" + +echo $'\n'"The release notes can be edited at ${RELEASE_NOTES_WEB_URL}" diff --git a/ci/scripts/get-latest-upstream b/ci/scripts/get-latest-upstream new file mode 100755 index 00000000..426369a9 --- /dev/null +++ b/ci/scripts/get-latest-upstream @@ -0,0 +1,114 @@ +#!/usr/bin/env bash + +set -e +set -o pipefail +export TERM=xterm-256color + +header() { + echo + echo "================================================================================" + echo "$1" + echo "--------------------------------------------------------------------------------" + echo +} + +bail() { + echo >&2 "[ERROR] $* Did you misconfigure Concourse?" + exit 2 +} + +test -n "${GIT_EMAIL:-}" || bail "GIT_EMAIL must be set to an email address to use for Git commits." +test -n "${GIT_NAME:-}" || bail "GIT_NAME must be set to something; it will be used for Git commits." + +# Configuration Section for different upstreams + +# Reference: commit of version based +ref=$(cat upstream/.git/ref) # commit-based ref +upstream_url="https://github.com/${UPSTREAM_REPO:?UPSTREAM_REPO not exported}/tree/${ref}" #commit-based ref +if [[ $ref =~ ^v ]] ; then + ref=$(cut -b 2- <<< "$ref") # version-based ref + upstream_url="https://github.com/${UPSTREAM_REPO:?UPSTREAM_REPO not exported}/releases/tag/v${ref}" #version-based ref +fi + +# Package paths +src_path="upstream" +dst_path="git/${UPSTREAM_PATH}" #relative to the git path + +# override if you need specific files (whitelist) +files=(); while IFS= read -r line ; do + files+=( "$line" ) +done < <(command ls -A1 "$src_path" | grep -v .git | grep -v '^tests$' | grep -v '^ci$' ) +# override if you don't want specific files (blacklist) +remove=( ) + +echo "CHECKING git status" +git -C git status + +header "Bumping embedded $UPSTREAM_PATH to $ref" + +[[ -d "$dst_path" ]] || bail "Could not find upstream directory '$UPSTREAM_PATH' in the git repo" + +echo "Clearing out existing files in $dst_path" +rm -rf "${dst_path:?Destination path not set}" +mkdir "$dst_path" + +echo +echo "Copying files from upstream repository:" +for f in ${files[@]+"${files[@]}"} ; do + cp -Rv "$src_path/$f" "${dst_path}/$f" +done + +if [[ ${#remove[@]} -gt 0 ]] ; then + echo + echo "Removing unneeded files:" + for f in ${remove[@]+"${remove[@]}"} ; do + rm -rfv "${dst_path:?Destination path not set}/$f" + done +fi + +echo +echo "Comparing changes from upstream:" +if [[ -n "$(git -C git status --porcelain)" ]] ; then + ts="$(date -u +"%Y-%b-%d %H:%M:%S %Z")" + cat > git/overlay/upstream_version.yml < /dev/null + # TODO: remove spec/{credhub,vault} if needed? because this will regenerate vault. + rm -rf results/ + ACK_GINKGO_RC=true ginkgo -p +popd + +header "Spec file changes:" +git -C git diff --color=always spec/results/ | cat + +header "Commiting updates to git" +git config --global user.name "${GIT_NAME}" +git config --global user.email "${GIT_EMAIL}" + +pushd git &>/dev/null +git add "$UPSTREAM_PATH" +git add spec/ +git add overlay/upstream_version.yml +git commit -m "Update to $UPSTREAM_REPO@$ref" + +# The following is done to ensure a clean push to the develop branch, while +# basing the input on a version that last passed the spec-tests. +https_url="$(git remote -v | grep '(fetch)' | sed -e 's/.*github.com:\(.*\) (fetch)/https:\/\/github.com\/\1/')" +git remote add live "$https_url" +git pull --rebase=merges live develop -X theirs --no-edit +git remote remove live + +popd &>/dev/null diff --git a/ci/scripts/pull-upstream b/ci/scripts/pull-upstream index 1a09fa88..ae4933f2 100755 --- a/ci/scripts/pull-upstream +++ b/ci/scripts/pull-upstream @@ -10,7 +10,7 @@ version="${1:-}" [[ -n "$version" ]] || error "Usage: $0 " #Config -upstream_name="cf-deployment" +upstream_name="${UPSTREAM_PATH:-cf-deployment}" upstream_path="https://github.com/cloudfoundry/${upstream_name}/archive/v${version}.tar.gz" upstream_url="https://github.com/cloudfoundry/${upstream_name}/releases/tag/v${version}" upstream_file="${upstream_name}-${version}.tar.gz" diff --git a/ci/scripts/release b/ci/scripts/release new file mode 100755 index 00000000..c7c9f6c1 --- /dev/null +++ b/ci/scripts/release @@ -0,0 +1,113 @@ +#!/bin/bash +# +# ci/scripts/shipit +# +# Script for generating Github release / tag assets +# and managing release notes for a BOSH Release pipeline +# +# author: James Hunt +# created: 2016-03-30 + +set -eu + +header() { + echo + echo "================================================================================" + echo "$1" + echo "--------------------------------------------------------------------------------" + echo +} + +bail() { + echo >&2 "$* Did you misconfigure Concourse?" + exit 2 +} +export REPO_ROOT="git" +export BUILD_ROOT="build" +export RELEASE_NOTES_ROOT="release-notes" +export VERSION_FROM="version/number" +[[ "${PRERELEASE:-0}" =~ (0|f|false|n|no) ]] && PRERELEASE="" + + +test -f "${VERSION_FROM}" || bail "Version file (${VERSION_FROM}) not found." +VERSION=$(cat ${VERSION_FROM}) +test -n "${VERSION}" || bail "Version file (${VERSION_FROM}) was empty." + +test -n "${DEVELOP_BRANCH:-}" || bail "DEVELOP_BRANCH must be set to the development Git repository branch." +test -n "${RELEASE_BRANCH:-}" || bail "RELEASE_BRANCH must be set to the main Git repository branch." + +if [[ -n "${PRERELEASE}" ]] ; then + RELEASE_NOTES_PATH="pre-release-notes" + cat < "$RELEASE_NOTES_PATH" + +This is a prerelease - please see commit messages for changes +EOF +else + test -n "${GIT_EMAIL:-}" || bail "GIT_EMAIL must be set to an email address to use for Git commits." + test -n "${GIT_NAME:-}" || bail "GIT_NAME must be set to something; it will be used for Git commits." + test -n "${RELEASE_ROOT:-}" || bail "RELEASE_ROOT must be set to the output directory where release artifacts should go." + test -n "${RELEASE_NOTES:-}" || bail "RELEASE_NOTES must be set to the filename of the release notes." + RELEASE_NOTES_PATH="${RELEASE_NOTES_ROOT}/${RELEASE_NOTES}" + test -f "${RELEASE_NOTES_PATH}" || \ + bail "Release notes file (${RELEASE_NOTES_PATH}) not found." +fi + +test -n "${KIT_SHORTNAME:-}" || bail "KIT_SHORTNAME must be set to the short name of this kit." +test -n "${GITHUB_OWNER:-}" || bail "GITHUB_OWNER must be set to the name of the Github user or organization that owns the Git repository." +echo "Environment OK" + +############################################################### +header "Assembling Github Release Artifacts..." +mkdir -p "${RELEASE_ROOT}/artifacts" +echo "v${VERSION} -> ${RELEASE_ROOT}/tag" +echo "v${VERSION}" > "${RELEASE_ROOT}/tag" +echo "v${VERSION} -> ${RELEASE_ROOT}/name" +echo "v${VERSION}" > "${RELEASE_ROOT}/name" +cp -av "${BUILD_ROOT}"/*.tar.gz "${RELEASE_ROOT}/artifacts" +if [[ -d spec-check ]] ; then + if ! grep "No Spec Changes to Consider" spec-check/diff-* ; then + echo spec-check/diff-* "-> ${RELEASE_ROOT}/artifacts/spec-diffs.html" + cat spec-check/diff-* | aha > "${RELEASE_ROOT}/artifacts/spec-diffs.html" + else + echo "spec-check -> no changes in specs to release" + fi +fi + +header "Release Notes for v${VERSION}" +header="$(sed -i -e '1{w /dev/stdout' -e 'd}' "${RELEASE_NOTES_PATH}")" +re="^<\!--- Release Notes for v${VERSION} -- Do not move --->$" +if [[ "$header" =~ $re ]] ; then + sed -i '/^---8<--- This line and everything below will be ignored ---8<---$/,$d' "${RELEASE_NOTES_PATH}" + cat "${RELEASE_NOTES_PATH}" + cp "${RELEASE_NOTES_PATH}" "$RELEASE_ROOT/notes.md" +else + echo "Failed to find release notes for v$VERSION in $RELEASE_NOTES_PATH. Found this instead:" + echo + echo "$header" + cat "${RELEASE_NOTES_PATH}" + exit 1 +fi + +if [[ -z "${PRERELEASE}" && "$DEVELOP_BRANCH" != "$RELEASE_BRANCH" ]] ; then + header "Fast-forward merge develop into ${RELEASE_BRANCH}" + pushd git-main &>/dev/null + git config --global user.name "${GIT_NAME}" + git config --global user.email "${GIT_EMAIL}" + if ! git pull ../git -X theirs --no-edit --ff-only ; then + # if this fails, manual intervention is required. + echo >&2 \ + $'\n'"'$RELEASE_BRANCH' release branch contains commits that the '$DEVELOP_BRANCH' does not have" \ + $'\n'"Cannot push changes to release branch..." + exit 1 + fi + popd &>/dev/null +fi + +cat > "${NOTIFICATION_OUT:-notifications}/message" <. +EOS + +echo +echo "--------------------------------------------------------------------------------" +echo "SUCCESS" +exit 0 diff --git a/ci/scripts/release-notes b/ci/scripts/release-notes index a8559d8a..ab4581a6 100755 --- a/ci/scripts/release-notes +++ b/ci/scripts/release-notes @@ -1,116 +1,323 @@ -#!/usr/bin/perl +#!/usr/bin/env perl use warnings; use strict; -use YAML qw(LoadFile); +use YAML qw(Load LoadFile); use JSON::PP qw(decode_json); use File::Find; -use Data::Dumper; +use FindBin; -sub print_table { - my ($title, $table) = @_; - my $width = {label => 7, version => 7, dated => 12}; - for my $row (@$table) { - $width->{$_} = (sort{$b <=> $a} (length($row->{$_}), $width->{$_}))[0] for (keys %$row); - } - print "\n\n"; - print "# $title\n\n"; - printf "| %-*s | %-*s | %-*s |\n", $width->{label}, "Release", $width->{version}, "Version", $width->{dated}, "Release Date"; - printf "| %s | %s | %s |\n", "-" x $width->{label}, "-" x $width->{version}, "-" x $width->{dated}; - for my $row (@$table) { - printf "| %-*s | %-*s | %-*s |\n", $width->{label}, $row->{label}, $width->{version}, $row->{version}, $width->{dated}, $row->{dated}; - } +sub header { + print STDERR "\n================================================================================\n"; + printf STDERR "%s", join(' ',@_); + print STDERR "\n--------------------------------------------------------------------------------\n\n"; } -my (%CORE, %BUILDPACKS); -if (!$ENV{GITHUB_ACCESS_TOKEN}) { - print STDERR "Please set \$GITHUB_ACCESS_TOKEN in your environment...\n"; - exit 1; +sub bail { + print "\n"; + printf @_; + print STDERR "Did you misconfigure Concourse?\n\n"; + exit 2; } -my @months = qw(ignored January February March April May June July August September October November December); -sub github { - my $url = shift; - my ($protocol,$org, $repo, $tag) = ($url =~ m{^(.*?)github.com/(.*?)/(.*?)/releases/tag/(.*?)$}); - - my $out = qx(curl -Ls -u "$ENV{GITHUB_ACCESS_TOKEN}:" https://api.github.com/repos/$org/$repo/releases/tags/$tag); - my $data = decode_json($out); - if (defined($data->{message}) && $data->{message} eq "Not Found") { - $out = qx(curl -Ls -u "$ENV{GITHUB_ACCESS_TOKEN}:" https://api.github.com/repos/$org/$repo/releases/tags/v$tag); - $data = decode_json($out); - $url = "${protocol}github.com/$org/$repo/releases/tag/v$tag"; +sub uniq { + my %items; + $items{$_} = 1 for (@_); + sort keys %items; +} + +sub get_new_commits { + my ($path, $since_commit) = @_; + my %lookup = ( + author => '%aN', + author_email => '%aE', + author_date => '%at', + author_since => '%ar', + committer => '%cN', + committer_email => '%cE', + commit_date => '%ct', + commit_since => '%cr', + body => '%b' + ); + my @commits = map { + my ($c,@s) = split(' '); + {commit => $c, subject => join(' ', @s)} + } qx(git -C "$path" log --reverse $since_commit...HEAD --pretty=format:'%H %s'); + + print STDERR " - fetching commit details"; + for my $commit (@commits) { + print STDERR '.'; + for my $detail (keys %lookup) { + $commit->{$detail} = join("", qx(git -C "$path" log -n1 --pretty=format:'$lookup{$detail}' $commit->{commit})); + chomp $commit->{$detail}; + } } + print STDERR "\n"; + return @commits; +} - return ($url, "-") unless $data->{published_at}; - my ($year, $mon, $day) = ($data->{published_at} =~ m/^(\d{4})-(\d{2})-(\d{2})/); +sub get_latest_commit_ref { + my ($path) = @_; + chomp(my $ref = qx(git -C $path rev-parse HEAD)); + $ref +} - return ($url,"$day $months[$mon] $year"); +sub get_latest_commit_tag { + my ($path) = @_; + my $tag; + if (-f "$path/.git/ref") { + $tag = qx(cat "$path/.git/ref"); + } else { + $tag = qx(git -C $path describe --tag --abbrev=0); + } + chomp $tag; + return $tag } -my %versions; -my %found; -find(sub { - return unless m/\.yml$/; - printf STDERR "reading $File::Find::name ...\n"; - my $data = LoadFile($_); - if ($data->{releases}) { - for my $r (@{$data->{releases}}) { - printf STDERR " found $r->{name} v$r->{version}\n"; - $versions{$r->{name}} = $r->{version}; - $found{$r->{name}} = 1; +sub parse_commits { + my ($path, $last_release_path) = @_; + my @commits = get_new_commits($path, get_latest_commit_ref($last_release_path)); + + # Remove all merges with no bodies + @commits = grep {$_->{body} || $_->{subject} !~ /^Merge .* into .*/} @commits; + + my $last_update; + my $updates=0; + unless ($ENV{NO_UPSTREAM_SYNC}) { + print STDERR " - determining upstream synchronization\n"; + @commits = grep { + if ($_->{subject} =~ /^Update to ([^\/]*)\/(.*)@(.*)$/) { + $last_update = $_; $updates++; 0; + } else { + 1; } + } @commits; + } + + my %messages; + if ($last_update) { + $last_update->{subject} =~ /Update to ([^\/]*)\/(.*)@(.*)$/; + $messages{'Upstream Convergence'} = [ + sprintf( + '* Synchronized embedded %s to [@%s](https://github.com/%s/%s/tree/%s)', + $2, $3, $1, $2, $3 + ) + ]; + } + print STDERR " - parsing commits for release notes."; + my $re=qr/(?:(?:\A|\n\n)\[([^\]\n]*)\]\n\n)/; + my ($header,$note,@data); + for my $commit (@commits) { + $commit->{body} =~ s/\r//g; # Remove windows CR characters + (undef, @data) = split($re, $commit->{body}||''); + print STDERR (@data ? "!" : "."); + while (@data) { + $commit->{used} = 1; + ($header, $note, @data) = @data; + # Notes can be stopped with a --- on a new line + ($note) = split("\n+---",$note); + if ($note !~ /\* /) { + # Auto-bullet + $note = "* ".join("\n ",split("\n", $note)); + } + $messages{$header} ||= []; + push @{$messages{$header}}, $note } - }, "manifests"); - -printf STDERR "checking ci/upstream.yml for more details...\n"; -my $upstream = LoadFile("ci/upstream.yml"); -for my $c (@{$upstream->{core}}) { - delete $found{$c->{name}}; - die "core component '$c->{name}' not found in manifest 'releases:' stanza...\n" - unless $versions{$c->{name}}; + } + print STDERR "\n"; + my @entries; + push(@entries, "# $_\n\n".join("\n\n",@{$messages{$_}})) for (sort keys %messages); + my $release_notes = join("\n\n", @entries); + return ($release_notes, \@commits); # TBD: Maybe filter out commits that contained release notes } -if (exists $upstream->{buildpacks}) { - for my $bp (@{$upstream->{buildpacks}}) { - delete $found{$bp->{name}.'-buildpack'}; - die "buildpack '$bp->{name}' not found in manifest 'releases:' stanza...\n" - unless $versions{$bp->{name}.'-buildpack'}; +sub get_releases { + my ($path) = @_; + my $filter = "$path/spec/results/*.yml"; + my @entries = qx(for x in $filter ; do spruce json \$x | jq -r '.releases[] | "\\(.name) \\(.version) \\(.url)"' 2>/dev/null ; done | sort | uniq); + my %releases; + for my $entry (@entries) { + my ($release, $version, $url) = split(' ',$entry); + push( @{$releases{$release}{$version} ||= []}, $url); } + return \%releases; } -if (%found) { - for (sort keys %found) { - printf STDERR "no ci/upstream.yml entry found for '$_'\n"; +my @months = qw(ignored January February March April May June July August September October November December); +sub github { + my ($org, $repo, $tag, $type) = @_; + my ($url, $day, $mon, $year); + if ($type eq 'release') { + printf STDERR " - $org/$repo release $tag\n"; + my $lookup_url="https://api.github.com/repos/$org/$repo/releases"; + my $out = qx(curl -Ls -u "$ENV{GITHUB_ACCESS_TOKEN}:" "$lookup_url" | jq -r --arg t "$tag" '.[] | select(.tag_name == \$t)'); + return ("Tag $tag not found",'') unless $out; + my $data = decode_json($out); + $url = $data->{html_url} || ''; + return ('-', $url) unless $data->{published_at}; + ($year, $mon, $day) = ($data->{published_at} =~ m/^(\d{4})-(\d{2})-(\d{2})/); + } elsif ($type eq 'commit') { + printf STDERR " - $org/$repo tag $tag\n"; + my $lookup_url="https://api.github.com/repos/$org/$repo/tags"; + my $out = qx(curl -Ls -u "$ENV{GITHUB_ACCESS_TOKEN}:" "$lookup_url"); + my $tags = decode_json($out); + my $target_tag = (grep {$_->{name} eq $tag} @$tags)[0]; + return ("Tag $tag not found",''); # TBD... + my $commit_url = $target_tag->{commit}{url}; + $out = qx(curl -Ls -u "$ENV{GITHUB_ACCESS_TOKEN}:" "$commit_url"); + my $data = decude_json($out); + (my $url = $data->{html_url}) =~ s#/commit/#/tree/#; + ($year, $mon, $day) = ($data->{commit}{author}{date} =~ m/^(\d{4})-(\d{2})-(\d{2})/); } - die "update ci/upstream.yml before continuing...\n"; + return ("$day $months[$mon] $year", $url); } -my @table = (); -for my $c (@{$upstream->{core}}) { - my $label = $c->{label} || $c->{name}; - my $version = $versions{$c->{name}}; - my $dated = '-'; - if ($c->{notes}) { - (my $url = $c->{notes}) =~ s/\$\{version}/$versions{$c->{name}}/g; - ($url, $dated) = github($url) if ($url =~ m/github.com/); - $version = "[$version]($url)"; + +sub calculate_software_updates { + my ($path, $last_path) = @_; + printf STDERR " - retrieving current releases found by spec tests...\n"; + my $release_candidate_releases = get_releases($path); + + printf STDERR " - retrieving previous releases found by spec tests...\n"; + my $last_release_releases = get_releases($last_path); + + printf STDERR " - checking ci/upstream.yml for more details...\n"; + my $upstream = LoadFile($FindBin::Bin."/../upstream.yml"); + my $sections = $upstream->{sections} || []; + delete $upstream->{sections}; + + my %found; + my %changed; + print STDERR " - retrieving release details:\n"; + for my $name (uniq(keys %$release_candidate_releases, keys %$last_release_releases)) { + + my @new_versions = keys %{$release_candidate_releases->{$name}||{}}; + my @old_versions = keys %{$last_release_releases->{$name}||{}}; + + #There should never be more than one version in a release + bail( + "Somehow there are multiple versions of %s (%s) in release candidate", + $name, join(', ',@new_versions) + ) if (@new_versions > 1); + bail( + "Somehow there are multiple versions of %s (%s) in last release (%s)", + $name, join(', ',@old_versions), get_latest_commit_tag($last_path) + ) if (@old_versions > 1); + + my $version = $new_versions[0]; + next unless $version; + $changed{$name} = ($version ne $old_versions[0]); + + my $section = (grep {defined($upstream->{$_}{$name})} keys(%$upstream))[0] || '-'; + for my $url (@{$release_candidate_releases->{$name}{$version} || []}) { + my ($type,$target); + if ($url =~ /$name-$version-([^-]*(?:-[^0-9][^-]*)*)-(\d*(?:\.\d*)*)-/) { + $type = 'compiled'; + $target = "$1\@$2"; + } else { + $type = 'source'; + } + my $tag = (defined($upstream->{$section}{$name}{tag_prefix}) ? $upstream->{$section}{$name}{tag_prefix} : 'v').$version; + + $found{$section} ||= {}; + unless (defined($found{$section}{"$name/$version"})) { + my ($date,$release_url) = github( + $upstream->{$section}{$name}{org}, + $upstream->{$section}{$name}{repo}, + $tag, + $upstream->{$section}{$name}{type}, + $upstream->{$section}{$name}{url} + ); + $found{$section}{"$name/$version"} = { + name => $name, + version => $version, + url => $release_url, + date => $date, + form => [] + }; + } + push(@{$found{$section}{"$name/$version"}{form}}, "compiled: $target") if $type eq 'compiled'; + push(@{$found{$section}{"$name/$version"}{form}}, "source") if $type eq 'source'; + } } - push (@table, {"label" => $label, "version" => $version, "dated" => $dated}); -} -print_table "Core Components", \@table; - -if (exists $upstream->{buildpacks}) { - @table = (); - for my $bp (@{$upstream->{buildpacks}}) { - my $label = $bp->{label} || $bp->{name}; - $bp->{name} .= '-buildpack'; - my $version = $versions{$bp->{name}}; - my $dated = '-'; - if ($bp->{notes}) { - (my $url = $bp->{notes}) =~ s/\$\{version}/$versions{$bp->{name}}/g; - ($url, $dated) = github($url) if ($url =~ m/github.com/); - $version = "[$version]($url)"; + + push(@$sections, {name => '-', label => "Other Components"}) if (defined($found{'-'})); + + my $software_notes = "# Software Components\n"; + for (@$sections) { + my $s = $_->{name}; + my $l = $_->{label} || uc($s)." Components"; + $software_notes .= + "\n## $l\n\n". + "| Release | Version | Release Date | Type | Changed |\n". + "| ------- | ------- | ------------ | ---- | :-----: |\n"; + for my $r (sort keys %{$found{$s}}) { + my $c = $found{$s}{$r}; + $c->{label} ||= $c->{name}; + $software_notes .= sprintf("| %s | %s | %s | %s | %s |\n", + $c->{label}, + $c->{url} ? sprintf("[%s](%s)", $c->{version}, $c->{url}) : $c->{version}, + $c->{date}, + join("
", sort @{$c->{form}}), + $changed{$c->{name}} ? "X" : "" + ); } - push (@table, {"label" => $label, "version" => $version, "dated" => $dated}); } - print_table "Buildpacks", \@table; + return $software_notes; +} + +sub build_commit_summaries { + my ($commits,$last_release, $url) = @_; + my $summary = + "---8<--- This line and everything below will be ignored ---8<---\n\n". + "### Raw commit messages since $last_release (oldest to latest)\n"; + + for (@$commits) { + $summary .= sprintf( + "\n\n\n----\n#### %s\n> *[%s](%s)*\n> *authored %s by %s (<%s>)*\n", + $_->{subject}, $_->{commit}, "$url/commit/$_->{commit}", + $_->{author_since}, $_->{author}, $_->{author_email} + ); + $summary .= sprintf( + "> *committed %s by %s (<%s>)*\n", + $_->{commit_since}, $_->{committer}, $_->{committer_email} + ) if $_->{author} ne $_->{committer}; + $summary .= sprintf("\n```markdown\n%s\n```\n", $_->{body}) if $_->{body}; + } + + return $summary; +} + +# ------------------------------------------------------------------------------ +# MAIN +# ------------------------------------------------------------------------------ +my ($version, $rc_path, $last_release_path, $notes_path)=@ARGV; + +# Required Environment Variables +my @missing = grep {! defined($ENV{$_->[0]})} ( + ['GITHUB_ACCESS_TOKEN', 'Access token needed to retrieve information about releases on Github'], +); +bail "[ERROR] Missing the following required environment variables:\n%s", join("", map {sprintf "- %s: %s\n", @{$_}} @missing) + if (@missing); + +my $last_release = get_latest_commit_tag($last_release_path); + +header "Building Release Notes from commit messages since $last_release"; +my ($release_notes,$commits) = parse_commits($rc_path, $last_release_path); + +my $software_updates; +unless ($ENV{NO_RELEASE_VERSIONS}) { + header "Calculating Release Changes since $last_release"; + $software_updates = calculate_software_updates($rc_path,$last_release_path); } + +qx(git -C $rc_path remote get-url origin) =~ /git\@github.com:(.*)$/; +my $commit_summaries = build_commit_summaries($commits, $last_release, "https://github.com/$1"); + +open(NOTES, '>', "$notes_path") + or bail "Cannot open $notes_path for writing."; + +printf NOTES "\n%s\n\n%s\n%s", + $version, $release_notes, $software_updates, $commit_summaries; +close NOTES; + +exit 0; diff --git a/ci/scripts/shipit b/ci/scripts/shipit deleted file mode 100755 index 6cb00a6e..00000000 --- a/ci/scripts/shipit +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/bash -# -# ci/scripts/shipit -# -# Script for generating Github release / tag assets -# and managing release notes for a BOSH Release pipeline -# -# author: James Hunt -# created: 2016-03-30 - -set -eu - -header() { - echo - echo "###############################################" - echo - echo $* - echo -} - -bail() { - echo >&2 $* " Did you misconfigure Concourse?" - exit 2 -} - -test -n "${REPO_ROOT:-}" || bail "REPO_ROOT must be set to the full path to the Git repository." -test -n "${RELEASE_ROOT:-}" || bail "RELEASE_ROOT must be set to the output directory where release artifacts should go." -test -n "${KIT_SHORTNAME:-}" || bail "KIT_SHORTNAME must be set to the short name of this kit." -test -n "${REPO_OUT:-}" || bail "REPO_OUT must be set to the output directory where the Git working copy should be moved to." -test -n "${BRANCH:-}" || bail "BRANCH must be set to the git branch you pulled the Git repository from." -test -n "${GITHUB_OWNER:-}" || bail "GITHUB_OWNER must be set to the name of the Github user or organization that owns the Git repository." -test -n "${VERSION_FROM:-}" || bail "VERSION_FROM must be set to the path of the input version file." -test -n "${GIT_EMAIL:-}" || bail "GIT_EMAIL must be set to an email address to use for Git commits." -test -n "${GIT_NAME:-}" || bail "GIT_NAME must be set to something; it will be used for Git commits." - -test -f "${VERSION_FROM}" || bail "Version file (${VERSION_FROM}) not found." -VERSION=$(cat ${VERSION_FROM}) -test -n "${VERSION}" || bail "Version file (${VERSION_FROM}) was empty." - -if [[ ! -f ${REPO_ROOT}/ci/release_notes.md ]]; then - echo >&2 - echo >&2 "#################################################" - echo >&2 - echo >&2 "ci/release_notes.md not found." - echo >&2 "Did you forget to write them?" - echo >&2 - echo >&2 "Ah well, happens to the best of us." - echo >&2 "We'll make do with what we've got, but please" - echo >&2 "don't forget to write them on GitHub!" - echo >&2 - echo >&2 "#################################################" - echo >&2 - - mkdir -p ${REPO_ROOT}/ci - echo "_no release notes were written for this release._" > ${REPO_ROOT}/ci/release_notes.md -fi - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -############################################################### - -VERSION=$(cat $VERSION_FROM) -header "Compiling v$VERSION of Genesis Kit..." -(cd $REPO_ROOT - genesis compile-kit --version "$VERSION" --name "$KIT_SHORTNAME") - -header "Assembling Github Release Artifacts..." -mkdir -p ${RELEASE_ROOT}/artifacts -echo "v${VERSION}" > ${RELEASE_ROOT}/tag -echo "v${VERSION}" > ${RELEASE_ROOT}/name -mv git/*.tar.gz ${RELEASE_ROOT}/artifacts -mv ${REPO_ROOT}/ci/release_notes.md ${RELEASE_ROOT}/notes.md -cat > ${RELEASE_ROOT}/notification < New ${KIT_SHORTNAME} v${VERSION} released! -EOF - - -header "Updating Git repository..." -git config --global user.name "${GIT_NAME}" -git config --global user.email "${GIT_EMAIL}" - -(cd ${REPO_ROOT} - git merge --no-edit ${BRANCH} - git add -A - git status - git commit -m "release v${VERSION}") - -# so that future steps in the pipeline can push our changes -cp -a ${REPO_ROOT} ${REPO_OUT} - -cat > ${NOTIFICATION_OUT:-notifications}/message <. -EOS diff --git a/ci/scripts/spec-check b/ci/scripts/spec-check new file mode 100755 index 00000000..9ac7abee --- /dev/null +++ b/ci/scripts/spec-check @@ -0,0 +1,23 @@ +#!/bin/bash +set -e + +# Resource Directories +REPO_ROOT="git" +CI_ROOT="git-ci" +TAG_ROOT="git-latest-tag" +OUTPUT_ROOT="spec-check" + +CI_PATH="$(cd "${CI_ROOT}" && pwd)" +TAG="$(cat "${TAG_ROOT}/.git/ref")" +results_file="$(cd "${OUTPUT_ROOT}" && pwd)/diff-$(date -u +%Y%m%d%H%M%S)" + +# Run as a script to preserve color output +export CI_PATH +export TAG +pushd "${REPO_ROOT}" +script --flush --quiet \ + --return "$results_file" \ + --command '"${CI_PATH}"/ci/scripts/compare-release-specs "$TAG"' + +# Trim script header/footer (ignore error) +sed -i '1d;$d' "$results_file" || true diff --git a/ci/scripts/test b/ci/scripts/test deleted file mode 100755 index ea1f41fd..00000000 --- a/ci/scripts/test +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash -set -eu - -SKIP_FRESH=${SKIP_FRESH:-"false"} -SKIP_DEPLOY=${SKIP_DEPLOY:-"false"} -SKIP_SMOKE_TESTS=${SKIP_SMOKE_TESTS:-"false"} -SKIP_CLEAN=${SKIP_CLEAN:-"false"} -DEPLOYMENT_NAMES=(${DEPLOYMENT_NAMES-"ci-baseline"}) - - -header() { - echo - echo "###############################################" - echo - echo $* - echo -} - -cleanup() { - for deployment in "$@"; do - echo "> deleting ${deployment}" - $BOSH -n -d "${deployment}" delete-deployment - - for disk in $($BOSH disks --orphaned | grep "${deployment}" | awk '{print $1}'); do - echo " - removing disk $disk" - $BOSH -n delete-disk "$disk" - done - done -} - -header "Checking previous deployments on ${BOSH_ENVIRONMENT}..." -$BOSH deployments - -header "Cleaning up from any previous deployments (if necessary)..." -for DEPLOYMENT_NAME in $DEPLOYMENT_NAMES; do - if [[ "$SKIP_FRESH" == "false" ]]; then - cleanup ${DEPLOYMENT_NAME}-${KIT} - - # replace - with / for safe commands - depname=${DEPLOYMENT_NAME//-//} - safe rm -rf secret/${depname}/${KIT} - safe ssh secret/${depname}/${KIT}/test-user - - credhub login --server $CREDHUB_URL --username $CREDHUB_USER --password $CREDHUB_PASSWORD --skip-tls-validation - credhub delete -p /snw-genesis-ci-bosh/${DEPLOYMENT_NAME}-${KIT}/ - - - else - echo "Skipping FRESH" - fi - - if [[ "$SKIP_DEPLOY" == "false" ]]; then - header "Deploying BASELINE environment to verify functionality..." - cd dev - # make sure hooks are executable before screaming at shipit for failing. - genesis compile-kit --name $KIT -v 9.9.9 --force - cd .. - genesis add-secrets ${DEPLOYMENT_NAME} - - # get and upload stemcell version if needed (handled by bosh cli if version and name are supplied) - stemcell_version=$(genesis manifest ${DEPLOYMENT_NAME} | spruce json | jq -r .stemcells[0].version) - bosh upload-stemcell "https://bosh.io/d/stemcells/bosh-vsphere-esxi-ubuntu-bionic-go_agent?v=${stemcell_version}" --version ${stemcell_version} --name bosh-vsphere-esxi-ubuntu-bionic-go_agent - - genesis deploy -y ${DEPLOYMENT_NAME} - $BOSH -d ${DEPLOYMENT_NAME}-${KIT} instances --ps - fi - - if [[ "$SKIP_SMOKE_TESTS" == "false" ]]; then - header "Validating BASELINE environment..." - $BOSH -d ${DEPLOYMENT_NAME}-${KIT} run-errand smoke_tests - else - echo "Skipping smoke_tests" - fi - - if [[ "$SKIP_CLEAN" == "false" ]]; then - cleanup ${DEPLOYMENT_NAME}-${KIT} - else - echo "Skipping CLEANUP" - fi -done diff --git a/ci/scripts/test-deployment b/ci/scripts/test-deployment new file mode 100755 index 00000000..0ff21822 --- /dev/null +++ b/ci/scripts/test-deployment @@ -0,0 +1,216 @@ +#!/bin/bash +set -eu + +DEPLOY_ENV=${DEPLOY_ENV:-"ci-baseline"} +SKIP_FRESH=${SKIP_FRESH:-"false"} +SKIP_REPLACE_SECRETS=${SKIP_REPLACE_SECRETS:-"false"} +SKIP_DEPLOY=${SKIP_DEPLOY:-"false"} +SKIP_SMOKE_TESTS=${SKIP_SMOKE_TESTS:-"false"} +SKIP_CLEAN=${SKIP_CLEAN:-"false"} + +header() { + echo + echo "================================================================================" + echo "$1" + echo "--------------------------------------------------------------------------------" + echo +} + +has_feature() { + genesis lookup "$1" kit.features 2>/dev/null | jq -e --arg feature "$2" '. | index($feature)' >/dev/null +} + +is_proto() { + has_feature $1 'proto' # This will need to be changed in v2.8.0 +} + +cleanup_environment() { + env="$1" + if [[ -f .genesis/manifests/$env-state.yml ]] ; then + header "Preparing to delete proto environment $env" + echo "Generating reference manifest..." + genesis manifest "$env" --no-redact > manifest.yml 2>/dev/null + echo $'\n'"Building BOSH variables file..." + genesis lookup --merged "${env}" bosh-variables > vars.yml 2>/dev/null + echo $'\n'"$env state file:" + echo "----------------->8------------------" + cat ".genesis/manifests/$env-state.yml" + echo "----------------->8------------------" + header "Deleting $DEPLOY_ENV environment..." + $BOSH delete-env --state ".genesis/manifests/$env-state.yml" --vars-file vars.yml manifest.yml + rm manifest.yml + rm vars.yml + else + echo "Cannot clean up previous $env environment - no state file found" + fi +} + +cleanup_deployment() { + deployment="$1" + echo "> deleting ${deployment}" + $BOSH -n -d "${deployment}" delete-deployment + + for disk in $($BOSH disks --orphaned | grep "${deployment}" | awk '{print $1}'); do + echo " - removing disk $disk" + $BOSH -n delete-disk "$disk" + done +} + +cleanup() { + for deployment in "$@"; do + if is_proto "$deployment" ; then + cleanup_environment "$deployment" + else + + ( + eval "$(genesis bosh --connect "${deployment}")" + cleanup_deployment "$deployment-${KIT_SHORTNAME}" + ) + fi + done +} + +# Replace this with genesis lookup --env once its available (2.8.5) +cat <<'EOF' > get-env.pl +#!/usr/bin/env perl +use lib "$ENV{HOME}/.geese/lib"; +use JSON::PP qw/encode_json/; +print encode_json(\%ENV); +EOF +chmod +x "get-env.pl" +vault_path="$(genesis sh "${DEPLOY_ENV}" -s "$(pwd)/get-env.pl" | grep '^{' | jq -r '.GENESIS_SECRETS_BASE')" +exodus_path="$(genesis sh "${DEPLOY_ENV}" -s "$(pwd)/get-env.pl" | grep '^{' | jq -r '.GENESIS_EXODUS_BASE')" +vault_path="${vault_path%/}" # trim any trailing slash +# ----- + +header "Pre-test Cleanup" +if [[ "$SKIP_FRESH" == "false" ]]; then + echo "Deleting any previous deploy" + cleanup "${DEPLOY_ENV}" +else + echo "Skipping cleaning up from any previous deploy" +fi + +if [[ -z "$vault_path" ]] ; then + echo >&2 "Failed to determine vault path. Cannot continue!" + exit 2 +fi + +if [[ "$SKIP_REPLACE_SECRETS" == "false" ]] ; then + # Remove safe values + [[ -n "${vault_path:-}" ]] && \ + echo "Removing existing secrets under $vault_path ..." && \ + safe rm -rf "$vault_path" && true + [[ -n "${exodus_path:-}" ]] && \ + echo "Removing existing exodus data under $exodus_path ..." && \ + safe rm -rf "$exodus_path" && true + + # Remove credhub values + if ! is_proto ; then ( + bosh_env="$(genesis lookup "$deployment" genesis | jq -r '.bosh_env // .env')" + [[ "$bosh_env" =~ / ]] || bosh_env="${bosh_env}/bosh" + + bosh_exodus="$(genesis lookup --exodus-for "$bosh_env" . "{}")" + CREDHUB_SERVER="$(jq -r '.credhub_url // ""' <<<"$bosh_exodus")" + if [[ -n "$CREDHUB_SERVER" ]] ; then + echo "Attempting to remove credhub secrets under /${bosh_env/\//-}/${DEPLOYMENT_NAME}-${KIT_SHORTNAME}/" + CREDHUB_CLIENT="$(jq -r '.credhub_username // ""' <<<"$bosh_exodus")" + CREDHUB_SECRET="$(jq -r '.credhub_password // ""' <<<"$bosh_exodus")" + CREDHUB_CA_CERT="$(jq -r '.credhub_ca_cert // ""' <<<"$bosh_exodus")" + export CREDHUB_SERVER CREDHUB_CLIENT CREDHUB_SECRET CREDHUB_CA_CERT + credhub login && \ + credhub delete -p "/${bosh_env/\//-}/${DEPLOYMENT_NAME}-${KIT_SHORTNAME}/" + fi + ) ; fi + + if [[ -n "$SECRETS_SEED_DATA" ]] ; then + + header "Importing required user-provided seed data for $DEPLOY_ENV" + # Replace and sanitize seed data + seed= + if ! seed="$(echo "$SECRETS_SEED_DATA" | spruce merge --skip-eval | spruce json | jq -M .)" ; then + echo >&2 "Secrets seed data is corrupt; expecting valid JSON" + exit 1 + fi + if ! bad_keys="$(jq -rM '. | with_entries( select(.key|test("^\\${GENESIS_SECRETS_BASE}/")|not))| keys| .[] | " - \(.)"' <<<"$seed")" ; then + echo >&2 "Failed to validate secrets seed data keys: $bad_keys" + exit 1 + fi + if [[ -n "$bad_keys" ]] ; then + echo >&2 "Secrets seed data contains bad keys. All keys must start with " + echo >&2 "'\${GENESIS_SECRETS_BASE}/', and the following do not:" + echo >&2 "$bad_keys" + exit 1 + fi + processed_data= + if ! processed_data="$( jq -M --arg p "$vault_path/" '. | with_entries( .key |= sub("^\\${GENESIS_SECRETS_BASE}/"; $p))' <<<"$seed")" ; then + echo >&2 "Failed to import secret seed data" + exit 1 + fi + if ! safe import <<<"$processed_data" ; then + echo >&2 "Failed to import secrets seed data" + exit 1 + fi + fi +else + echo "Skipping replacing secrets" +fi + +if [[ "$SKIP_DEPLOY" == "false" ]]; then + header "Deploying ${DEPLOY_ENV} environment to verify functionality..." + genesis "do" "${DEPLOY_ENV}" -- list + genesis add-secrets "${DEPLOY_ENV}" + + # get and upload stemcell version if needed (handled by bosh cli if version and name are supplied) + stemcell_iaas= + case "${INFRASTRUCTURE:-none}" in + aws) stemcell_iaas="aws-xen-hvm" ;; + azure) stemcell_iaas="azure-hyperv" ;; + openstack) stemcell_iaas="openstack-kvm" ;; + warden) stemcell_iaas="warden-boshlite" ;; + google|gcp) stemcell_iaas="google-kvm" ;; + vsphere) stemcell_iaas="vsphere-esxi" ;; + *) echo >&2 "Unknown or missing INFRASTRUCTURE value -- cannot upload stemcell" ;; + esac + + if [[ -n "$stemcell_iaas" ]] ; then + stemcell_data="$(genesis lookup --manifest "${DEPLOY_ENV}" stemcells)" + stemcell_os="$(jq -r '.[0].os' <<<"$stemcell_data")" + stemcell_version="$(jq -r '.[0].version' <<<"$stemcell_data")" + stemcell_name="bosh-${stemcell_iaas} -${stemcell_os}-go_agent" + bosh upload-stemcell "https://bosh.io/d/stemcells/$stemcell_name?v=${stemcell_version}" \ + --version "${stemcell_version}" --name "$stemcell_name" + fi + + genesis deploy -y "${DEPLOY_ENV}" + + if [[ -f .genesis/manifests/${DEPLOY_ENV}-state.yml ]] ; then + echo $'\n'"${DEPLOY_ENV} state file:" + echo "----------------->8------------------" + cat ".genesis/manifests/${DEPLOY_ENV}-state.yml" + echo "----------------->8------------------" + fi + + genesis info "${DEPLOY_ENV}" + + header "Validating addons..." + genesis "do" "${DEPLOY_ENV}" -- setup-cli + genesis "do" "${DEPLOY_ENV}" -- login + + if ! is_proto "$DEPLOY_ENV" ; then + genesis bosh "${DEPLOY_ENV}" instances --ps + fi + +fi + +if [[ "$SKIP_SMOKE_TESTS" == "false" ]]; then + genesis "do" "${DEPLOY_ENV}" -- smoketest +else + echo "Skipping smoke_tests" +fi + +if [[ "$SKIP_CLEAN" == "false" ]]; then + cleanup "${DEPLOY_ENV}" +else + echo "Skipping CLEANUP" +fi diff --git a/ci/settings.yml b/ci/settings.yml index 7dcb95b3..7f0fe9d6 100644 --- a/ci/settings.yml +++ b/ci/settings.yml @@ -2,12 +2,17 @@ meta: kit: cf release: CF Genesis Kit - target: pipes@genesis - url: https://pipes.starkandwayne.com + upstream_path: cf-deployment + target: cloudpipes/genesis + url: https://cloudpipes.starkandwayne.com + iaas: gcp exposed: yes version_file: version initial_version: 2.1.0 + upstream: + package: cf-deployment + bosh: uri: ((bosh.uri)) username: ((bosh.username)) diff --git a/ci/tasks/build-kit.yml b/ci/tasks/build-kit.yml new file mode 100644 index 00000000..5051636b --- /dev/null +++ b/ci/tasks/build-kit.yml @@ -0,0 +1,21 @@ +--- +platform: linux + +image_resource: + type: registry-image + source: + repository: starkandwayne/genesis + +inputs: +- name: version +- name: git +- name: git-ci + +outputs: +- name: build + +params: + GENESIS_HONOR_ENV: 1 + +run: + path: git-ci/ci/scripts/build-kit diff --git a/ci/tasks/cats.yml b/ci/tasks/cats.yml new file mode 100644 index 00000000..a4915261 --- /dev/null +++ b/ci/tasks/cats.yml @@ -0,0 +1,26 @@ +--- +platform: linux + +image_resource: + type: docker-image + source: + repository: starkandwayne/concourse-go + tag: '1.14' + +inputs: +- name: git + path: kit +- name: git-ci +- name: cats +- name: work + +run: + path: git-ci/ci/scripts/cats + +params: + GENESIS_HONOR_ENV: 1 + SKIP_CATS: false + FAILFAST: true + VERBOSE: false + CATS_NODES: 2 + RUN_CATS: nul diff --git a/ci/tasks/cats/task b/ci/tasks/cats/task deleted file mode 100755 index be6b7c82..00000000 --- a/ci/tasks/cats/task +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -e - -source git-ci/ci/scripts/cats -git config --global user.name "$GIT_NAME" -git config --global user.email "$GIT_EMAIL" - -ln -sf $(pwd)/kit/ $(pwd)/work/cf-deployments/dev - -safe target da-vault "$VAULT_URI" -k -echo "$VAULT_TOKEN" | safe auth token -safe read secret/handshake -pushd kit - run_cats --deployment-dir $(pwd)/../ -popd \ No newline at end of file diff --git a/ci/tasks/cats/task.yml b/ci/tasks/cats/task.yml deleted file mode 100644 index 291f0504..00000000 --- a/ci/tasks/cats/task.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: - repository: starkandwayne/concourse-go - tag: '1.16' - -inputs: -- name: git - path: kit -- name: git-ci -- name: cats -- name: work - -run: - path: git-ci/ci/tasks/cats/task - -params: - GENESIS_HONOR_ENV: 1 - GIT_NAME: Stark & Wayne CI Bot - GIT_EMAIL: ci@starkandwayne.com - VAULT_URI: ((vault.url)) - VAULT_TOKEN: ((vault.token)) - KIT_SHORTNAME: cf - BOSH_ENVIRONMENT: ((bosh.uri)) - BOSH_CA_CERT: ((bosh.ca)) - BOSH_CLIENT: ((bosh.username)) - BOSH_CLIENT_SECRET: ((bosh.password)) - SKIP_CATS: false - FAILFAST: true - VERBOSE: false - CATS_NODES: 2 - RUN_CATS: nul - CREDHUB_URL: ((bosh.credhub_url)) - CREDHUB_USER: ((bosh.credhub_username)) - CREDHUB_PASSWORD: ((bosh.credhub_password)) diff --git a/ci/tasks/deploy-stable/task.yml b/ci/tasks/deploy-stable.yml similarity index 52% rename from ci/tasks/deploy-stable/task.yml rename to ci/tasks/deploy-stable.yml index dd596658..4b618c7d 100644 --- a/ci/tasks/deploy-stable/task.yml +++ b/ci/tasks/deploy-stable.yml @@ -2,29 +2,26 @@ platform: linux image_resource: - type: docker-image + type: registry-image source: - repository: starkandwayne/concourse + repository: starkandwayne/genesis inputs: - name: git-latest-tag - path: kit - name: git-ci outputs: - name: work run: - path: git-ci/ci/tasks/test-flight/task + path: git-ci/ci/scripts/deploy params: + TAG_ROOT: git-latest-tag GENESIS_HONOR_ENV: 1 GIT_NAME: Stark & Wayne CI Bot GIT_EMAIL: ci@starkandwayne.com VAULT_URI: ((vault.url)) VAULT_TOKEN: ((vault.token)) - KIT_SHORTNAME: cf - BOSH_ENVIRONMENT: ((bosh.uri)) - BOSH_CA_CERT: ((bosh.ca)) - BOSH_CLIENT: ((bosh.username)) - BOSH_CLIENT_SECRET: ((bosh.password)) \ No newline at end of file + KIT_SHORTNAME: bosh + SECRETS_SEED_DATA: ((secrets_seed_data)) diff --git a/ci/tasks/deploy.yml b/ci/tasks/deploy.yml new file mode 100644 index 00000000..8323726a --- /dev/null +++ b/ci/tasks/deploy.yml @@ -0,0 +1,28 @@ +--- +platform: linux + +image_resource: + type: registry-image + source: + repository: starkandwayne/genesis + +inputs: +- name: git +- name: git-ci +- name: version +- name: build + +outputs: +- name: work + +run: + path: git-ci/ci/scripts/deploy + +params: + BUILD_ROOT: build + GENESIS_HONOR_ENV: 1 + GIT_NAME: Stark & Wayne CI Bot + GIT_EMAIL: ci@starkandwayne.com + VAULT_URI: ((vault.url)) + VAULT_TOKEN: ((vault.token)) + SECRETS_SEED_DATA: ((secrets_seed_data)) diff --git a/ci/tasks/generate-release-notes.yml b/ci/tasks/generate-release-notes.yml new file mode 100644 index 00000000..48f7befb --- /dev/null +++ b/ci/tasks/generate-release-notes.yml @@ -0,0 +1,23 @@ +--- +platform: linux + +image_resource: + type: registry-image + source: + repository: starkandwayne/genesis + +inputs: +- name: git +- name: git-ci +- name: git-latest-tag +- name: version +- name: release-notes + +outputs: +- name: release-notes + +params: + GITHUB_ACCESS_TOKEN: ((github.access_token)) + +run: + path: git-ci/ci/scripts/generate-release-notes diff --git a/ci/tasks/generate-release-notes/task b/ci/tasks/generate-release-notes/task deleted file mode 100755 index 08b02027..00000000 --- a/ci/tasks/generate-release-notes/task +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -o pipefail -# set -x - -custom_notes=$(pwd)/git-ci/ci/release_notes.md -notes=$(pwd)/release-notes/notes.md - -touch ${custom_notes} -cp ${custom_notes} ${notes} - -get_releases() { - spruce merge --go-patch spec/results/*.yml | spruce json |jq '.releases | sort_by(.name) | unique_by(.name) | .[]' -} - -release_paths=$(curl -s https://bosh.io/releases/ | grep ' /dev/null -stable_releases=$(get_releases) -popd > /dev/null - -echo $stable_releases | jq '.name' - -pushd git > /dev/null -new_releases=$(get_releases) -popd > /dev/null - -notes_releases="" -for release in $(echo ${new_releases} | jq -c '.'); do - name=$(echo ${release} | jq -r '.name') - version=$(echo ${release} | jq -r '.version') - stable_version=$(echo ${stable_releases} | jq -s -r --arg name "${name}" '.[] | select(.name == $name).version') - - release_url=$(get_release_url ${name} ${version}) - if [ ! -z "${release_url}" ]; then - notes_releases+="| ${name} | [${version}](${release_url}) |" - else - notes_releases+="| ${name} | ${version} |" - fi - - if [[ "${version}" != "${stable_version}" ]]; then - notes_releases+=" yes |\n" - else - notes_releases+=" no |\n" - fi -done - -if [ ! -z "${notes_releases}" ]; then - echo "## Release updates:" >> ${notes} - echo "| Name | Version | Changed |" >> ${notes} - echo "| ---- | ------- | ------- |" >> ${notes} - echo -e "${notes_releases}" >> ${notes} -fi - -cat ${notes} \ No newline at end of file diff --git a/ci/tasks/generate-release-notes/task.yml b/ci/tasks/generate-release-notes/task.yml deleted file mode 100644 index 3be0cd26..00000000 --- a/ci/tasks/generate-release-notes/task.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: - repository: starkandwayne/concourse - -inputs: -- name: git -- name: git-ci -- name: git-latest-tag - -outputs: - - name: release-notes - -run: - path: git-ci/ci/tasks/generate-release-notes/task \ No newline at end of file diff --git a/ci/tasks/get-latest-cf-deployment/task b/ci/tasks/get-latest-cf-deployment/task deleted file mode 100755 index e72eb5ab..00000000 --- a/ci/tasks/get-latest-cf-deployment/task +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -o pipefail - -pushd cf-deployment - version=$(cat .git/ref | cut -b 2- ) -popd - -pushd git > /dev/null - - if [[ -z $(git config --global user.email) ]]; then - git config --global user.email "ci@starkandwayne.com" - fi - if [[ -z $(git config --global user.name) ]]; then - git config --global user.name "CI Bot" - fi - - echo "CHECKING git status" - git status - - echo "Bump CF deployment to $version" - ../git-ci/ci/scripts/pull-upstream $version - - echo "recreate spec-test results to validate: the cf-deployment update " - - pushd spec > /dev/null - # TODO: remove spec/{credhub,vault} if needed? because this will regenerate vault. - rm -rf results/ - ginkgo -p - popd - - git add cf-deployment/ - git add spec/ - git add overlay/upstream_version.yml - git commit -m "Update cf-deployment to v$version" - -popd > /dev/null diff --git a/ci/tasks/get-latest-cf-deployment/task.yml b/ci/tasks/get-latest-upstream.yml similarity index 64% rename from ci/tasks/get-latest-cf-deployment/task.yml rename to ci/tasks/get-latest-upstream.yml index 26eb0ac8..8bfdc2c1 100644 --- a/ci/tasks/get-latest-cf-deployment/task.yml +++ b/ci/tasks/get-latest-upstream.yml @@ -2,7 +2,7 @@ platform: linux image_resource: - type: docker-image + type: registry-image source: repository: starkandwayne/concourse-go tag: '1.16' @@ -10,10 +10,10 @@ image_resource: inputs: - name: git-ci - name: git - - name: cf-deployment + - name: upstream outputs: - name: git run: - path: git-ci/ci/tasks/get-latest-cf-deployment/task + path: git-ci/ci/scripts/get-latest-upstream diff --git a/ci/tasks/prerelease.yml b/ci/tasks/prerelease.yml new file mode 100644 index 00000000..ebf80120 --- /dev/null +++ b/ci/tasks/prerelease.yml @@ -0,0 +1,30 @@ +--- +platform: linux + +image_resource: + type: registry-image + source: + repository: starkandwayne/genesis + +inputs: +- name: version +- name: git +- name: git-ci +- name: build + +outputs: +- name: gh +- name: notifications + +params: + PRERELEASE: 1 + RELEASE_ROOT: gh + NOTIFICATION_OUT: notifications + DEVELOP_BRANCH: develop + RELEASE_BRANCH: main + GITHUB_OWNER: genesis-community + GIT_EMAIL: ci@starkandwayne.com + GIT_NAME: Stark & Wayne CI Bot + +run: + path: git-ci/ci/scripts/release diff --git a/ci/tasks/shipit/task.yml b/ci/tasks/release.yml similarity index 59% rename from ci/tasks/shipit/task.yml rename to ci/tasks/release.yml index eeec7de0..9263bf96 100644 --- a/ci/tasks/shipit/task.yml +++ b/ci/tasks/release.yml @@ -2,32 +2,31 @@ platform: linux image_resource: - type: docker-image + type: registry-image source: - repository: starkandwayne/concourse + repository: starkandwayne/genesis inputs: - name: version - name: git - name: git-ci - name: git-main +- name: spec-check +- name: build +- name: release-notes outputs: - name: gh - name: notifications -- name: git-main -- name: git-ci params: - REPO_ROOT: git - VERSION_FROM: version/number RELEASE_ROOT: gh NOTIFICATION_OUT: notifications - BRANCH: master + DEVELOP_BRANCH: develop + RELEASE_BRANCH: main GITHUB_OWNER: genesis-community GIT_EMAIL: ci@starkandwayne.com GIT_NAME: Stark & Wayne CI Bot - KIT_SHORTNAME: cf run: - path: git-ci/ci/tasks/shipit/task \ No newline at end of file + path: git-ci/ci/scripts/release diff --git a/ci/tasks/shipit/task b/ci/tasks/shipit/task deleted file mode 100755 index feda02d4..00000000 --- a/ci/tasks/shipit/task +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash -# -# ci/scripts/shipit -# -# Script for generating Github release / tag assets -# and managing release notes for a BOSH Release pipeline -# -# author: James Hunt -# created: 2016-03-30 - -set -eu - -header() { - echo - echo "###############################################" - echo - echo $* - echo -} - -bail() { - echo >&2 $* " Did you misconfigure Concourse?" - exit 2 -} - -test -n "${REPO_ROOT:-}" || bail "REPO_ROOT must be set to the full path to the Git repository." -test -n "${RELEASE_ROOT:-}" || bail "RELEASE_ROOT must be set to the output directory where release artifacts should go." -test -n "${KIT_SHORTNAME:-}" || bail "KIT_SHORTNAME must be set to the short name of this kit." -test -n "${GITHUB_OWNER:-}" || bail "GITHUB_OWNER must be set to the name of the Github user or organization that owns the Git repository." -test -n "${VERSION_FROM:-}" || bail "VERSION_FROM must be set to the path of the input version file." -test -n "${GIT_EMAIL:-}" || bail "GIT_EMAIL must be set to an email address to use for Git commits." -test -n "${GIT_NAME:-}" || bail "GIT_NAME must be set to something; it will be used for Git commits." - -test -f "${VERSION_FROM}" || bail "Version file (${VERSION_FROM}) not found." -VERSION=$(cat ${VERSION_FROM}) -test -n "${VERSION}" || bail "Version file (${VERSION_FROM}) was empty." - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -############################################################### - -VERSION=$(cat $VERSION_FROM) -header "Compiling v$VERSION of Genesis Kit..." -pushd $REPO_ROOT - genesis compile-kit --version "$VERSION" --name "$KIT_SHORTNAME" -popd - -header "Assembling Github Release Artifacts..." -mkdir -p ${RELEASE_ROOT}/artifacts -echo "v${VERSION}" > ${RELEASE_ROOT}/tag -echo "v${VERSION}" > ${RELEASE_ROOT}/name -mv $REPO_ROOT/*.tar.gz ${RELEASE_ROOT}/artifacts - -cat > ${RELEASE_ROOT}/notification < New ${KIT_SHORTNAME} v${VERSION} released! -EOF - -header "Updating Git repository..." -git config --global user.name "${GIT_NAME}" -git config --global user.email "${GIT_EMAIL}" - -pushd git-ci - # Clean release-notes - echo -e "\n" > ci/release_notes.md - echo -e "## Version: ${VERSION}" > ci/version - git add -A - git status - git commit -m "release v${VERSION}" -popd - -header "Merge develop into master" -pushd git-main - git pull ../git-ci -X theirs --no-edit - git pull ../git -X theirs --no-edit -popd - - - -cat > ${NOTIFICATION_OUT:-notifications}/message <. -EOS diff --git a/ci/tasks/spec-check.yml b/ci/tasks/spec-check.yml new file mode 100644 index 00000000..176c5be8 --- /dev/null +++ b/ci/tasks/spec-check.yml @@ -0,0 +1,21 @@ +--- +platform: linux + +image_resource: + type: registry-image + source: + repository: starkandwayne/genesis + +inputs: +- name: git +- name: git-ci +- name: git-latest-tag + +outputs: +- name: spec-check + +params: + GENESIS_HONOR_ENV: 1 + +run: + path: git-ci/ci/scripts/spec-check diff --git a/ci/tasks/spec-check/task b/ci/tasks/spec-check/task deleted file mode 100755 index 182be477..00000000 --- a/ci/tasks/spec-check/task +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set -e - -version=$(cat git-latest-tag/.git/ref) - -pushd git - -../git-ci/ci/tasks/spec-check/spec-check ${version} diff --git a/ci/tasks/spec-check/task.yml b/ci/tasks/spec-check/task.yml deleted file mode 100644 index 64b0f9ba..00000000 --- a/ci/tasks/spec-check/task.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: - repository: starkandwayne/concourse - -inputs: -- name: git -- name: git-ci -- name: git-latest-tag - -run: - path: git-ci/ci/tasks/spec-check/task \ No newline at end of file diff --git a/ci/tasks/spec-tests/task.yml b/ci/tasks/spec-tests.yml similarity index 66% rename from ci/tasks/spec-tests/task.yml rename to ci/tasks/spec-tests.yml index f8137c69..12421fde 100644 --- a/ci/tasks/spec-tests/task.yml +++ b/ci/tasks/spec-tests.yml @@ -2,7 +2,7 @@ platform: linux image_resource: - type: docker-image + type: registry-image source: repository: starkandwayne/concourse-go tag: '1.16' @@ -12,4 +12,6 @@ inputs: - name: git-ci run: - path: git-ci/ci/tasks/spec-tests/task + dir: git/spec + path: ginkgo + args: [ "-p", "." ] diff --git a/ci/tasks/spec-tests/task b/ci/tasks/spec-tests/task deleted file mode 100755 index 8842126f..00000000 --- a/ci/tasks/spec-tests/task +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -pushd git/spec - -ginkgo -p diff --git a/ci/tasks/test-flight/task b/ci/tasks/test-flight/task deleted file mode 100755 index 4b3240fa..00000000 --- a/ci/tasks/test-flight/task +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -# -# ci/scripts/testflight -# -# Script for testing a Genesis Kit -# -# author: James Hunt -# created: 2018-01-23 -# - -set -eu - -: ${BOSH_ENVIRONMENT:?required} -: ${BOSH_CA_CERT:?required} -: ${BOSH_CLIENT:?required} -: ${BOSH_CLIENT_SECRET:?required} -: ${VAULT_URI:?required} -: ${VAULT_TOKEN:?required} -: ${KIT_SHORTNAME:?required} - -header() { - echo - echo "###############################################" - echo - echo $* - echo -} - -header "Setting up git..." -git config --global user.name "$GIT_NAME" -git config --global user.email "$GIT_EMAIL" - -safe target da-vault "$VAULT_URI" -k -echo "$VAULT_TOKEN" | safe auth token -safe read secret/handshake - -header "Setting up Genesis *-deployments/ directory..." -rm -rf work/*; mkdir -p work/ -genesis -v -genesis -C work/ init -L ../kit/ --vault da-vault ${KIT_SHORTNAME} - -header "Copying test environment YAMLs from \$kit/ci/envs..." -cp -av git-ci/ci/envs/*.yml work/${KIT_SHORTNAME}-deployments/ -export PATH=$PATH:$PWD/git-ci/ci/scripts - -header "Handing off to \$kit/ci/scripts/test..." -cd work/${KIT_SHORTNAME}-deployments -export GENESIS_BOSH_ENVIRONMENT=$BOSH_ENVIRONMENT -KIT=${KIT_SHORTNAME} BOSH=bosh ../../git-ci/ci/scripts/test - -echo -echo "SUCCESS" -exit 0 diff --git a/ci/tasks/test-flight/task.yml b/ci/tasks/upgrade.yml similarity index 60% rename from ci/tasks/test-flight/task.yml rename to ci/tasks/upgrade.yml index 2830feb1..e17aadec 100644 --- a/ci/tasks/test-flight/task.yml +++ b/ci/tasks/upgrade.yml @@ -2,29 +2,33 @@ platform: linux image_resource: - type: docker-image + type: registry-image source: - repository: starkandwayne/concourse + repository: starkandwayne/genesis inputs: - name: git - path: kit - name: git-ci +- name: version +- name: build +- name: work outputs: - name: work run: - path: git-ci/ci/tasks/test-flight/task + path: git-ci/ci/scripts/deploy params: GENESIS_HONOR_ENV: 1 + BUILD_ROOT: build GIT_NAME: Stark & Wayne CI Bot GIT_EMAIL: ci@starkandwayne.com VAULT_URI: ((vault.url)) VAULT_TOKEN: ((vault.token)) - KIT_SHORTNAME: cf BOSH_ENVIRONMENT: ((bosh.uri)) BOSH_CA_CERT: ((bosh.ca)) BOSH_CLIENT: ((bosh.username)) - BOSH_CLIENT_SECRET: ((bosh.password)) \ No newline at end of file + BOSH_CLIENT_SECRET: ((bosh.password)) + KEEP_STATE: true + SECRETS_SEED_DATA: ((secrets_seed_data)) diff --git a/ci/upstream.yml b/ci/upstream.yml deleted file mode 100644 index 95e17d7c..00000000 --- a/ci/upstream.yml +++ /dev/null @@ -1,73 +0,0 @@ -buildpacks: -- name: binary - notes: https://github.com/cloudfoundry/binary-buildpack-release/releases/tag/${version} -- name: dotnet-core - notes: https://github.com/cloudfoundry/dotnet-core-buildpack-release/releases/tag/${version} -- name: go - notes: https://github.com/cloudfoundry/go-buildpack-release/releases/tag/${version} -- name: java - notes: https://github.com/cloudfoundry/java-buildpack-release/releases/tag/${version} -- name: nginx - notes: https://github.com/cloudfoundry/nginx-buildpack-release/releases/tag/${version} -- name: nodejs - notes: https://github.com/cloudfoundry/nodejs-buildpack-release/releases/tag/${version} -- name: php - notes: https://github.com/cloudfoundry/php-buildpack-release/releases/tag/${version} -- name: python - notes: https://github.com/cloudfoundry/python-buildpack-release/releases/tag/${version} -- name: r - notes: https://github.com/cloudfoundry/r-buildpack-release/releases/tag/${version} -- name: ruby - notes: https://github.com/cloudfoundry/ruby-buildpack-release/releases/tag/${version} -- name: staticfile - notes: https://github.com/cloudfoundry/staticfile-buildpack-release/releases/tag/${version} -core: -- name: bpm - notes: https://github.com/cloudfoundry/bpm-release/releases/tag/${version} -- name: capi - notes: https://github.com/cloudfoundry/capi-release/releases/tag/${version} -- name: cf-networking - notes: https://github.com/cloudfoundry/cf-networking-release/releases/tag/${version} -- name: cf-smoke-tests - notes: https://github.com/cloudfoundry/cf-smoke-tests-release/releases/tag/${version} -- name: cflinuxfs3 - notes: https://github.com/cloudfoundry/cflinuxfs3-release/releases/tag/${version} -- name: cf-cli - notes: https://github.com/bosh-packages/cf-cli-release/releases/tag/${version} -- name: diego - notes: https://github.com/cloudfoundry/diego-release/releases/tag/${version} -- name: garden-runc - notes: https://github.com/cloudfoundry/garden-runc-release/releases/tag/${version} -- name: loggregator - notes: https://github.com/cloudfoundry/loggregator-release/releases/tag/${version} -- name: loggregator-agent - notes: https://github.com/cloudfoundry/loggregator-agent-release/releases/tag/${version} -- name: log-cache - notes: https://github.com/cloudfoundry/log-cache-release/releases/tag/${version} -- name: nats - notes: https://github.com/cloudfoundry/nats-release/releases/tag/${version} -- name: routing - notes: https://github.com/cloudfoundry/routing-release/releases/tag/${version} -- name: statsd-injector - notes: https://github.com/cloudfoundry/statsd-injector-release/releases/tag/${version} -- name: cf-syslog-drain - notes: https://github.com/cloudfoundry/cf-syslog-drain-release/releases/tag/${version} -- name: uaa - notes: https://github.com/cloudfoundry/uaa-release/releases/tag/${version} -- name: silk - notes: https://github.com/cloudfoundry/silk-release/releases/tag/${version} -- name: bosh-dns-aliases - notes: https://github.com/cloudfoundry/bosh-dns-aliases-release/releases/tag/${version} -- name: cflinuxfs2 - notes: https://github.com/cloudfoundry/cflinuxfs2-release/releases/tag/${version} -- name: app-autoscaler - notes: https://github.com/cloudfoundry-incubator/app-autoscaler-release/releases/tag/${version} -- name: nfs-volume - notes: https://github.com/cloudfoundry/nfs-volume-release/releases/tag/${version} -- name: mapfs - notes: https://github.com/cloudfoundry/mapfs-release/releases/tag/${version} -- name: postgres - notes: https://github.com/cloudfoundry-community/postgres-boshrelease/releases/tag/${version} -- name: haproxy - notes: https://github.com/cloudfoundry-incubator/haproxy-boshrelease/releases/tag/${version} -