From 63b6e0faa2d7df238b093bdd51ab775d40e568f2 Mon Sep 17 00:00:00 2001 From: Sean Arms <67096+lesserwhirls@users.noreply.github.com> Date: Mon, 16 Dec 2024 16:12:24 -0700 Subject: [PATCH] migrate aws jenkins environment to fully dockerized environment Also, upgrade packages, available JDKs, etc. --- README.md | 154 +++++++--------- .../roles/general-packages/vars/main.yml | 1 + .../roles/security/tasks/users-and-groups.yml | 8 + .../ansible/roles/security/vars/main.yml | 3 + .../roles/temurin/tasks/fetch_remote.yml | 2 +- .../ansible/roles/temurin/tasks/main.yml | 12 -- .../ansible/roles/temurin/vars/main.yml | 2 +- .../tasks/main.yml | 23 +-- .../vars/main.yml | 10 +- .../ansible/roles/zulu/vars/main.yml | 3 +- packer/provisioners/ansible/site.yml | 19 +- packer/provisioners/file/jenkins-agent.sh | 149 ++++++++++++++++ .../scripts/bootstrap_first_aws.sh | 4 - .../scripts/bootstrap_last_aws.sh | 12 -- packer/thredds-test-env.json | 167 ------------------ 15 files changed, 234 insertions(+), 335 deletions(-) create mode 100755 packer/provisioners/file/jenkins-agent.sh delete mode 100644 packer/provisioners/scripts/bootstrap_first_aws.sh delete mode 100644 packer/provisioners/scripts/bootstrap_last_aws.sh delete mode 100644 packer/thredds-test-env.json diff --git a/README.md b/README.md index 6db7561..8680730 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,20 @@ # THREDDS Test Environment -A packer + ansible project to build and deploy an Amazon Machine Image and Docker Container for use by the various THREDDS projects at Unidata for running automated tests. -The AMI is used by our Jenkins server to launch worker nodes for running tests. -The Docker Image will be used as the basis for a custom GitHub action for testing the various THREDDS projects. -Both image types are based off the latest available image, AMI or Docker, for `ubuntu 20.04`. -The AMI and docker images produced by packer are both called "thredds-test-environment". +A packer + ansible project to build Docker images for use by the various THREDDS projects at Unidata for running automated tests. +The Docker images are used as the basis for Jenkins build nodes as well as a custom GitHub action for testing the various THREDDS projects. +The image is based off the latest available image for `ubuntu 20.04`. +The docker images produced by packer are called "thredds-test-environment" (for use by Jenkins) of `thredds-test-action` (for use by the GitHub action). ## Requirements * Packer ([download](https://www.packer.io/downloads)) -* Docker, for building the Docker Image ([download](https://www.docker.com/products/docker-desktop)) -* AWS EC2 Credentials, for building the AMI. - * The packer configuration expects the AWS credentials to be stored under a profile named `ucar-unidata-profile` - One way of setting this up would be to have an entry in the file `/.aws/credentials` that looks like the following: - ~~~ - [ucar-unidata-profile] - aws_access_key_id=XXXXXXXXX - aws_secret_access_key=XXXXXXXXX - ~~~ - See the [packer docs](https://www.packer.io/docs/builders/amazon#authentication) for more information. +* Docker, for building the Docker images ([download](https://www.docker.com/products/docker-desktop)) + +If this is your first time running the build, you will need to initialize packer using: + +~~~ +packer init . +~~~ While this project's packer configuration relies heavily on Ansible, you do not need to have Ansible installed locally. We utilize the `ansible-local` provisioner of Packer, which means Ansible is run on the remote/guest machine, and not on the build machine (the machine running packer). @@ -27,52 +23,58 @@ However, Since Ansible does not support the use of Windows systems as a control ## Building the images -To generate the golden AMI and Docker images, start off by validating the packer configuration by first moving into the `packer/` directory and then running: +To generate the golden Docker images, start off by validating the packer configuration by first moving into the `packer/` directory and then running: ~~~bash -packer validate thredds-test-env.json +packer validate . ~~~ Once validated, you may run the build using: ~~~bash -packer build --only= thredds-test-env.json +packer build --only= thredds-test-env.pkr.hcl ~~~ `` will one or more (separated by commas) of the following: -* `ami`: Provision an AWS EC2 instance and generate an AMI (`thredds-test-environment-`) -* `docker-commit`: Provision a Docker container and generate and tag a local Docker Image (`docker.unidata.ucar.edu/thredds-test-environment:latest`). -* `docker-export`: Provision a Docker container and generate a local Docker Image as a file (`image.tar`). -* `docker-github-action`: Provision a Docker container for use with GitHub Actions and publish to the GitHub Package Repository (runs using manual `workflow_dispatch` trigger on github actions). -* `docker-github-action-nexus`: Same as `docker-github-action`, but publish to the Unidata Nexus Repository. +* `docker.docker-jenkins`: Provision a Docker container and generate and tag a local Docker image (`docker.unidata.ucar.edu/thredds-test-environment:latest`). +* `docker.docker-export`: Provision a Docker container and generate a local Docker image as a file (`image.tar`). +* `docker.docker-github-action`: Provision a Docker container for use with GitHub Actions and publish to the GitHub Package Repository (runs using manual `workflow_dispatch` trigger on github actions). +* `docker.docker-github-action-nexus`: Same as `docker-github-action`, but tags for publishing to the Unidata Nexus Repository. -Typically, we would run the following to update the AMI and Github Action Docker Image (hosted on Nexus) at the same time: +Typically, we would run the following to update the Jenkins and Github Action Docker images (hosted on Nexus) at the same time: ~~~bash -packer build --only=ami,docker-github-action-nexus thredds-test-env.json +packer build --only=docker.docker-jenkins,docker.docker-github-action-nexus thredds-test-env.pkr.hcl ~~~ -The Docker Image build takes about 1 hour to create, where as the AMI build takes around 30 minutes. +The Docker image builds takes about 1 hour to create. Packer will run the builders in parallel, so the total time to create the `thredds-test-environment` images is around an hour. -If using `docker-commit`, then once the image is built you can test out the environment by using: +If using `docker-jenkins`, then once the image is built you can test out the environment by using: ~~~bash docker run -i -t --rm docker.unidata.ucar.edu/thredds-test-environment:latest ~~~ +Note that images are not pushed as part of this build process. +Pushes can be done via the normal docker mechanisms, e.g. `docker image push docker.unidata.ucar.edu/thredds-test-environment:latest`. + ## Project layout -Inside the packer directory is a file called `thredds-test-env.json`. -This contains the packer configuration for the builders (amazon-ebs builder, docker builder), the provisioners (shell scripts, ansible playbooks), and the post-processors (tagging the docker image). +Inside the packer directory is a file called `thredds-test-env.pkr.hcl`. +This contains the packer configuration for the builders (docker builder), the provisioners (shell scripts, ansible playbooks), and the post-processors (tagging the docker image). The provisioners directory contains the provisioner configurations used by packer. -There are two types of provisioners used by the packer configuration: ansible and scripts. +There are three types of provisioners used by the packer configuration: ansible, file, and scripts. + +The `files/` directory contains Docker entrypoint scripts: + +* `entrypoint.sh`: drives the GitHub Action. +* `jenkins-agent.sh`: connects a container to the jenkins control . + The `scripts/` directory contains the following shell scripts: * `bootstrap-common.sh`: Install and configure ansible (common to both builders) -* `bootstrap_first_aws.sh`: This will run first on AWS. Waits for cloud-init to finish before running `bootstrap-common.sh`. -* `bootstrap_last_aws.sh`: This runs immediately after `bootstrap-common.sh` on AWS. Installs a system-wide JRE, which is needed when using the AMS as a Jenkins worker node. * `cleanup.sh`: Runs after the ansible provisioner and cleans up the `apt` cache, as well as some general build environment things. The `ansible/` directory contains the ansible playbooks that are used to configure the testing environment. @@ -86,7 +88,7 @@ We use the following roles when provisioning our images: * `cleanup`: General cleanup related tasks, such as remove the temporary build directory and running `ldconfig` * `general-packages`: Install general packages needed for the build environment using the OS package manager. * `gradle-builds-cache-bootstrap`: Pull in and build netCDF-Java to populate the gradle cache for user ubuntu. -* `init`: Initialize the build environment by ensuring the temporary ansible build directory exists. +* `init`: Initialize the build environment by ensuring the temporary ansible build directory exists. * `libnetcdf-and-deps`: Configure, build, and install `zlib`, `HDF5`, and `netCDF-C`. * `maven`: Obtain and install the Apache Maven software project management and comprehension tool. * `miniconda`: Obtain and install the Anaconda miniconda python distribution. @@ -95,9 +97,9 @@ We use the following roles when provisioning our images: * Add a default `maven-settings.xml` file configured to publish to the Unidata artifacts server. * Configure `ssh` (uses modified version of a task from Jeff Geerling's [ansible-role-security](https://github.com/geerlingguy/ansible-role-security) project - see `packer/provisioners/ansible/roles/security/README.md`). * Configure a system wide bash environment. -* `temurin`: Obtain and install Temurin 8, 11, and 14. +* `temurin`: Obtain and install LTS versions of Temurin. * `test-data-mount-prep`: Prepare the environment to mount the `thredds-test-data` datasets when available (currently used on Jenkins worker nodes). -* `zulu`: Obtain and install Zulu 8, 11, and 14. +* `zulu`: Obtain and install LTS versions of Zulu. We also use a role from [Ansible Galaxy](https://galaxy.ansible.com/) to setup a Ruby environment ([geerlingguy.ruby](https://galaxy.ansible.com/geerlingguy/ruby)). @@ -122,19 +124,20 @@ We also use a role from [Ansible Galaxy](https://galaxy.ansible.com/) to setup a * Temurin (latest version available from adoptium.net) * 8 (`/usr/thredds-test-environment/temurin8`) * 11 (`/usr/thredds-test-environment/temurin11`) - * 16 (`/usr/thredds-test-environment/temurin16`) + * 17 (`/usr/thredds-test-environment/temurin17`) + * 21 (`/usr/thredds-test-environment/temurin21`) + * Zulu (latest version available from azul.com) * 8 (`/usr/thredds-test-environment/zulu8`) * 11 (`/usr/thredds-test-environment/zulu11`) - * 14 (`/usr/thredds-test-environment/zulu14`) - * 16 (`/usr/thredds-test-environment/zulu16`) + * 17 (`/usr/thredds-test-environment/zulu17`) + * 21 (`/usr/thredds-test-environment/zulu21`) ### Ruby * ruby (via [geerlingguy.ruby](https://galaxy.ansible.com/geerlingguy/ruby) from [Ansible Galaxy](https://galaxy.ansible.com/)) ### Bash functions: - * `select-java ` (where version is 8, 11, 14, or 16, and vendor is `temurin` or `zulu`) - * note that Temurin does not have a version 14 binary at this time. + * `select-java ` (where version is 8, 11, 17, or 21, and vendor is `temurin` or `zulu`) * `activate-conda` * `get_pw ` @@ -151,53 +154,26 @@ We also use a role from [Ansible Galaxy](https://galaxy.ansible.com/) to setup a ### Docker Image ~~~ - docker-commit: Tuesday 26 January 2021 02:31:10 +0000 (0:00:02.518) 0:27:27.982 ******* - docker-commit: =============================================================================== - docker-commit: Wait for the gradle builds to complete. ------------------------------- 502.58s - docker-commit: libnetcdf-and-deps : Install hdf5. ------------------------------------ 196.71s - docker-commit: general-packages : Install os managed tools. -------------------------- 156.95s - docker-commit: general-packages : Install os managed tools. -------------------------- 147.62s - docker-commit: temurin : Fetch latest Temurin Java builds. ------------------------- 142.41s - docker-commit: general-packages : Install os managed tools. -------------------------- 129.50s - docker-commit: geerlingguy.ruby : Install ruby and other required dependencies. ------- 85.59s - docker-commit: libnetcdf-and-deps : Configure netCDF-c. ------------------------------- 47.61s - docker-commit: temurin : Unpack Temurin Java Installations. ------------------------- 47.37s - docker-commit: maven : Fetch Latest Maven. -------------------------------------------- 32.04s - docker-commit: gradle-builds-cache-bootstrap : Fetch latest commits from github. ------ 31.14s - docker-commit: libnetcdf-and-deps : Download and unpack hdf5. ------------------------- 21.72s - docker-commit: libnetcdf-and-deps : Configure hdf5. ----------------------------------- 20.46s - docker-commit: libnetcdf-and-deps : Download and unpack zlib. ------------------------- 17.42s - docker-commit: miniconda : Download and unpack miniconda. ----------------------------- 11.38s - docker-commit: libnetcdf-and-deps : Install netCDF-c. --------------------------------- 10.12s - docker-commit: cleanup : Remove packages that are not needed in final environment. ----- 4.45s - docker-commit: geerlingguy.ruby : Install rubygems. ------------------------------------ 3.31s - docker-commit: security : Update SSH configuration to be more secure. ------------------ 2.93s - docker-commit: libnetcdf-and-deps : Download and unpack netcdf-c. ---------------------- 2.69s -~~~ - -### AMI - -~~~ - ami: Tuesday 26 January 2021 02:32:50 +0000 (0:00:00.709) 0:29:56.463 ******* - ami: =============================================================================== - ami: Wait for the gradle builds to complete. ------------------------------- 606.89s - ami: Wait for the HDF5 async test task to complete. ------------------------ 455.60s - ami: libnetcdf-and-deps : Install hdf5. ------------------------------------ 335.69s - ami: temurin : Unpack Temurin Java Installations. ------------------------- 55.85s - ami: temurin : Fetch latest Temurin Java builds. -------------------------- 38.67s - ami: gradle-builds-cache-bootstrap : Fetch latest commits from github. ------ 32.56s - ami: geerlingguy.ruby : Install ruby and other required dependencies. ------- 31.75s - ami: Wait for netcdf-c async test task to complete. ------------------------- 31.35s - ami: general-packages : Install os managed tools. --------------------------- 28.81s - ami: libnetcdf-and-deps : Install netCDF-c. --------------------------------- 26.35s - ami: libnetcdf-and-deps : Configure netCDF-c. ------------------------------- 20.54s - ami: general-packages : Install os managed tools. --------------------------- 16.75s - ami: libnetcdf-and-deps : Configure hdf5. ----------------------------------- 16.54s - ami: maven : Fetch Latest Maven. -------------------------------------------- 12.57s - ami: libnetcdf-and-deps : Download and unpack hdf5. -------------------------- 7.72s - ami: security : Update SSH configuration to be more secure. ------------------ 6.17s - ami: libnetcdf-and-deps : Test HDF5. ----------------------------------------- 5.53s - ami: libnetcdf-and-deps : Install zlib. -------------------------------------- 4.84s - ami: cleanup : Remove packages that are not needed in final environment. ----- 4.82s - ami: general-packages : Install os managed tools. ---------------------------- 3.44s + docker.docker-jenkins: Friday 22 November 2024 21:12:32 +0000 (0:00:02.871) 0:20:20.107 ******* + docker.docker-jenkins: =============================================================================== + docker.docker-jenkins: Wait for the HDF5 async test task to complete. ------------------------ 362.21s + docker.docker-jenkins: libnetcdf-and-deps : Install hdf5. ------------------------------------ 272.84s + docker.docker-jenkins: libnetcdf-and-deps : Configure netCDF-c. ------------------------------ 128.28s + docker.docker-jenkins: libnetcdf-and-deps : Configure hdf5. ---------------------------------- 114.44s + docker.docker-jenkins: zulu : Fetch latest Zulu Java builds. ---------------------------------- 54.52s + docker.docker-jenkins: temurin : Fetch latest Temurin Java builds. ---------------------------- 46.20s + docker.docker-jenkins: zulu : Unpack Zulu Java Installations. --------------------------------- 35.03s + docker.docker-jenkins: libnetcdf-and-deps : Install netCDF-c. --------------------------------- 28.84s + docker.docker-jenkins: general-packages : Install os managed tools. --------------------------- 28.32s + docker.docker-jenkins: temurin : Unpack Temurin Java Installations. --------------------------- 26.71s + docker.docker-jenkins: general-packages : Install os managed tools. --------------------------- 24.31s + docker.docker-jenkins: geerlingguy.ruby : Install ruby and other required dependencies. ------- 12.96s + docker.docker-jenkins: general-packages : Install os managed tools. --------------------------- 12.77s + docker.docker-jenkins: miniconda : Download and unpack miniconda. ------------------------------ 9.31s + docker.docker-jenkins: libnetcdf-and-deps : Install zlib. -------------------------------------- 5.46s + docker.docker-jenkins: libnetcdf-and-deps : Configure zlib. ------------------------------------ 4.13s + docker.docker-jenkins: security : Update SSH configuration to be more secure. ------------------ 3.78s + docker.docker-jenkins: libnetcdf-and-deps : Download and unpack hdf5. -------------------------- 3.59s + docker.docker-jenkins: zulu : Read versions of installed Zulu. --------------------------------- 3.34s + docker.docker-jenkins: cleanup : Remove packages that are not needed in final environment. ----- 3.07s ~~~ diff --git a/packer/provisioners/ansible/roles/general-packages/vars/main.yml b/packer/provisioners/ansible/roles/general-packages/vars/main.yml index 6b915a1..963667f 100644 --- a/packer/provisioners/ansible/roles/general-packages/vars/main.yml +++ b/packer/provisioners/ansible/roles/general-packages/vars/main.yml @@ -5,3 +5,4 @@ needed_packages: - git - fonts-dejavu # Needed by UI tests because OpenJDK does not ship with fonts - fontconfig # Needed by UI tests because OpenJDK does not ship with fonts + - curl # Needed by static analysis pipelines diff --git a/packer/provisioners/ansible/roles/security/tasks/users-and-groups.yml b/packer/provisioners/ansible/roles/security/tasks/users-and-groups.yml index 005b3bb..75298d4 100644 --- a/packer/provisioners/ansible/roles/security/tasks/users-and-groups.yml +++ b/packer/provisioners/ansible/roles/security/tasks/users-and-groups.yml @@ -1,6 +1,14 @@ --- +- name: Create groups. + group: + name: "{{ item.name }}" + gid: "{{ item.gid }}" + loop: "{{ users }}" + - name: Create users. user: name: "{{ item.name }}" shell: "{{ item.shell | default('/bin/bash') }}" + uid: "{{ item.uid }}" + group: "{{ item.name }}" loop: "{{ users }}" diff --git a/packer/provisioners/ansible/roles/security/vars/main.yml b/packer/provisioners/ansible/roles/security/vars/main.yml index cb4611a..6b68604 100644 --- a/packer/provisioners/ansible/roles/security/vars/main.yml +++ b/packer/provisioners/ansible/roles/security/vars/main.yml @@ -7,6 +7,9 @@ custom_bash_profile_dest: "/etc/profile.d/{{ custom_bash_profile_src }}" users: - name: "{{ thredds_test_user }}" shell: /bin/bash + uid: 395 + gid: 395 + group: "{{ thredds_test_user }}" files2copy: - label: "maven settings" diff --git a/packer/provisioners/ansible/roles/temurin/tasks/fetch_remote.yml b/packer/provisioners/ansible/roles/temurin/tasks/fetch_remote.yml index 36d374e..fd549f0 100644 --- a/packer/provisioners/ansible/roles/temurin/tasks/fetch_remote.yml +++ b/packer/provisioners/ansible/roles/temurin/tasks/fetch_remote.yml @@ -1,7 +1,7 @@ --- - name: Fetch latest Temurin Java builds. uri: - url: "https://api.adoptium.net/v3/binary/latest/{{ item }}/ga/linux/x64/jdk/hotspot/normal/adoptium?project=jdk" + url: "https://api.adoptium.net/v3/binary/latest/{{ item }}/ga/linux/x64/jdk/hotspot/normal/eclipse?project=jdk" method: GET follow_redirects: safe dest: "/tmp/{{ base_install_name }}{{ item }}.tar.gz" diff --git a/packer/provisioners/ansible/roles/temurin/tasks/main.yml b/packer/provisioners/ansible/roles/temurin/tasks/main.yml index 610a108..a6b656e 100644 --- a/packer/provisioners/ansible/roles/temurin/tasks/main.yml +++ b/packer/provisioners/ansible/roles/temurin/tasks/main.yml @@ -7,20 +7,8 @@ loop_control: label: "Create Directory for Temurin {{ item }} Installation" -# Not sure why, but sometimes we cannot fetch Temurin 11 from aws (8 works -# just fine). When that happens, we can use a local copy of the tar.gz files -# rather than using the web API to fetch them. Uncomment the following task, -# as well as the "when" statement under the fetch_remote task below to use -# the local workflow. You will need to run one of the scripts in ../files -# prior to running packer to make sure the tar.gz files exists and are -# up-to-date. -#- import_tasks: fetch_local.yml -# tags: [ fetch-local ] -# when: packer_builder_type == "amazon-ebs" - - import_tasks: fetch_remote.yml tags: [ fetch-remote ] - #when: packer_builder_type != "amazon-ebs" - name: Unpack Temurin Java Installations. unarchive: diff --git a/packer/provisioners/ansible/roles/temurin/vars/main.yml b/packer/provisioners/ansible/roles/temurin/vars/main.yml index d811d8d..d8698e7 100644 --- a/packer/provisioners/ansible/roles/temurin/vars/main.yml +++ b/packer/provisioners/ansible/roles/temurin/vars/main.yml @@ -4,5 +4,5 @@ base_install_name: temurin java_versions: - 8 - 11 - - 16 - 17 + - 21 diff --git a/packer/provisioners/ansible/roles/thredds-test-data-mount-prep/tasks/main.yml b/packer/provisioners/ansible/roles/thredds-test-data-mount-prep/tasks/main.yml index 2a891f2..2fcff4b 100644 --- a/packer/provisioners/ansible/roles/thredds-test-data-mount-prep/tasks/main.yml +++ b/packer/provisioners/ansible/roles/thredds-test-data-mount-prep/tasks/main.yml @@ -1,25 +1,8 @@ --- -- name: List information about '{{ volume_device_name }}' block device. - command: lsblk {{ volume_device_name }} - register: lsblk_output - changed_when: false # Task is purely informational; it doesn't change anything. - ignore_errors: true # Task is expected to fail when device doesn't exist. We handle that case below. - -- name: Determine if '{{ volume_device_name }}' block device exists. - set_fact: - volume_device_exists: "{{ lsblk_output.rc == 0 }}" - - name: Ensure the '{{ cdmUnitTest_mount_dir }}' exists. file: path: "{{ cdmUnitTest_mount_dir }}" state: directory - -- name: Add entry for '{{ volume_device_name }}' to /etc/fstab, and mount it if it's available. - mount: - src: "{{ volume_device_name }}" - path: "{{ cdmUnitTest_mount_dir }}" - fstype: "{{ filesystem_type }}" - opts: defaults,nofail # Don't fail when the device isn't available, which will happen in dev and test environments. - dump: "0" - passno: "2" - state: "{{ volume_device_exists | ternary('mounted', 'present') }}" + owner: "{{ thredds_test_user }}" + group: "{{ thredds_test_user }}" + mode: "0755" diff --git a/packer/provisioners/ansible/roles/thredds-test-data-mount-prep/vars/main.yml b/packer/provisioners/ansible/roles/thredds-test-data-mount-prep/vars/main.yml index 6d990fd..1f51ee2 100644 --- a/packer/provisioners/ansible/roles/thredds-test-data-mount-prep/vars/main.yml +++ b/packer/provisioners/ansible/roles/thredds-test-data-mount-prep/vars/main.yml @@ -1,11 +1,3 @@ --- -# The name of the device to which AWS will mount the cdmUnitTest snapshot. We set this in Jenkins under: -# "Manage Jenkins"->"Configure System"->"Cloud"->"Amazon EC2"->"AMIs"->"Advanced..."->"Block device mapping". -# For example, we have something like: "/dev/xvdf=::true" -volume_device_name: /dev/xvdf - -# The directory at which the cdmUnitTest EBS snapshot should be mounted. +# The directory at which the THREDDS test data should be mounted. cdmUnitTest_mount_dir: /share/testdata/cdmUnitTest - -# The filesystem of the snapshot. -filesystem_type: ext4 diff --git a/packer/provisioners/ansible/roles/zulu/vars/main.yml b/packer/provisioners/ansible/roles/zulu/vars/main.yml index 08d5308..9e76f21 100644 --- a/packer/provisioners/ansible/roles/zulu/vars/main.yml +++ b/packer/provisioners/ansible/roles/zulu/vars/main.yml @@ -4,6 +4,5 @@ base_install_name: zulu java_versions: - 8 - 11 - - 14 - - 16 - 17 + - 21 diff --git a/packer/provisioners/ansible/site.yml b/packer/provisioners/ansible/site.yml index 5da05ce..6e0209a 100644 --- a/packer/provisioners/ansible/site.yml +++ b/packer/provisioners/ansible/site.yml @@ -11,7 +11,7 @@ name: init tags: [ init ] - # The next two tasks take a good amount of time to run, and as a result + # The next two tasks take a good amount of time to run, and as a result # utilize asynchronous task execution. We run these early on in the # playbook to keep things moving. - name: Include 'libnetcdf-and-deps' role. @@ -62,11 +62,6 @@ name: thredds-test-data-mount-prep tags: [ thredds-test-data-mount-prep ] - - name: Include 'gradle cache bootstrap' role. - include_role: - name: gradle-builds-cache-bootstrap - tags: [ gradle-builds-cache-bootstrap ] - # Wait for the asynchronous tasks to complete. - name: Wait for zlib async test task to complete. when: (async_zlib is defined) and (async_zlib|length > 0) @@ -95,18 +90,6 @@ retries: "{{ netcdf_c_async_check_interval }}" delay: "{{ netcdf_c_async_check_retries }}" - - name: "Wait for the gradle builds to complete." - when: (async_bootstrap_gradle_cache is defined) and (async_bootstrap_gradle_cache|length > 0) - async_status: - jid: "{{ async_bootstrap_gradle_cache_item.ansible_job_id }}" - loop: "{{ async_bootstrap_gradle_cache.results }}" - loop_control: - loop_var: "async_bootstrap_gradle_cache_item" - register: async_poll_results - until: async_poll_results.finished - retries: "{{ bootstrap_gradle_cache_async_check_retries }}" - delay: "{{ bootstrap_gradle_cache_async_check_interval }}" - - name: Wait for the HDF5 async test task to complete. when: (async_hdf5 is defined) and (async_hdf5|length > 0) async_status: diff --git a/packer/provisioners/file/jenkins-agent.sh b/packer/provisioners/file/jenkins-agent.sh new file mode 100755 index 0000000..dcaf1e0 --- /dev/null +++ b/packer/provisioners/file/jenkins-agent.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash + +JENKINS_JAVA_BIN="/usr/thredds-test-environment/temurin21/bin/java" +JENKINS_AGENT_FILE="/home/jenkins/agent.jar" +JENKINS_WEB_SOCKET=true +JENKINS_AGENT_WORKDIR="/home/jenkins/workspace" + +# copied from https://github.com/jenkinsci/docker-agent/blob/master/jenkins-agent +# modified above these lines to to use custom image settings + +# The MIT License +# +# Copyright (c) 2015-2020, CloudBees, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +# Usage jenkins-agent.sh [options] -url http://jenkins -secret [SECRET] -name [AGENT_NAME] +# Optional environment variables : +# * JENKINS_JAVA_BIN : Java executable to use instead of the default in PATH or obtained from JAVA_HOME +# * JENKINS_JAVA_OPTS : Java Options to use for the remoting process, otherwise obtained from JAVA_OPTS +# * JENKINS_AGENT_FILE : Jenkins agent jar file location, /usr/share/jenkins/agent.jar will be used if this is not set +# * REMOTING_OPTS : Generic way to pass additional CLI options to agent.jar (see -help) +# +# Deprecated environment variables (prefer setting REMOTING_OPTS) +# * JENKINS_TUNNEL : HOST:PORT for a tunnel to route TCP traffic to jenkins host, when jenkins can't be directly accessed over network +# * JENKINS_URL : alternate jenkins URL +# * JENKINS_SECRET : agent secret, if not set as an argument +# * JENKINS_AGENT_NAME : agent name, if not set as an argument +# * JENKINS_AGENT_WORKDIR : agent work directory, if not set by optional parameter -workDir +# * JENKINS_WEB_SOCKET: true if the connection should be made via WebSocket rather than TCP +# * JENKINS_DIRECT_CONNECTION: Connect directly to this TCP agent port, skipping the HTTP(S) connection parameter download. +# Value: ":" +# * JENKINS_INSTANCE_IDENTITY: The base64 encoded InstanceIdentity byte array of the Jenkins controller. When this is set, +# the agent skips connecting to an HTTP(S) port for connection info. +# * JENKINS_PROTOCOLS: Specify the remoting protocols to attempt when instanceIdentity is provided. + +if [ $# -eq 1 ] && [ "${1#-}" = "$1" ] ; then + + # if `docker run` only has one arguments and it is not an option as `-help`, we assume user is running alternate command like `bash` to inspect the image + exec "$@" + +else + + # if -tunnel is not provided, try env vars + case "$@" in + *"-tunnel "*) ;; + *) + if [ ! -z "$JENKINS_TUNNEL" ]; then + TUNNEL="-tunnel $JENKINS_TUNNEL" + fi ;; + esac + + # if -workDir is not provided, try env vars + if [ ! -z "$JENKINS_AGENT_WORKDIR" ]; then + case "$@" in + *"-workDir"*) echo "Warning: Work directory is defined twice in command-line arguments and the environment variable" ;; + *) + WORKDIR="-workDir $JENKINS_AGENT_WORKDIR" ;; + esac + fi + + if [ -n "$JENKINS_URL" ]; then + URL="-url $JENKINS_URL" + fi + + if [ -n "$JENKINS_NAME" ]; then + JENKINS_AGENT_NAME="$JENKINS_NAME" + fi + + if [ "$JENKINS_WEB_SOCKET" = true ]; then + WEB_SOCKET=-webSocket + fi + + if [ -n "$JENKINS_PROTOCOLS" ]; then + PROTOCOLS="-protocols $JENKINS_PROTOCOLS" + fi + + if [ -n "$JENKINS_DIRECT_CONNECTION" ]; then + DIRECT="-direct $JENKINS_DIRECT_CONNECTION" + fi + + if [ -n "$JENKINS_INSTANCE_IDENTITY" ]; then + INSTANCE_IDENTITY="-instanceIdentity $JENKINS_INSTANCE_IDENTITY" + fi + + if [ "$JENKINS_JAVA_BIN" ]; then + JAVA_BIN="$JENKINS_JAVA_BIN" + else + # if java home is defined, use it + JAVA_BIN="java" + if [ "$JAVA_HOME" ]; then + JAVA_BIN="$JAVA_HOME/bin/java" + fi + fi + + if [ "$JENKINS_JAVA_OPTS" ]; then + JAVA_OPTIONS="$JENKINS_JAVA_OPTS" + else + # if JAVA_OPTS is defined, use it + if [ "$JAVA_OPTS" ]; then + JAVA_OPTIONS="$JAVA_OPTS" + fi + fi + + if [ "$JENKINS_AGENT_FILE" ]; then + AGENT_FILE="$JENKINS_AGENT_FILE" + else + AGENT_FILE="/usr/share/jenkins/agent.jar" + fi + + # if both required options are defined, do not pass the parameters + if [ -n "$JENKINS_SECRET" ]; then + case "$@" in + *"${JENKINS_SECRET}"*) echo "Warning: SECRET is defined twice in command-line arguments and the environment variable" ;; + *) + SECRET="-secret ${JENKINS_SECRET}" ;; + esac + fi + + if [ -n "$JENKINS_AGENT_NAME" ]; then + case "$@" in + *"${JENKINS_AGENT_NAME}"*) echo "Warning: AGENT_NAME is defined twice in command-line arguments and the environment variable" ;; + *) + AGENT_NAME="-name ${JENKINS_AGENT_NAME}" ;; + esac + fi + + #TODO: Handle the case when the command-line and Environment variable contain different values. + #It is fine it blows up for now since it should lead to an error anyway. + + exec $JAVA_BIN $JAVA_OPTIONS -jar $AGENT_FILE $SECRET $AGENT_NAME $TUNNEL $URL $WORKDIR $WEB_SOCKET $DIRECT $PROTOCOLS $INSTANCE_IDENTITY $REMOTING_OPTS "$@" + +fi \ No newline at end of file diff --git a/packer/provisioners/scripts/bootstrap_first_aws.sh b/packer/provisioners/scripts/bootstrap_first_aws.sh deleted file mode 100644 index df8e992..0000000 --- a/packer/provisioners/scripts/bootstrap_first_aws.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/bash -set -e - -while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo '...waiting for cloud-init to finish...'; sleep 1; done diff --git a/packer/provisioners/scripts/bootstrap_last_aws.sh b/packer/provisioners/scripts/bootstrap_last_aws.sh deleted file mode 100644 index 7480569..0000000 --- a/packer/provisioners/scripts/bootstrap_last_aws.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/bash -set -ex - -# Check that we are root, and if not, use sudo to run commands. -SUDO_CMD="" -if [ "$EUID" -ne 0 ]; then - SUDO_CMD="sudo" -fi - -# Need openjdk-xx-jre-headless so that jenkins can connect to worker nodes -# using this ami. -${SUDO_CMD} apt install --yes openjdk-17-jre-headless diff --git a/packer/thredds-test-env.json b/packer/thredds-test-env.json deleted file mode 100644 index d28fdca..0000000 --- a/packer/thredds-test-env.json +++ /dev/null @@ -1,167 +0,0 @@ -{ - "variables": { - "aws_access_key": "", - "aws_secret_key": "", - "thredds_test_user": "ubuntu", - "base_ami": "ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*", - "base_docker_image": "ubuntu:20.04" - }, - "sensitive-variables": [ - "aws_access_key", - "aws_secret_key" - ], - "builders": [ - { - "name": "ami", - "type": "amazon-ebs", - "access_key": "{{user `aws_access_key`}}", - "secret_key": "{{user `aws_secret_key`}}", - "region": "us-east-1", - "profile": "ucar-unidata-profile", - "ami_name": "thredds-test-environment-{{isotime | clean_resource_name}}", - "instance_type": "t3.large", - "ssh_username": "{{user `thredds_test_user`}}", - "launch_block_device_mappings": { - "device_name": "/dev/sda1", - "volume_type": "gp2", - "delete_on_termination": true, - "volume_size": 16 - }, - "source_ami_filter": { - "filters": { - "virtualization-type": "hvm", - "name": "{{user `base_ami`}}", - "root-device-type": "ebs" - }, - "owners": ["099720109477"], - "most_recent": true - }, - "tags": { - "OS_Version": "Ubuntu Focal (20.04)", - "Release": "Latest", - "Base_AMI_Name": "{{ .SourceAMIName }}", - "Group": "THREDDS", - "Name": "Jenkins Worker {{isotime | clean_resource_name}}", - "Project": "Jenkins" - } - }, - { - "name": "docker-export", - "type": "docker", - "image": "{{user `base_docker_image`}}", - "export_path": "image.tar", - "changes": [ - "USER {{user `thredds_test_user`}}", - "ENTRYPOINT [\"/bin/bash\"]" - ] - }, - { - "name": "docker-commit", - "type": "docker", - "image": "{{user `base_docker_image`}}", - "commit": true, - "changes": [ - "USER {{user `thredds_test_user`}}", - "ENTRYPOINT [\"/bin/bash\"]" - ] - }, - { - "name": "docker-github-action", - "type": "docker", - "image": "{{user `base_docker_image`}}", - "commit": true, - "changes": [ - "USER root", - "ENV GITHUB_ACTIONS=\"YEP\"", - "ENTRYPOINT [\"/entrypoint.sh\"]" - ] - }, - { - "name": "docker-github-action-nexus", - "type": "docker", - "image": "{{user `base_docker_image`}}", - "commit": true, - "changes": [ - "USER root", - "ENV GITHUB_ACTIONS=\"YEP\"", - "ENTRYPOINT [\"/entrypoint.sh\"]" - ] - } - ], - "provisioners": [ - { - "type": "shell", - "script": "provisioners/scripts/bootstrap_first_aws.sh", - "only": "ami" - }, - { - "type": "shell", - "script": "provisioners/scripts/bootstrap-common.sh" - }, - { - "type": "shell", - "script": "provisioners/scripts/bootstrap_last_aws.sh", - "only": "ami" - }, - { - "type": "file", - "source": "provisioners/file/entrypoint.sh", - "destination": "/entrypoint.sh", - "only": [ "docker-github-action", "docker-github-action-nexus" ] - }, - { - "type": "ansible-local", - "group_vars": "provisioners/ansible/group_vars", - "extra_arguments": [ - "--extra-vars", - "\"thredds_test_user={{ user `thredds_test_user`}}\"" - ], - "role_paths": [ - "provisioners/ansible/roles/cleanup", - "provisioners/ansible/roles/general-packages", - "provisioners/ansible/roles/gradle-builds-cache-bootstrap", - "provisioners/ansible/roles/init", - "provisioners/ansible/roles/libnetcdf-and-deps", - "provisioners/ansible/roles/maven", - "provisioners/ansible/roles/miniconda", - "provisioners/ansible/roles/security", - "provisioners/ansible/roles/temurin", - "provisioners/ansible/roles/thredds-test-data-mount-prep", - "provisioners/ansible/roles/zulu" - ], - "playbook_file": "provisioners/ansible/site.yml", - "galaxy_file": "provisioners/ansible/requirements.yml", - "command": "ANSIBLE_CONFIG=/ansible_config/ansible.cfg ANSIBLE_FORCE_COLOR=1 PYTHONUNBUFFERED=1 ansible-playbook", - "clean_staging_directory": true - }, - { - "type": "shell", - "inline": ["dos2unix /entrypoint.sh"], - "only": [ "docker-github-action", "docker-github-action-nexus" ] - }, - { - "type": "shell", - "script": "provisioners/scripts/cleanup.sh" - } - ], - "post-processors": [ - { - "type": "docker-tag", - "repository": "docker.unidata.ucar.edu/thredds-test-environment", - "tags": "latest", - "only": [ "docker-commit" ] - }, - { - "type": "docker-tag", - "repository": "ghcr.io/unidata/thredds-test-action", - "tags": "v2", - "only": [ "docker-github-action" ] - }, - { - "type": "docker-tag", - "repository": "docker.unidata.ucar.edu/thredds-test-action", - "tags": "v2", - "only": [ "docker-github-action-nexus" ] - } - ] -}