[CI / FIPS] Convert smoke tests to use FIPS agent (elastic#186858)

## Summary

Closes elastic/kibana-operations#137

- Convert FIPS daily smoke test pipeline to use FIPS agents
  - This allows the smoke tests to run in parallel now
- Removes FIPS Vagrant box from package testing
- Adds verify FIPS enabled check to pipeline 

Pipeline run: https://buildkite.com/elastic/kibana-fips/builds/90

.buildkite/pipelines/fips.yml

@@ -1,32 +1,63 @@
 env:
-  DISABLE_CI_STATS_SHIPPING: "true"
+  DISABLE_CI_STATS_SHIPPING: 'true'
+  KBN_ENABLE_FIPS: 'true'
+  TEST_BROWSER_HEADLESS: 1
+agents:
+  provider: 'gcp'
+  image: 'family/kibana-fips-ubuntu-2004'
+  imageProject: 'elastic-images-prod'
 steps:
+  - command: .buildkite/scripts/lifecycle/pre_build.sh
+    label: Pre-Build
+    key: pre-build
+    timeout_in_minutes: 10
+    agents:
+      machineType: n2-standard-2
+
+  - wait
+
   - command: .buildkite/scripts/steps/build_kibana.sh
     label: Build Kibana Distribution and Plugins
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-16
       preemptible: true
     key: build
     if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+    depends_on: pre-build
     timeout_in_minutes: 60
     retry:
       automatic:
-        - exit_status: "-1"
+        - exit_status: '-1'
           limit: 3
 
   - wait
 
-  - command: TEST_PACKAGE=fips .buildkite/scripts/steps/package_testing/test.sh
-    label: "Smoke testing for FIPS"
+  - command: .buildkite/scripts/steps/checks/verify_fips_enabled.sh
+    label: 'Verify FIPS Enabled'
+    depends_on: build
+    timeout_in_minutes: 10
+    agents:
+      machineType: n2-standard-2
+      preemptible: true
+
+  - command: .buildkite/scripts/steps/fips/smoke_test.sh
+    label: 'Pick Smoke Test Group Run Order'
+    depends_on: build
+    timeout_in_minutes: 10
+    env:
+      FTR_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/ftr_configs.sh'
+      FTR_EXTRA_ARGS: '$FTR_EXTRA_ARGS'
+      LIMIT_CONFIG_TYPE: 'functional'
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - wait: ~
+    continue_on_failure: true
+
+  - command: .buildkite/scripts/lifecycle/post_build.sh
+    label: Post-Build
+    timeout_in_minutes: 10
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
-      enableNestedVirtualization: true
-      localSsds: 1
-      localSsdInterface: nvme
-      machineType: n2-standard-4
-    timeout_in_minutes: 600
+      machineType: n2-standard-2

.buildkite/scripts/steps/checks/verify_fips_enabled.sh

@@ -2,7 +2,11 @@
 
 set -euo pipefail
 
-source .buildkite/scripts/common/util.sh
+# This script is part of checks.sh in the PR pipeline but is called directly in the FIPS pipeline, so we need to bootstrap
+if [[ -z "${BASH_SOURCE[1]+x}" || "${BASH_SOURCE[1]}" != *"checks.sh"* ]]; then
+  export DISABLE_BOOTSTRAP_VALIDATION=false
+  .buildkite/scripts/bootstrap.sh
+fi
 
 .buildkite/scripts/download_build_artifacts.sh
 

.buildkite/scripts/steps/fips/smoke_test.sh

@@ -1,12 +1,10 @@
 #!/usr/bin/env bash
 
-if [ -z "$KIBANA_BUILD_LOCATION" ]; then
-  export KIBANA_BUILD_LOCATION="/usr/share/kibana"
-fi
-
-# a FTR failure will result in the script returning an exit code of 10
-exitCode=0
+set -euo pipefail
 
+# Limit the FTR configs for now to avoid running all the tests. Once we're
+# ready to utilize the full FTR suite in FIPS mode, we can remove this file and
+# call pick_test_group_run_order.sh directly in .buildkite/pipelines/fips.yml.
 configs=(
   "x-pack/test/reporting_functional/reporting_and_security.config.ts"
   "x-pack/test/saved_object_api_integration/security_and_spaces/config_trial.ts"
@@ -19,34 +17,8 @@ configs=(
   "x-pack/test/functional/apps/security/config.ts"
 )
 
-cd /home/vagrant/kibana
-
-for config in "${configs[@]}"; do
-  set +e
-  node /home/vagrant/kibana/scripts/functional_tests \
-    --bail \
-    --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
-    --config="$config"
-  lastCode=$?
-  set -e
-
-  if [ $lastCode -ne 0 ]; then
-    exitCode=10
-    echo "FTR exited with code $lastCode"
-    echo "^^^ +++"
-
-    if [[ "$failedConfigs" ]]; then
-      failedConfigs="${failedConfigs}"$'\n'"- ${config}"
-    else
-      failedConfigs="### Failed FTR Configs"$'\n'"- ${config}"
-    fi
-  fi
-done
-
-if [[ "$failedConfigs" ]]; then
-  echo "$failedConfigs" >/home/vagrant/ftr_failed_configs
-fi
-
-echo "--- FIPS smoke test complete"
+printf -v FTR_CONFIG_PATTERNS '%s,' "${configs[@]}"
+FTR_CONFIG_PATTERNS="${FTR_CONFIG_PATTERNS%,}"
+export FTR_CONFIG_PATTERNS
 
-exit $exitCode
+.buildkite/scripts/steps/test/pick_test_group_run_order.sh

.buildkite/scripts/steps/package_testing/test.sh

@@ -21,25 +21,17 @@ elif [[ "$TEST_PACKAGE" == "rpm" ]]; then
 elif [[ "$TEST_PACKAGE" == "docker" ]]; then
   download_artifact "kibana-$KIBANA_PKG_VERSION*-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
   KIBANA_IP_ADDRESS="192.168.56.7"
-elif [[ "$TEST_PACKAGE" == "fips" ]]; then
-  download_artifact kibana-default.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
-  download_artifact kibana-default-plugins.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
 fi
 cd ..
 
 export VAGRANT_CWD=$PWD/test/package
+vagrant up "$TEST_PACKAGE" --no-provision
 
-if [[ "$TEST_PACKAGE" == "fips" ]]; then
-  vagrant up "$TEST_PACKAGE"
-else
-  vagrant up "$TEST_PACKAGE" --no-provision
-
-  node scripts/es snapshot \
-    -E network.bind_host=127.0.0.1,192.168.56.1 \
-    -E discovery.type=single-node \
-    --license=trial &
-  while ! timeout 1 bash -c "echo > /dev/tcp/localhost/9200"; do sleep 30; done
-fi
+node scripts/es snapshot \
+  -E network.bind_host=127.0.0.1,192.168.56.1 \
+  -E discovery.type=single-node \
+  --license=trial &
+while ! timeout 1 bash -c "echo > /dev/tcp/localhost/9200"; do sleep 30; done
 
 function echoKibanaLogs {
   if [[ "$TEST_PACKAGE" == "deb" ]] || [[ "$TEST_PACKAGE" == "rpm" ]]; then
@@ -55,29 +47,13 @@ function echoKibanaLogs {
 }
 trap "echoKibanaLogs" EXIT
 
-if [[ "$TEST_PACKAGE" == "fips" ]]; then
-  set +e
-  vagrant ssh $TEST_PACKAGE -t -c "/home/vagrant/kibana/.buildkite/scripts/steps/fips/smoke_test.sh"
-  exitCode=$?
-
-  vagrant ssh $TEST_PACKAGE -t -c "cat /home/vagrant/ftr_failed_configs 2>/dev/null" >ftr_failed_configs
-  set -e
-
-  if [ -s ftr_failed_configs ]; then
-    cat ftr_failed_configs | buildkite-agent annotate --style "error"
-  fi
-
-  exit $exitCode
-else
-  vagrant provision "$TEST_PACKAGE"
+vagrant provision "$TEST_PACKAGE"
 
-  export TEST_BROWSER_HEADLESS=1
-  export TEST_KIBANA_URL="http://elastic:changeme@$KIBANA_IP_ADDRESS:5601"
-  export TEST_ES_URL="http://elastic:[email protected]:9200"
+export TEST_BROWSER_HEADLESS=1
+export TEST_KIBANA_URL="http://elastic:changeme@$KIBANA_IP_ADDRESS:5601"
+export TEST_ES_URL="http://elastic:[email protected]:9200"
 
-  echo "--- FTR - Reporting"
+cd x-pack
 
-  cd x-pack
-
-  node scripts/functional_test_runner.js --config test/functional/apps/visualize/config.ts --include-tag=smoke --quiet
-fi
+echo "--- FTR - Reporting"
+node scripts/functional_test_runner.js --config test/functional/apps/visualize/config.ts --include-tag=smoke --quiet

test/package/Vagrantfile

@@ -39,16 +39,4 @@ Vagrant.configure("2") do |config|
     end
     docker.vm.network "private_network", ip: "192.168.56.7"
   end
-
-  config.vm.define "fips" do |fips|
-    fips.vm.synced_folder '../../', '/home/vagrant/kibana', SharedFoldersEnableSymlinksCreate: false
-    fips.vm.provider :virtualbox do |vb|
-      vb.memory = 4096
-      vb.cpus = 2
-    end
-    fips.vm.box = 'ubuntu/jammy64'
-    fips.vm.provision "ansible" do |ansible|
-      ansible.playbook = "fips.yml"
-    end
-  end
 end