diff --git a/dev-support/ci/selective_ci_checks.sh b/dev-support/ci/selective_ci_checks.sh index 869d36fc6cc..ffe6886b33d 100755 --- a/dev-support/ci/selective_ci_checks.sh +++ b/dev-support/ci/selective_ci_checks.sh @@ -247,6 +247,7 @@ function get_count_doc_files() { local pattern_array=( "^hadoop-hdds/docs" "^hadoop-ozone/dev-support/checks/docs.sh" + "^hadoop-ozone/dev-support/checks/install/hugo.sh" ) filter_changed_files true COUNT_DOC_CHANGED_FILES=${match_count} @@ -278,6 +279,8 @@ function get_count_kubernetes_files() { start_end::group_start "Count kubernetes files" local pattern_array=( "^hadoop-ozone/dev-support/checks/kubernetes.sh" + "^hadoop-ozone/dev-support/checks/install/flekszible.sh" + "^hadoop-ozone/dev-support/checks/install/k3s.sh" "^hadoop-ozone/dist" ) local ignore_array=( @@ -356,7 +359,7 @@ function check_needs_bats() { local pattern_array=( "\.bash$" "\.bats$" - "\.sh$" # includes hadoop-ozone/dev-support/checks/bats.sh + "\.sh$" # includes hadoop-ozone/dev-support/checks/bats.sh and hadoop-ozone/dev-support/checks/install/bats.sh ) filter_changed_files @@ -416,6 +419,7 @@ function check_needs_findbugs() { start_end::group_start "Check if findbugs is needed" local pattern_array=( "^hadoop-ozone/dev-support/checks/findbugs.sh" + "^hadoop-ozone/dev-support/checks/install/spotbugs.sh" "findbugsExcludeFile.xml" "pom.xml" "src/..../java" diff --git a/hadoop-ozone/dev-support/checks/_lib.sh b/hadoop-ozone/dev-support/checks/_lib.sh index d1e1db59cea..ec59c16b632 100644 --- a/hadoop-ozone/dev-support/checks/_lib.sh +++ b/hadoop-ozone/dev-support/checks/_lib.sh @@ -63,106 +63,3 @@ _install_tool() { fi fi } - -install_bats() { - _install_tool bats "bats-core-${BATS_VERSION}/bin" -} - -_install_bats() { - curl -LSs "https://github.com/bats-core/bats-core/archive/v${BATS_VERSION}.tar.gz" | tar -xz -f - -} - -install_k3s() { - _install_tool k3s -} - -_install_k3s() { - curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="${K3S_VERSION}" sh - - sudo chmod a+r $KUBECONFIG -} - -install_flekszible() { - _install_tool flekszible bin -} - -_install_flekszible() { - mkdir bin - - local os=$(uname -s) - local arch=$(uname -m) - - curl -LSs "https://github.com/elek/flekszible/releases/download/v${FLEKSZIBLE_VERSION}/flekszible_${FLEKSZIBLE_VERSION}_${os}_${arch}.tar.gz" | tar -xz -f - -C bin - - chmod +x bin/flekszible -} - -install_hugo() { - _install_tool hugo bin -} - -_install_hugo() { - local os=$(uname -s) - local arch=$(uname -m) - - mkdir bin - - case "${os}" in - Darwin) - os=macOS - ;; - esac - - case "${arch}" in - x86_64) - arch=64bit - ;; - esac - - curl -LSs "https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_${HUGO_VERSION}_${os}-${arch}.tar.gz" | tar -xz -f - -C bin hugo - chmod +x bin/hugo -} - -install_virtualenv() { - _install_tool virtualenv -} - -_install_virtualenv() { - sudo pip3 install virtualenv -} - -install_robot() { - _install_tool robot venv/bin -} - -_install_robot() { - virtualenv venv - source venv/bin/activate - pip install robotframework -} - -install_spotbugs() { - _install_tool spotbugs "spotbugs-${SPOTBUGS_VERSION}/bin" -} - -_install_spotbugs() { - curl -LSs "https://repo.maven.apache.org/maven2/com/github/spotbugs/spotbugs/${SPOTBUGS_VERSION}/spotbugs-${SPOTBUGS_VERSION}.tgz" | tar -xz -f - -} - -download_hadoop_aws() { - local dir="$1" - - if [[ -z ${dir} ]]; then - echo "Required argument: target directory for Hadoop AWS sources" >&2 - return 1 - fi - - if [[ ! -e "${dir}" ]] || [[ ! -d "${dir}"/src/test/resources ]]; then - mkdir -p "${dir}" - if [[ ! -f "${dir}.tar.gz" ]]; then - local url="https://www.apache.org/dyn/closer.lua?action=download&filename=hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}-src.tar.gz" - echo "Downloading Hadoop from ${url}" - curl -LSs --fail -o "${dir}.tar.gz" "$url" || return 1 - fi - tar -x -z -C "${dir}" --strip-components=3 -f "${dir}.tar.gz" --wildcards 'hadoop-*-src/hadoop-tools/hadoop-aws' || return 1 - fi -} diff --git a/hadoop-ozone/dev-support/checks/acceptance.sh b/hadoop-ozone/dev-support/checks/acceptance.sh index ea9fa819ec3..1e877ade9f1 100755 --- a/hadoop-ozone/dev-support/checks/acceptance.sh +++ b/hadoop-ozone/dev-support/checks/acceptance.sh @@ -50,6 +50,25 @@ if [[ "${OZONE_ACCEPTANCE_SUITE}" == "s3a" ]]; then export HADOOP_AWS_DIR=${OZONE_ROOT}/target/hadoop-src fi + download_hadoop_aws() { + local dir="$1" + + if [[ -z ${dir} ]]; then + echo "Required argument: target directory for Hadoop AWS sources" >&2 + return 1 + fi + + if [[ ! -e "${dir}" ]] || [[ ! -d "${dir}"/src/test/resources ]]; then + mkdir -p "${dir}" + if [[ ! -f "${dir}.tar.gz" ]]; then + local url="https://www.apache.org/dyn/closer.lua?action=download&filename=hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}-src.tar.gz" + echo "Downloading Hadoop from ${url}" + curl -LSs --fail -o "${dir}.tar.gz" "$url" || return 1 + fi + tar -x -z -C "${dir}" --strip-components=3 -f "${dir}.tar.gz" --wildcards 'hadoop-*-src/hadoop-tools/hadoop-aws' || return 1 + fi + } + if ! download_hadoop_aws "${HADOOP_AWS_DIR}"; then echo "Failed to download Hadoop ${HADOOP_VERSION}" > "${REPORT_FILE}" exit 1 diff --git a/hadoop-ozone/dev-support/checks/bats.sh b/hadoop-ozone/dev-support/checks/bats.sh index 007f04c683f..3dec6052a92 100755 --- a/hadoop-ozone/dev-support/checks/bats.sh +++ b/hadoop-ozone/dev-support/checks/bats.sh @@ -21,14 +21,8 @@ set -u -o pipefail DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" cd "${DIR}/../../.." || exit 1 -: ${BATS_VERSION:="1.2.1"} - source "${DIR}/_lib.sh" - -install_bats - -git clone https://github.com/bats-core/bats-assert dev-support/ci/bats-assert -git clone https://github.com/bats-core/bats-support dev-support/ci/bats-support +source "${DIR}/install/bats.sh" REPORT_DIR=${OUTPUT_DIR:-"${DIR}/../../../target/bats"} mkdir -p "${REPORT_DIR}" diff --git a/hadoop-ozone/dev-support/checks/docs.sh b/hadoop-ozone/dev-support/checks/docs.sh index ff72e82a892..045811c16c3 100755 --- a/hadoop-ozone/dev-support/checks/docs.sh +++ b/hadoop-ozone/dev-support/checks/docs.sh @@ -21,10 +21,8 @@ set -u -o pipefail DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" cd "${DIR}/../../.." || exit 1 -: ${HUGO_VERSION:=0.83.1} - source "${DIR}/_lib.sh" -install_hugo +source "${DIR}/install/hugo.sh" REPORT_DIR=${OUTPUT_DIR:-"${DIR}/../../../target/docs"} mkdir -p "${REPORT_DIR}" diff --git a/hadoop-ozone/dev-support/checks/findbugs.sh b/hadoop-ozone/dev-support/checks/findbugs.sh index 7e92edf7937..0bd7a5717e2 100755 --- a/hadoop-ozone/dev-support/checks/findbugs.sh +++ b/hadoop-ozone/dev-support/checks/findbugs.sh @@ -22,11 +22,9 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" cd "$DIR/../../.." || exit 1 : ${OZONE_WITH_COVERAGE:="false"} -: ${SPOTBUGS_VERSION:=3.1.12} source "${DIR}/_lib.sh" - -install_spotbugs +source "${DIR}/install/spotbugs.sh" REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/findbugs"} mkdir -p "$REPORT_DIR" diff --git a/hadoop-ozone/dev-support/checks/install/bats.sh b/hadoop-ozone/dev-support/checks/install/bats.sh new file mode 100644 index 00000000000..b8d3aa7a27a --- /dev/null +++ b/hadoop-ozone/dev-support/checks/install/bats.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script installs bats. +# Requires _install_tool from _lib.sh. Use `source` for both scripts, because it modifies $PATH. + +: ${BATS_VERSION:="1.2.1"} + +_install_bats() { + curl -LSs "https://github.com/bats-core/bats-core/archive/v${BATS_VERSION}.tar.gz" | tar -xz -f - +} + +_install_tool bats "bats-core-${BATS_VERSION}/bin" + +git clone https://github.com/bats-core/bats-assert dev-support/ci/bats-assert +git clone https://github.com/bats-core/bats-support dev-support/ci/bats-support diff --git a/hadoop-ozone/dev-support/checks/install/flekszible.sh b/hadoop-ozone/dev-support/checks/install/flekszible.sh new file mode 100644 index 00000000000..ea887b84900 --- /dev/null +++ b/hadoop-ozone/dev-support/checks/install/flekszible.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script installs Flekszible. +# Requires _install_tool from _lib.sh. Use `source` for both scripts, because it modifies $PATH. + +: ${FLEKSZIBLE_VERSION:="2.3.0"} + +_install_flekszible() { + mkdir bin + + local os=$(uname -s) + local arch=$(uname -m) + + curl -LSs "https://github.com/elek/flekszible/releases/download/v${FLEKSZIBLE_VERSION}/flekszible_${FLEKSZIBLE_VERSION}_${os}_${arch}.tar.gz" | tar -xz -f - -C bin + + chmod +x bin/flekszible +} + +_install_tool flekszible bin diff --git a/hadoop-ozone/dev-support/checks/install/hugo.sh b/hadoop-ozone/dev-support/checks/install/hugo.sh new file mode 100644 index 00000000000..279d668438d --- /dev/null +++ b/hadoop-ozone/dev-support/checks/install/hugo.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script installs Hugo. +# Requires _install_tool from _lib.sh. Use `source` for both scripts, because it modifies $PATH. + +: ${HUGO_VERSION:=0.83.1} + +_install_hugo() { + local os=$(uname -s) + local arch=$(uname -m) + + mkdir bin + + case "${os}" in + Darwin) + os=macOS + ;; + esac + + case "${arch}" in + x86_64) + arch=64bit + ;; + esac + + curl -LSs "https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_${HUGO_VERSION}_${os}-${arch}.tar.gz" | tar -xz -f - -C bin hugo + chmod +x bin/hugo +} + +_install_tool hugo bin diff --git a/hadoop-ozone/dev-support/checks/install/k3s.sh b/hadoop-ozone/dev-support/checks/install/k3s.sh new file mode 100644 index 00000000000..2cd59874267 --- /dev/null +++ b/hadoop-ozone/dev-support/checks/install/k3s.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script installs K3S. +# Requires _install_tool from _lib.sh. Use `source` for both scripts, because it modifies $PATH. + +: ${K3S_VERSION:="v1.21.2+k3s1"} +: ${KUBECONFIG:=/etc/rancher/k3s/k3s.yaml} + +_install_k3s() { + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="${K3S_VERSION}" sh - + sudo chmod a+r $KUBECONFIG +} + +_install_tool k3s diff --git a/hadoop-ozone/dev-support/checks/install/spotbugs.sh b/hadoop-ozone/dev-support/checks/install/spotbugs.sh new file mode 100644 index 00000000000..0d0cce3a394 --- /dev/null +++ b/hadoop-ozone/dev-support/checks/install/spotbugs.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script installs SpotBugs. +# Requires _install_tool from _lib.sh. Use `source` for both scripts, because it modifies $PATH. + +: ${SPOTBUGS_VERSION:=3.1.12} + +_install_spotbugs() { + curl -LSs "https://repo.maven.apache.org/maven2/com/github/spotbugs/spotbugs/${SPOTBUGS_VERSION}/spotbugs-${SPOTBUGS_VERSION}.tgz" | tar -xz -f - +} + +_install_tool spotbugs "spotbugs-${SPOTBUGS_VERSION}/bin" diff --git a/hadoop-ozone/dev-support/checks/kubernetes.sh b/hadoop-ozone/dev-support/checks/kubernetes.sh index d5e5286f777..4699ed5520e 100755 --- a/hadoop-ozone/dev-support/checks/kubernetes.sh +++ b/hadoop-ozone/dev-support/checks/kubernetes.sh @@ -19,21 +19,37 @@ set -u -o pipefail DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" cd "$DIR/../../.." || exit 1 -: ${FLEKSZIBLE_VERSION:="2.3.0"} -: ${K3S_VERSION:="v1.21.2+k3s1"} -: ${KUBECONFIG:=/etc/rancher/k3s/k3s.yaml} - export KUBECONFIG source "${DIR}/_lib.sh" +source "${DIR}/install/flekszible.sh" + +# TODO these functions will be removed in HDDS-12099 +install_virtualenv() { + _install_tool virtualenv +} + +_install_virtualenv() { + sudo pip3 install virtualenv +} + +install_robot() { + _install_tool robot venv/bin +} + +_install_robot() { + virtualenv venv + source venv/bin/activate + pip install robotframework +} -install_flekszible install_virtualenv install_robot + if [[ "$(uname -s)" = "Darwin" ]]; then echo "Skip installing k3s, not supported on Mac. Make sure a working Kubernetes cluster is available." >&2 else - install_k3s + source "${DIR}/install/k3s.sh" fi REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/kubernetes"}