-
Notifications
You must be signed in to change notification settings - Fork 90
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add parent dependency check on samples (#397)
* ignore cached remote samples Signed-off-by: Michael Valdron <[email protected]> * nightly script for building sample parent stack list Signed-off-by: Michael Valdron <[email protected]> * init check parent stacks script Signed-off-by: Michael Valdron <[email protected]> * convert nightly parents build to on the fly parents build. Signed-off-by: Michael Valdron <[email protected]> * multi version support for build parent stacks Signed-off-by: Michael Valdron <[email protected]> * ignore any local parents file. Signed-off-by: Michael Valdron <[email protected]> * rename check_parent_stacks.sh to build_parents_file.sh, build_parents_file.sh now builds the parents yaml file and returns the children samples of the parent stacks. Signed-off-by: Michael Valdron <[email protected]> * validate devfile schemas test suite parameters now changeable Signed-off-by: Michael Valdron <[email protected]> * validate samples workflow Signed-off-by: Michael Valdron <[email protected]> * add check for if there are samples to be checked under validate samples job. Signed-off-by: Michael Valdron <[email protected]> * add README instruction on dependency check changes Signed-off-by: Michael Valdron <[email protected]> * yq 4.x installation guide link added to testing prerequisites list. Signed-off-by: Michael Valdron <[email protected]> * add missing link to test suite Signed-off-by: Michael Valdron <[email protected]> * validate_devfile_schemas script accepts relative or absolute paths Signed-off-by: Michael Valdron <[email protected]> * use registry-support cache samples Signed-off-by: Michael Valdron <[email protected]> * use yq v4.44.1 Signed-off-by: Michael Valdron <[email protected]> * validate-samples actions match revision from other workflows Signed-off-by: Michael Valdron <[email protected]> * create verbose mode for build_parents_file.sh script, defaults to false Signed-off-by: Michael Valdron <[email protected]> * validate_devfile_schemas.sh script unzips resource files if samples Signed-off-by: Michael Valdron <[email protected]> * update README Signed-off-by: Michael Valdron <[email protected]> * correct plural meaning to singular meaning for operation names Signed-off-by: Michael Valdron <[email protected]> * add verbose mode to validate_devfile_schemas script Signed-off-by: Michael Valdron <[email protected]> * set default sample cache dir to .cache/samples Signed-off-by: Michael Valdron <[email protected]> * remove unused minikube env vars Signed-off-by: Michael Valdron <[email protected]> --------- Signed-off-by: Michael Valdron <[email protected]>
- Loading branch information
1 parent
56269fc
commit 66a5a95
Showing
7 changed files
with
393 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| # | ||
| # Copyright 2023 Red Hat, Inc. | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| name: Validate child samples | ||
|
|
||
| on: | ||
| push: | ||
| branches: [main] | ||
| pull_request: | ||
| branches: [main] | ||
| schedule: | ||
| - cron: 0 5 * * * | ||
|
|
||
| concurrency: | ||
| group: ${{ github.workflow }}-${{ github.event.number || github.ref }} | ||
| cancel-in-progress: true | ||
|
|
||
| env: | ||
| YQ_VERSION: "v4.44.1" | ||
| TEST_DELTA: false | ||
|
|
||
| jobs: | ||
| validate-devfile-schema: | ||
| name: validate devfile schemas | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5 | ||
| with: | ||
| fetch-depth: 0 | ||
|
|
||
| - name: Install Go | ||
| uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1 | ||
| with: | ||
| go-version: "1.19" | ||
|
|
||
| - name: Install Ginkgo | ||
| run: go install -mod=mod github.com/onsi/ginkgo/v2/[email protected] | ||
|
|
||
| - name: Install yq | ||
| run: curl -sL -O https://github.com/mikefarah/yq/releases/download/${{ env.YQ_VERSION }}/yq_linux_amd64 -o /usr/local/bin/yq && mv ./yq_linux_amd64 /usr/local/bin/yq && chmod +x /usr/local/bin/yq | ||
|
|
||
| - name: Test delta if on a pull request | ||
| if: ${{ github.event_name == 'pull_request' }} | ||
| run: echo "TEST_DELTA=true" >> $GITHUB_ENV | ||
|
|
||
| - name: Build parents file and get child samples | ||
| run: echo "STACKS=$(bash tests/build_parents_file.sh)" >> $GITHUB_ENV | ||
|
|
||
| - name: Validate samples | ||
| if: ${{ env.STACKS != '' }} | ||
| run: STACKS_DIR=$(pwd)/samples/.cache bash tests/validate_devfile_schemas.sh --samples |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -3,4 +3,6 @@ registry-support/ | |
| .idea/ | ||
| devfile-web/ | ||
| vendor/ | ||
| .odo | ||
| .odo | ||
| .cache | ||
| parents.yaml | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,251 @@ | ||
| #!/bin/bash | ||
|
|
||
| POSITIONAL_ARGS=() | ||
| VERBOSE="false" | ||
|
|
||
| while [[ $# -gt 0 ]]; do | ||
| case $1 in | ||
| -v|--verbose) | ||
| VERBOSE="true" | ||
| shift # past argument | ||
| ;; | ||
| -*|--*) | ||
| echo "Unknown option $1" | ||
| exit 1 | ||
| ;; | ||
| *) | ||
| POSITIONAL_ARGS+=("$1") # save positional arg | ||
| shift # past argument | ||
| ;; | ||
| esac | ||
| done | ||
|
|
||
| set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters | ||
|
|
||
| # default samples file path | ||
| samples_file=$(pwd)/extraDevfileEntries.yaml | ||
| # Cached remote samples directory | ||
| samples_dir=$(pwd)/.cache/samples | ||
| # default stacks directory | ||
| stacks_dir=${STACKS:-"$(pwd)/stacks"} | ||
| parents_file=$(pwd)/parents.yaml | ||
| # Base directory path | ||
| base_path=$(realpath $(dirname $(dirname $0))) | ||
|
|
||
| # YAML query cmd path | ||
| YQ_PATH=${YQ_PATH:-yq} | ||
|
|
||
| # Read samples file as first argument | ||
| # if unset use default samples file path | ||
| if [ ! -z "${1}" ]; then | ||
| samples_file=${1} | ||
| fi | ||
|
|
||
| get_parent_version() { | ||
| devfile=$1 | ||
| name=$2 | ||
| version=$($YQ_PATH eval .parent.version ${devfile}) | ||
|
|
||
| if [ "${version}" == "null" ] && [ -f "${stacks_dir}/${name}/stack.yaml" ]; then | ||
| version=$($YQ_PATH eval '.versions | filter(.default) | .[0].version' ${stacks_dir}/${name}/stack.yaml) | ||
| fi | ||
|
|
||
| echo ${version} | ||
| } | ||
|
|
||
| # Get parent index if exists, else returns -1 | ||
| parent_index() { | ||
| name=$1 | ||
| version=$2 | ||
|
|
||
| if [ -z "${version}" ]; then | ||
| result=$($YQ_PATH eval ".parents | to_entries | filter(.value.name == \"${name}\") | .[0].key" ${parents_file}) | ||
| else | ||
| result=$($YQ_PATH eval ".parents | to_entries | filter(.value.name == \"${name}\" and .value.version == \"${version}\") | .[0].key" ${parents_file}) | ||
| fi | ||
|
|
||
| if [ "${result}" == "null" ]; then | ||
| echo "-1" | ||
| else | ||
| echo ${result} | ||
| fi | ||
| } | ||
|
|
||
| # Get child index if exists, else returns -1 | ||
| child_index() { | ||
| parent_idx=$1 | ||
| name=$2 | ||
| version=$3 | ||
|
|
||
| if [ -z "${version}" ]; then | ||
| result=$($YQ_PATH eval ".parents.[${parent_idx}].children | to_entries | filter(.value.name == \"${name}\") | .[0].key" ${parents_file}) | ||
| else | ||
| result=$($YQ_PATH eval ".parents.[${parent_idx}].children | to_entries | filter(.value.name == \"${name}\" and .value.version == \"${version}\") | .[0].key" ${parents_file}) | ||
| fi | ||
|
|
||
| if [ "${result}" == "null" ]; then | ||
| echo "-1" | ||
| else | ||
| echo ${result} | ||
| fi | ||
| } | ||
|
|
||
| # Builds sample parent | ||
| build_parent() { | ||
| parent_name=$1 | ||
| parent_version=$2 | ||
|
|
||
| if [ "${parent_version}" == "null" ]; then | ||
| parent_version="" | ||
| fi | ||
|
|
||
| if [ "${parent_name}" != "null" ]; then | ||
| if [ ! -f ${parents_file} ]; then | ||
| $YQ_PATH eval -n ".parents[0].name = \"${parent_name}\"" > ${parents_file} | ||
| if [ "${parent_version}" != "" ]; then | ||
| $YQ_PATH eval ".parents[0].version = \"${parent_version}\"" -i ${parents_file} | ||
| fi | ||
|
|
||
| return | ||
| fi | ||
|
|
||
| if [ "$($YQ_PATH eval .parents ${parents_file})" == "null" ]; then | ||
| $YQ_PATH eval ".parents[0].name = \"${parent_name}\"" -i ${parents_file} | ||
| if [ "${parent_version}" != "" ]; then | ||
| $YQ_PATH eval ".parents[0].version = \"${parent_version}\"" -i ${parents_file} | ||
| fi | ||
|
|
||
| return | ||
| fi | ||
|
|
||
| parent_idx=$(parent_index ${parent_name} ${parent_version}) | ||
| if [ "${parent_idx}" == "-1" ]; then | ||
| next_idx=$($YQ_PATH eval ".parents | length" ${parents_file}) | ||
| $YQ_PATH eval ".parents[${next_idx}].name = \"${parent_name}\"" -i ${parents_file} | ||
| if [ "${parent_version}" != "" ]; then | ||
| $YQ_PATH eval ".parents[${next_idx}].version = \"${parent_version}\"" -i ${parents_file} | ||
| fi | ||
| fi | ||
| else | ||
| return 1 | ||
| fi | ||
| } | ||
|
|
||
| # Builds a child of a parent stack | ||
| build_child() { | ||
| parent_name=$1 | ||
| parent_version=$2 | ||
| sample_name=$3 | ||
| sample_version=$4 | ||
|
|
||
| parent_idx=$(parent_index ${parent_name} ${parent_version}) | ||
|
|
||
| if [ "$($YQ_PATH eval .parents[${parent_idx}].children ${parents_file})" == "null" ]; then | ||
| $YQ_PATH eval ".parents[${parent_idx}].children[0].name = \"${sample_name}\"" -i ${parents_file} | ||
| if [ "${sample_version}" != "" ]; then | ||
| $YQ_PATH eval ".parents[${parent_idx}].children[0].version = \"${sample_version}\"" -i ${parents_file} | ||
| fi | ||
|
|
||
| return | ||
| fi | ||
|
|
||
| child_idx=$(child_index ${parent_idx} ${sample_name} ${sample_version}) | ||
| if [ "${child_idx}" == "-1" ]; then | ||
| next_idx=$($YQ_PATH eval ".parents[${parent_idx}].children | length" ${parents_file}) | ||
| $YQ_PATH eval ".parents[${parent_idx}].children[${next_idx}].name = \"${sample_name}\"" -i ${parents_file} | ||
| if [ "${sample_version}" != "" ]; then | ||
| $YQ_PATH eval ".parents[${parent_idx}].children[${next_idx}].version = \"${sample_version}\"" -i ${parents_file} | ||
| fi | ||
| fi | ||
| } | ||
|
|
||
| build_parents_file() { | ||
| samples_len=$($YQ_PATH eval '.samples | length' ${samples_file}) | ||
|
|
||
| for ((s_idx=0;s_idx<${samples_len};s_idx++)); do | ||
| sample_name=$($YQ_PATH eval .samples.${s_idx}.name ${samples_file}) | ||
| sample_versions=($($YQ_PATH eval .samples.${s_idx}.versions.[].version ${samples_file})) | ||
|
|
||
| # Iterate through sample versions if sample has multi version support | ||
| if [ ${#sample_versions[@]} -ne 0 ]; then | ||
| for ((v_idx=0;v_idx<${#sample_versions[@]};v_idx++)); do | ||
| devfile=${samples_dir}/${sample_name}/${sample_versions[$v_idx]}/devfile.yaml | ||
| parent_name=$($YQ_PATH eval .parent.id ${devfile}) | ||
| parent_version=$(get_parent_version ${devfile} ${parent_name}) | ||
| build_parent ${parent_name} ${parent_version} | ||
|
|
||
| if [ $? -eq 0 ]; then | ||
| build_child "${parent_name}" "${parent_version}" "${sample_name}" "${sample_versions[$v_idx]}" | ||
| fi | ||
| done | ||
| else | ||
| devfile=${samples_dir}/${sample_name}/devfile.yaml | ||
| parent_name=$($YQ_PATH eval .parent.id ${devfile}) | ||
| parent_version=$(get_parent_version ${devfile} ${parent_name}) | ||
| build_parent ${parent_name} ${parent_version} | ||
|
|
||
| if [ $? -eq 0 ]; then | ||
| build_child "${parent_name}" "${parent_version}" "${sample_name}" "" | ||
| fi | ||
| fi | ||
| done | ||
| } | ||
|
|
||
| # Gets the children sample paths of parents. | ||
| # When TEST_DELTA is set to true, only children of parents | ||
| # with changes are returned. | ||
| get_children_of_parents() { | ||
| stack_dirs=$(bash $(pwd)/tests/get_stacks.sh) | ||
| children=() | ||
|
|
||
| for stack_dir in $stack_dirs; do | ||
| if [ "$(basename $(dirname $stack_dir))" == "." ]; then | ||
| stack_name=$(basename $stack_dir) | ||
|
|
||
| names=($($YQ_PATH eval ".parents | filter(.name == \"${stack_name}\") | .[0].children.[].name" ${parents_file})) | ||
| versions=($($YQ_PATH eval ".parents | filter(.name == \"${stack_name}\") | .[0].children.[].version" ${parents_file})) | ||
| else | ||
| stack_name=$(basename $(dirname $stack_dir)) | ||
| stack_version=$(basename $stack_dir) | ||
|
|
||
| names=($($YQ_PATH eval ".parents | filter(.name == \"${stack_name}\" and .version == \"${stack_version}\") | .[0].children.[].name" ${parents_file})) | ||
| versions=($($YQ_PATH eval ".parents | filter(.name == \"${stack_name}\" and .version == \"${stack_version}\") | .[0].children.[].version" ${parents_file})) | ||
| fi | ||
|
|
||
|
|
||
| for ((c_idx=0;c_idx<${#names[@]};c_idx++)); do | ||
| if [ "${versions[$c_idx]}" == "null" ]; then | ||
| children+=("${names[$c_idx]}") | ||
| else | ||
| children+=("${names[$c_idx]}/${versions[$c_idx]}") | ||
| fi | ||
| done | ||
| done | ||
|
|
||
| echo ${children[@]} | ||
| } | ||
|
|
||
| if [ ! -d ${base_path}/registry-support ] | ||
| then | ||
| if [ "$VERBOSE" == "true" ] | ||
| then | ||
| git clone https://github.com/devfile/registry-support ${base_path}/registry-support | ||
| else | ||
| git clone -q https://github.com/devfile/registry-support ${base_path}/registry-support | ||
| fi | ||
| fi | ||
|
|
||
| if [ "$VERBOSE" == "true" ] | ||
| then | ||
| bash ${base_path}/registry-support/build-tools/cache_samples.sh ${samples_file} ${samples_dir} | ||
| else | ||
| bash ${base_path}/registry-support/build-tools/cache_samples.sh ${samples_file} ${samples_dir} > /dev/null 2>&- echo | ||
| fi | ||
|
|
||
| if [ -f ${parents_file} ]; then | ||
| rm ${parents_file} | ||
| fi | ||
|
|
||
| build_parents_file | ||
|
|
||
| get_children_of_parents |
Oops, something went wrong.