[yugabyte#13840] Do not error out if the LLVM URL in thirdparty is di…

…fferent than in llvm_installer

Summary:
If the third-party dependencies specify a LLVM package URL different from the one determined by llvm_installer, we should log a warning but let the build proceed.

Also fix a number of insecure variable evaluations in shell scripts when trying to check if some boolean variable is true.

Test Plan: Jenkins

Reviewers: neil, steve.varnau

Reviewed By: steve.varnau

Subscribers: jenkins-bot, ybase, yugaware

Differential Revision: https://phabricator.dev.yugabyte.com/D19310

bin/repeat_unit_test.sh

@@ -194,7 +194,7 @@ set_build_root
 set_sanitizer_runtime_options
 
 declare -i -r num_pos_args=${#positional_args[@]}
-if "$is_java_test"; then
+if [[ ${is_java_test} == "true" ]]; then
   if [[ $num_pos_args -ne 2 ]]; then
     fatal "Expected two positional arguments for Java tests, not including build type:" \
           "<maven_module_name> <class_name_with_package>"
@@ -254,7 +254,7 @@ if [[ $iteration -gt 0 ]]; then
   # One iteration with a specific "id" ($iteration).
   test_log_path_prefix=$log_dir/$iteration
   test_log_path=$test_log_path_prefix.log
-  if "$is_java_test"; then
+  if [[ ${is_java_test} == "true" ]]; then
     export YB_SUREFIRE_REPORTS_DIR=$test_log_path_prefix.reports
   fi
   export YB_FATAL_DETAILS_PATH_PREFIX=$test_log_path_prefix.fatal_failure_details
@@ -269,7 +269,7 @@ if [[ $iteration -gt 0 ]]; then
   fi
 
   # TODO: deduplicate the setup here against run_one_cxx_test() in common-test-env.sh.
-  if "$is_java_test"; then
+  if [[ ${is_java_test} == "true" ]]; then
     test_wrapper_cmd_line=(
       "$YB_BUILD_SUPPORT_DIR"/run-test.sh "${positional_args[@]}"
     )
@@ -287,7 +287,7 @@ if [[ $iteration -gt 0 ]]; then
   start_time_sec=$( date +%s )
   (
     cd "$TEST_TMPDIR"
-    if "$verbose"; then
+    if [[ ${verbose} == "true" ]]; then
       log "Iteration $iteration logging to $test_log_path"
     fi
     ulimit -c unlimited
@@ -316,9 +316,9 @@ if [[ $iteration -gt 0 ]]; then
       keep_log=true
     fi
   fi
-  if "$keep_log"; then
+  if [[ ${keep_log} == "true" ]]; then
     if ! "$skip_log_compression"; then
-      if "$is_java_test"; then
+      if [[ ${is_java_test} == "true" ]]; then
         # Compress Java test log.
         mv "$test_log_path" "$YB_SUREFIRE_REPORTS_DIR"
         pushd "$log_dir"
@@ -362,7 +362,7 @@ if [[ $iteration -gt 0 ]]; then
     comment+="; test log path: $test_log_path"
   else
     rm -f "$test_log_path"
-    if "$is_java_test"; then
+    if [[ ${is_java_test} == "true" ]]; then
       set +e
       rm -rf "$YB_SUREFIRE_REPORTS_DIR"
       set -e
@@ -397,7 +397,7 @@ else
   log "$gtest_filter_info"
   if [[ -n ${YB_EXTRA_GTEST_FLAGS:-} ]]; then
     log "Extra test flags from YB_EXTRA_GTEST_FLAGS: $YB_EXTRA_GTEST_FLAGS"
-  elif "$verbose"; then
+  elif [[ ${verbose} == "true" ]]; then
     log "YB_EXTRA_GTEST_FLAGS is not set"
   fi
   log "Saving repeated test execution logs to: $log_dir"

build-support/common-build-env.sh

@@ -283,27 +283,27 @@ is_clean_build=false
 yb_thirdparty_dir_origin=""
 
 if [[ -n ${YB_THIRDPARTY_DIR:-} ]]; then
-  yb_thirdparty_dir_origin=" (from environment)"
+  yb_thirdparty_dir_origin="from environment"
 fi
 
 yb_thirdparty_url_origin=""
 if [[ -n ${YB_THIRDPARTY_URL:-} ]]; then
-  yb_thirdparty_url_origin=" (from environment)"
+  yb_thirdparty_url_origin="from environment"
 fi
 
 yb_linuxbrew_dir_origin=""
 if [[ -n ${YB_LINUXBREW_DIR:-} ]]; then
-  yb_linuxbrew_dir_origin=" (from environment)"
+  yb_linuxbrew_dir_origin="from environment"
 fi
 
 yb_llvm_toolchain_url_origin=""
 if [[ -n ${YB_LLVM_TOOLCHAIN_URL:-} ]]; then
-  yb_llvm_toolchain_url_origin=" (from environment)"
+  yb_llvm_toolchain_url_origin="from environment"
 fi
 
 yb_llvm_toolchain_dir_origin=""
 if [[ -n ${YB_LLVM_TOOLCHAIN_DIR:-} ]]; then
-  yb_llvm_toolchain_dir_origin=" (from environment)"
+  yb_llvm_toolchain_dir_origin="from environment"
 fi
 
 # To deduplicate Maven arguments
@@ -395,7 +395,7 @@ set_build_root() {
 
   normalize_build_root
 
-  if "$make_build_root_readonly"; then
+  if [[ ${make_build_root_readonly} == "true" ]]; then
     readonly BUILD_ROOT
   fi
 
@@ -714,7 +714,7 @@ create_mvn_repo_path_file() {
 }
 
 set_mvn_parameters() {
-  if "$yb_mvn_parameters_already_set"; then
+  if [[ ${yb_mvn_parameters_already_set} == "true" ]]; then
     return
   fi
   if is_jenkins; then
@@ -1217,7 +1217,7 @@ download_thirdparty() {
           "'$extracted_dir'"
   fi
   export YB_THIRDPARTY_DIR=$extracted_dir
-  yb_thirdparty_dir_origin=" (downloaded from $YB_THIRDPARTY_URL)"
+  yb_thirdparty_dir_origin="downloaded from $YB_THIRDPARTY_URL"
   save_thirdparty_info_to_build_dir
   download_toolchain
 }
@@ -1251,6 +1251,7 @@ download_toolchain() {
   if [[ -z ${YB_LLVM_TOOLCHAIN_URL:-} &&
         ${YB_COMPILER_TYPE:-} =~ ^clang[0-9]+$ ]] && is_linux; then
     YB_LLVM_TOOLCHAIN_URL=$(
+      activate_virtualenv &>/dev/null
       python3 -m llvm_installer --print-url "--llvm-major-version=${YB_COMPILER_TYPE#clang}"
     )
     if [[ ${YB_LLVM_TOOLCHAIN_URL} != https://* ]]; then
@@ -1292,20 +1293,24 @@ download_toolchain() {
               "'$extracted_dir'"
       fi
       export YB_LINUXBREW_DIR=$extracted_dir
-      yb_linuxbrew_dir_origin=" (downloaded from $toolchain_url)"
+      yb_linuxbrew_dir_origin="downloaded from $toolchain_url"
       save_brew_path_to_build_dir
     fi
 
     if [[ ${is_llvm} == "true" ]]; then
       if [[ -n ${YB_LLVM_TOOLCHAIN_DIR:-} &&
-            $YB_LLVM_TOOLCHAIN_DIR != "$extracted_dir" ]]; then
+            $YB_LLVM_TOOLCHAIN_DIR != "$extracted_dir" &&
+            ${YB_LLVM_TOOLCHAIN_DIR_MISMATCH_WARNING_LOGGED:-0} == "0" ]]; then
         log_thirdparty_and_toolchain_details
-        fatal "YB_LLVM_TOOLCHAIN_DIR is already set to '$YB_LLVM_TOOLCHAIN_DIR', cannot set it " \
-              "to '$extracted_dir'"
+        log "Warning: YB_LLVM_TOOLCHAIN_DIR is already set to '$YB_LLVM_TOOLCHAIN_DIR', cannot " \
+            "set it to '$extracted_dir'. This may happen in case the LLVM toolchain version used " \
+            "to built third-party dependencies is different from the most recent one for the " \
+            "same major version that we have just picked to build YugabyteDB with."
+        export YB_LLVM_TOOLCHAIN_DIR_MISMATCH_WARNING_LOGGED=1
       fi
       export YB_LLVM_TOOLCHAIN_DIR=$extracted_dir
-      yb_llvm_toolchain_dir_origin=" (downloaded from $toolchain_url)"
-      save_llvm_toolchain_path_to_build_dir
+      yb_llvm_toolchain_dir_origin="downloaded from $toolchain_url"
+      save_llvm_toolchain_info_to_build_dir
     fi
   done
 }
@@ -1375,9 +1380,10 @@ save_brew_path_to_build_dir() {
   fi
 }
 
-save_llvm_toolchain_path_to_build_dir() {
+save_llvm_toolchain_info_to_build_dir() {
   if is_linux; then
     save_var_to_file_in_build_dir "${YB_LLVM_TOOLCHAIN_DIR:-}" "llvm_path.txt"
+    save_var_to_file_in_build_dir "${YB_LLVM_TOOLCHAIN_URL:-}" "llvm_url.txt"
   fi
 }
 
@@ -1386,10 +1392,10 @@ save_thirdparty_info_to_build_dir() {
   save_var_to_file_in_build_dir "${YB_THIRDPARTY_URL:-}" "thirdparty_url.txt"
 }
 
-save_paths_to_build_dir() {
+save_paths_and_archive_urls_to_build_dir() {
   save_brew_path_to_build_dir
   save_thirdparty_info_to_build_dir
-  save_llvm_toolchain_path_to_build_dir
+  save_llvm_toolchain_info_to_build_dir
 }
 
 detect_linuxbrew() {
@@ -1411,7 +1417,7 @@ detect_linuxbrew() {
   then
     YB_LINUXBREW_DIR=$(<"$BUILD_ROOT/linuxbrew_path.txt")
     export YB_LINUXBREW_DIR
-    yb_linuxbrew_dir_origin=" (from file '$BUILD_ROOT/linuxbrew_path.txt')"
+    yb_linuxbrew_dir_origin="from file '$BUILD_ROOT/linuxbrew_path.txt')"
     return
   fi
 }
@@ -1425,7 +1431,7 @@ detect_llvm_toolchain() {
   if [[ $is_clean_build != "true" && -n ${BUILD_ROOT:-} && -f $BUILD_ROOT/llvm_path.txt ]]; then
     YB_LLVM_TOOLCHAIN_DIR=$(<"$BUILD_ROOT/llvm_path.txt")
     export YB_LLVM_TOOLCHAIN_DIR
-    yb_llvm_toolchain_dir_origin=" (from file '$BUILD_ROOT/llvm_path.txt')"
+    yb_llvm_toolchain_dir_origin="from file '$BUILD_ROOT/llvm_path.txt')"
   fi
 }
 
@@ -1583,7 +1589,7 @@ get_build_worker_list() {
           break
         fi
       done
-      if "$all_worker_names_valid"; then
+      if [[ ${all_worker_names_valid} == "true" ]]; then
         return
       fi
     fi
@@ -1715,7 +1721,6 @@ debugging_remote_compilation() {
 }
 
 cmd_line_to_env_vars_for_remote_cmd() {
-  declare -i i=1
   YB_ENCODED_REMOTE_CMD_LINE=""
   # This must match the separator in remote_cmd.sh.
   declare -r ARG_SEPARATOR=$'=:\t:='
@@ -1851,7 +1856,7 @@ find_or_download_thirdparty() {
               "'$BUILD_ROOT/thirdparty_path.txt' contains '$thirdparty_dir_from_file'"
       fi
       export YB_THIRDPARTY_DIR=$thirdparty_dir_from_file
-      yb_thirdparty_dir_origin=" (from file '$BUILD_ROOT/thirdparty_path.txt')"
+      yb_thirdparty_dir_origin="from file '$BUILD_ROOT/thirdparty_path.txt')"
 
       # Check if we've succeeded in setting YB_THIRDPARTY_DIR now.
       if [[ -n ${YB_THIRDPARTY_DIR:-} ]]; then
@@ -1871,7 +1876,7 @@ find_or_download_thirdparty() {
               "'$BUILD_ROOT/thirdparty_url.txt' contains '$thirdparty_url_from_file'"
       fi
       export YB_THIRDPARTY_URL=$thirdparty_url_from_file
-      yb_thirdparty_url_origin=" (from file '$BUILD_ROOT/thirdparty_url.txt')"
+      yb_thirdparty_url_origin="from file '$BUILD_ROOT/thirdparty_url.txt')"
       if [[ ${YB_DOWNLOAD_THIRDPARTY:-} == "0" ]]; then
         fatal "YB_DOWNLOAD_THIRDPARTY is explicitly set to 0 but file" \
               "$BUILD_ROOT/thirdparty_url.txt exists"
@@ -1895,7 +1900,7 @@ find_or_download_thirdparty() {
 
   if [[ -z ${YB_THIRDPARTY_DIR:-} ]]; then
     export YB_THIRDPARTY_DIR=$YB_SRC_ROOT/thirdparty
-    yb_thirdparty_dir_origin=" (default)"
+    yb_thirdparty_dir_origin="default"
   fi
   save_thirdparty_info_to_build_dir
 }
@@ -1920,28 +1925,33 @@ find_or_download_ysql_snapshots() {
   done
 }
 
+log_env_var() {
+  expect_num_args 2 "$@"
+  local env_var_name=$1
+  local env_var_value=${!env_var_name:-}
+  if [[ -z ${env_var_value} ]]; then
+    return
+  fi
+  local description=$2
+  if [[ -n ${description} ]]; then
+    description=" (${description})"
+  fi
+  echo "    ${env_var_name}: ${env_var_value}${description}"
+}
+
 log_thirdparty_and_toolchain_details() {
   (
     echo "Details of third-party dependencies:"
-    echo "    YB_THIRDPARTY_DIR: ${YB_THIRDPARTY_DIR:-undefined}$yb_thirdparty_dir_origin"
-    if is_linux && [[ -n ${YB_LINUXBREW_DIR:-} ]]; then
-      echo "    YB_LINUXBREW_DIR: $YB_LINUXBREW_DIR$yb_linuxbrew_dir_origin"
-    fi
-    if [[ -n ${YB_LLVM_TOOLCHAIN_URL:-} ]]; then
-      echo "    YB_LLVM_TOOLCHAIN_URL: $YB_LLVM_TOOLCHAIN_URL$yb_llvm_toolchain_url_origin"
-    fi
-    if [[ -n ${YB_LLVM_TOOLCHAIN_DIR:-} ]]; then
-      echo "    YB_LLVM_TOOLCHAIN_DIR: $YB_LLVM_TOOLCHAIN_DIR$yb_llvm_toolchain_dir_origin"
-    fi
-    if [[ -n ${YB_THIRDPARTY_URL:-} ]]; then
-      echo "    YB_THIRDPARTY_URL: $YB_THIRDPARTY_URL$yb_thirdparty_url_origin"
-    fi
-    if [[ -n ${YB_DOWNLOAD_THIRDPARTY:-} ]]; then
-      echo "    YB_DOWNLOAD_THIRDPARTY: $YB_DOWNLOAD_THIRDPARTY"
-    fi
-    if [[ -n ${NO_REBUILD_THIRDPARTY:-} ]]; then
-      echo "    NO_REBUILD_THIRDPARTY: ${NO_REBUILD_THIRDPARTY}"
+    log_env_var YB_THIRDPARTY_DIR "${yb_thirdparty_dir_origin}"
+    log_env_var YB_THIRDPARTY_URL "${yb_thirdparty_url_origin}"
+
+    if is_linux; then
+      log_env_var YB_LINUXBREW_DIR "${yb_linuxbrew_dir_origin}"
     fi
+    log_env_var YB_LLVM_TOOLCHAIN_URL "${yb_llvm_toolchain_url_origin}"
+    log_env_var YB_LLVM_TOOLCHAIN_DIR "${yb_llvm_toolchain_dir_origin}"
+    log_env_var YB_DOWNLOAD_THIRDPARTY ""
+    log_env_var NO_REBUILD_THIRDPARTY ""
   ) >&2
 }
 
@@ -2229,7 +2239,7 @@ activate_virtualenv() {
   fi
 
   if [[ ! -d $virtualenv_dir ]]; then
-    if "$yb_readonly_virtualenv"; then
+    if [[ ${yb_readonly_virtualenv} == "true" ]]; then
       fatal "virtualenv does not exist at '$virtualenv_dir', and we are not allowed to create it"
     fi
     if [[ -n ${VIRTUAL_ENV:-} && -f $VIRTUAL_ENV/bin/activate ]]; then
@@ -2438,7 +2448,7 @@ run_with_retries() {
 debug_log_boolean_function_result() {
   expect_num_args 1 "$@"
   local fn_name=$1
-  if "$fn_name"; then
+  if [[ ${fn_name} == "true" ]]; then
     log "$fn_name is true"
   else
     log "$fn_name is false"
@@ -2489,7 +2499,7 @@ set_prebuilt_thirdparty_url() {
       "${thirdparty_tool_cmd_line[@]}"
       YB_THIRDPARTY_URL=$(<"$BUILD_ROOT/thirdparty_url.txt")
       export YB_THIRDPARTY_URL
-      yb_thirdparty_url_origin=" (determined automatically based on the OS and compiler type)"
+      yb_thirdparty_url_origin="determined automatically based on the OS and compiler type"
       if [[ -z $YB_THIRDPARTY_URL ]]; then
         fatal "Could not automatically determine the third-party archive URL to download."
       fi
@@ -2499,7 +2509,7 @@ set_prebuilt_thirdparty_url() {
       if [[ -f $llvm_url_file_path ]]; then
         YB_LLVM_TOOLCHAIN_URL=$(<"$llvm_url_file_path")
         export YB_LLVM_TOOLCHAIN_URL
-        yb_llvm_toolchain_url_origin=" (determined automatically based on the OS and compiler type)"
+        yb_llvm_toolchain_url_origin="determined automatically based on the OS and compiler type"
         log "Setting LLVM toolchain URL to $YB_LLVM_TOOLCHAIN_URL"
         save_var_to_file_in_build_dir "$YB_LLVM_TOOLCHAIN_URL" llvm_url.txt
       fi

build-support/common-test-env.sh

@@ -476,7 +476,7 @@ prepare_for_running_cxx_test() {
   mkdir_safe "$test_dir"
 
   rel_test_log_path_prefix="$test_binary_sanitized"
-  if "$run_at_once"; then
+  if [[ ${run_at_once} == "true" ]]; then
     # Make this similar to the case when we run tests separately. Pretend that the test binary name
     # is the test name.
     rel_test_log_path_prefix+="/${rel_test_binary##*/}"
@@ -802,7 +802,7 @@ handle_cxx_test_xml_output() {
     log "Process tree supervisor reported that the test failed"
     test_failed=true
   fi
-  if "$test_failed"; then
+  if [[ ${test_failed} == "true" ]]; then
     log "Test failed, updating $xml_output_file"
   else
     log "Test succeeded, updating $xml_output_file"
@@ -962,7 +962,7 @@ handle_cxx_test_failure() {
     test_failed=true
   fi
 
-  if "$test_failed"; then
+  if [[ ${test_failed} == "true" ]]; then
     (
       rewrite_test_log "${test_log_path}"
       echo
@@ -2083,7 +2083,7 @@ run_cmake_unit_tests() {
       fi
     done
   done
-  if "$error"; then
+  if [[ ${error} == "true" ]]; then
     fatal "Found some disallowed patterns in CMake files"
   else
     log "Validated ${#cmake_files[@]} CMake files using light-weight grep checks"