test: runner script

lib/tests/README.md

@@ -0,0 +1,34 @@
+## Running bazel-lib tests
+
+These four commands should be used to validate aspect-lib code
+
+```
+cd <workspace_dir>
+1. bazel test //lib/tests/... --enable_runfiles
+2. bazel test //lib/tests/... --noenable_runfiles
+3. lib/tests/test_with_run.sh //lib/tests/... --enable_runfiles
+4. lib/tests/test_with_run.sh //lib/tests/... --noenable_runfiles
+```
+
+These four commands each give a different set of results. `bazel run` behaves differently 
+to `bazel test` in how it sets up the environment, so commands 3 and 4 give a more accurate 
+validation of tools used in this way.
+
+Note: On linux, commands 1 and 2 currently have the same behaviour (the --noenable_runfiles 
+flag appears to be ignored with bazel test), I have an issue requesting clarification.
+
+### test_with_run.sh
+
+#### !This script makes changes to the source tree!
+WARNING: This script will cause data to be written to the source tree. Some outputs may interfere between test cases. I have not found a way to avoid that. This is discussed here: https://github.com/bazelbuild/bazel/issues/3325. I've tried --run_under and can't get it to work on windows. 
+
+We may be able to use git to automate deletion of test temporary data
+
+#### Exit code
+test_with_run.sh returns 1 if there are any failures and 0 if all tests pass
+
+### test_with_run.bat
+On windows, there is a bat wrapper. %BAZEL_SH% must be set to bash
+```
+lib\tests\test_with_run.bat //lib/tests/... --enable_runfiles
+```

lib/tests/test.bat

@@ -0,0 +1,4 @@
+@set dir=%~dp0
+@set dir=%dir:\=/%
+@echo %BAZEL_SH% -c "%dir%test.sh %*"
+@%BAZEL_SH% -c "%dir%test.sh %*"

lib/tests/test.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set +x
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+diffs=$(git diff --name-only lib/tests)
+if [ "$diffs" != "" ]; then
+    echo "ERROR: changes to test source tree prior to running tests:"
+    echo "${diffs[@]}"
+    echo commit or run 'git checkout lib/tests/*' to discard
+    exit 1
+fi
+
+bazel clean >/dev/null 2>&1
+bazel test "$@" --enable_runfiles > bazel_test_enable_runfiles.log 2>&1
+result=$(grep -m1 -E "Executed" bazel_test_enable_runfiles.log)
+echo "[test enable runfiles  ] $result"
+
+bazel clean >/dev/null 2>&1
+bazel test "$@" --noenable_runfiles > bazel_test_noenable_runfiles.log 2>&1
+result=$(grep -m1 -E "Executed" bazel_test_noenable_runfiles.log)
+echo "[test noenable runfiles] $result"
+
+bazel clean >/dev/null 2>&1
+$SCRIPT_DIR/test_with_run.sh "$@" --enable_runfiles > bazel_run_enable_runfiles.log 2>&1
+result=$(grep -m1 -E "Executed" bazel_run_enable_runfiles.log)
+echo "[run enable runfiles   ] $result"
+git checkout lib/tests/ >/dev/null
+
+bazel clean >/dev/null 2>&1
+$SCRIPT_DIR/test_with_run.sh "$@" --noenable_runfiles > bazel_run_noenable_runfiles.log 2>&1
+result=$(grep -m1 -E "Executed" bazel_run_noenable_runfiles.log)
+echo "[run noenable runfiles ] $result"
+git checkout lib/tests/ >/dev/null

lib/tests/test_with_run.bat

@@ -0,0 +1,4 @@
+@set dir=%~dp0
+@set dir=%dir:\=/%
+@echo %BAZEL_SH% -c "%dir%test_with_run.sh %*"
+@%BAZEL_SH% -c "%dir%test_with_run.sh %*"

lib/tests/test_with_run.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+target=${1}
+
+env=$(printenv | grep -E "^(RUNFILES|BUILD|TEST)_.*=")
+if [ "$env" != "" ]; then
+    echo "WARNING: bazel env set outside this test harness:"
+    echo "${env[@]}"
+fi
+diffs=$(git diff --name-only lib/tests)
+if [ "$diffs" != "" ]; then
+    echo "WARNING: changes to test source tree prior to running tests:"
+    echo "${diffs[@]}"
+    echo commit or run 'git checkout lib/tests/*' to discard
+fi
+
+tests=$(eval "bazel query 'kind(\".*_test\", ${target})'" | tr -d '\r')
+
+failures=0
+runs=0
+passes=0
+skips=0
+quiet_opts="--noshow_progress --ui_event_filters=,+error,+fail --show_result=0 --logging=0"
+test_opts="--skip_incompatible_explicit_targets"
+
+echo "running each target with:"
+echo "bazel run <target> $quiet_opts $test_opts $run_opts ${@:2}"
+for test in ${tests}
+do
+    runs=$((runs + 1))
+    #echo "bazel run $test $quiet_opts $test_opts $run_opts ${@:2}"
+    out=$(bazel run $test $quiet_opts $test_opts $run_opts ${@:2} 2>&1)
+    status=$?
+    if [[ $status != 0 ]]; then
+        if [[ " ${out[@]} " =~ "ERROR: No targets found to run" ]]; then
+            skips=$((++skips))
+            echo $test: skipped
+        else
+            failures=$((failures+1))
+            echo "${out[@]}"
+            echo -----------------------------------------------------------------------------
+            echo $test: fail $status
+       fi
+    else
+        #echo $test: pass
+        passes=$((++passes))
+    fi
+done
+
+echo "Executed $((runs-skips)) out of $runs tests: $passes tests pass, $failures fail and $skips were skipped."
+if [[ $failures == 0 ]]; then
+    exit 0
+else
+    exit 1
+fi