Merge #33

33: Run more CI tests. r=ltratt a=vext01

I've enabled only those that run successfully to completion "quickly enough" when `YKD_SERIALISE_COMPILATION` is set.

Obviously we want to converge upon running all tests via `main.lua`, but this gives us a little more confidence in our work for now.

Co-authored-by: Edd Barrett <[email protected]>

.buildbot.sh

@@ -30,12 +30,24 @@ cd ..
 
 export YK_BUILD_TYPE=debug
 make -j `nproc`
+
 cd tests
 # YKFIXME: The JIT can't yet run the test suite, but the following commented
 # commands are what we are aiming at having work.
 #
-# YKFIXME: The tests are very short-lived and will probably need to be made to
-# run longer to allow for JIT compilation to complete.
+# ../src/lua -e"_U=true" all.lua
+# YKD_SERIALISE_COMPILATION=1 ../src/lua -e"_U=true" all.lua
+#
+# (Adding `YKD_SERIALISE_COMPILATION` exercises different threading behaviour
+# that could help to shake out more bugs)
 #
-#../src/lua -e"_U=true" all.lua
-#YKD_SERIALISE_COMPILATION=1 ../src/lua -e"_U=true" all.lua
+# Until we can run `all.lua` reliably, we just run the tests that we know to
+# run within reasonable time).
+LUA=../src/lua
+for serialise in 0 1; do
+    for test in api bwcoercion closure code constructs coroutine events \
+        files gengc goto literals pm tpack tracegc utf8 vararg; do
+        echo "### YKD_SERIALISE_COMPILATION=$serialise $test.lua ###"
+        YKD_SERIALISE_COMPILATION=$serialise ${LUA} ${test}.lua
+    done
+done

try_all_tests.sh

@@ -0,0 +1,65 @@
+#!/bin/sh
+#
+# This script tries to run as many individual tests as possible, reporting
+# those which succeed for both YKD_SERIALISE_COMPILATION=0 and
+# YKD_SERIALISE_COMPILATION=1.
+#
+# You can use the output to adjust which tests run in `.buildbot.sh`.
+#
+# As soon as `all.lua` runs, we can just run that and delete this script.
+#
+# Note that many tests require setup code in `all.lua` to generate some files
+# for the tests to run properly. So some tests will be failing due to that.
+
+set -e
+
+if [ $(uname) != "Linux" ]; then
+    # Due to descrepancies of the exit status of `timeout`.
+    echo "this script only runs on linux"
+    exit 1
+fi
+
+SECS=120
+LUA=src/lua
+
+OK=""
+FAIL=""
+for tst in tests/*.lua; do
+    btst=$(basename ${tst})
+    echo -n "> $tst(YKD_SERIALISE_COMPILATION=0)..."
+    exstatus=0
+    YKD_SERIALISE_COMPILATION=0 timeout -s9 ${SECS}s ${LUA} ${tst} \
+        >/dev/null 2>&1 || exstatus=$?
+
+    if [ ${exstatus} -eq 0 ]; then
+        # Try test again with JIT compilation on the main thread. Only if this
+        # also succeeds do we classify the test as "OK".
+        echo "OK"
+        echo -n "> $tst(YKD_SERIALISE_COMPILATION=1)..."
+        exstatus=0
+        YKD_SERIALISE_COMPILATION=1 timeout -s9 ${SECS}s ${LUA} ${tst} \
+            >/dev/null 2>&1 || exstatus=$?
+        if [ ${exstatus} -eq 0 ]; then
+            echo "OK"
+            OK="${OK} ${btst}"
+        else
+            if [ ${exstatus} -eq 137 ]; then
+                echo "TIMEOUT"
+            else
+                echo "FAIL"
+            fi
+            FAIL="${FAIL} ${btst}"
+        fi
+    else
+        if [ ${exstatus} -eq 137 ]; then
+            echo "TIMEOUT"
+        else
+            echo "FAIL"
+        fi
+        FAIL="${FAIL} ${btst}"
+    fi
+done
+
+echo "\n---"
+echo "OK: ${OK}"
+echo "FAIL: ${FAIL}"