fix: txes handle sigint and sigterm, use dump fail to see any txe err…

…ors (#12427)

Please read [contributing guidelines](CONTRIBUTING.md) and remove this
line.

bootstrap.sh

@@ -118,29 +118,33 @@ function test_cmds {
   parallel -k --line-buffer './{}/bootstrap.sh test_cmds 2>/dev/null' ::: $@ | filter_test_cmds
 }
 
-function test {
-  echo_header "test all"
-
-  # Make sure KIND starts so it is running by the time we do spartan tests.
-  spartan/bootstrap.sh kind &>/dev/null &
-
+function start_txes {
   # Starting txe servers with incrementing port numbers.
-  trap 'kill $(jobs -p) &>/dev/null || true' EXIT
+  trap 'kill -SIGTERM $(jobs -p) &>/dev/null || true' EXIT
   for i in $(seq 0 $((NUM_TXES-1))); do
     existing_pid=$(lsof -ti :$((45730 + i)) || true)
     [ -n "$existing_pid" ] && kill -9 $existing_pid
-    # TODO: I'd like to use dump_fail here, but TXE needs to exit 0 on receiving a SIGTERM.
-    (cd $root/yarn-project/txe && LOG_LEVEL=silent TXE_PORT=$((45730 + i)) retry yarn start) &>/dev/null &
+    dump_fail "cd $root/yarn-project/txe && LOG_LEVEL=info TXE_PORT=$((45730 + i)) node --no-warnings ./dest/bin/index.js" &
   done
   echo "Waiting for TXE's to start..."
   for i in $(seq 0 $((NUM_TXES-1))); do
       local j=0
       while ! nc -z 127.0.0.1 $((45730 + i)) &>/dev/null; do
-        [ $j == 60 ] && echo_stderr "Warning: TXE's taking too long to start. Check them manually." && exit 1
+        [ $j == 15 ] && echo_stderr "Warning: TXE's taking too long to start. Check them manually." && exit 1
         sleep 1
         j=$((j+1))
       done
   done
+}
+export -f start_txes
+
+function test {
+  echo_header "test all"
+
+  # Make sure KIND starts so it is running by the time we do spartan tests.
+  spartan/bootstrap.sh kind &>/dev/null &
+
+  start_txes
 
   # We will start half as many jobs as we have cpu's.
   # This is based on the slightly magic assumption that many tests can benefit from 2 cpus,

ci3/aws_terminate_instance

@@ -7,8 +7,8 @@ sir=${2:-}
 export AWS_DEFAULT_REGION=us-east-2
 
 echo "Terminating instance: $iid..."
-dump_fail aws ec2 terminate-instances --instance-ids $iid > /dev/null
+dump_fail "aws ec2 terminate-instances --instance-ids $iid" > /dev/null
 
 if [ -n "$sir" ]; then
-    dump_fail aws ec2 cancel-spot-instance-requests --spot-instance-request-ids $sir > /dev/null
-fi
+    dump_fail "aws ec2 cancel-spot-instance-requests --spot-instance-request-ids $sir" > /dev/null
+fi

ci3/dump_fail

@@ -15,23 +15,31 @@ if [ "$#" -lt 1 ]; then
 fi
 
 # Need a temp to capture stderr.
+stdout=$(mktemp)
 stderr=$(mktemp)
-trap "rm $stderr" EXIT
+trap "rm $stdout $stderr" EXIT
 
-output=$(eval "$*" 2>$stderr)
+bash -c "$1" >$stdout 2>$stderr &
+pid=$!
+trap 'kill -SIGTERM $pid' SIGTERM
+# We wait twice, as the first awakening might be due to this script receiving SIGTERM.
+# In that case wait will return with 143, and the trap handler is immediately called.
+# We then wait on the pid again, to get the actual status code.
+wait $pid
+wait $pid
 status=$?
 
 if [ "$status" -ne 0 ]; then
   {
-    echo -e "${red}command failed${reset}: $*"
+    echo -e "${red}command failed${reset}: $* (exit: $status)"
     echo -e "${blue}--- stdout ---${reset}"
-    echo -e "$output"
+    cat $stdout
     echo -e "${blue}--- stderr ---${reset}"
     cat "$stderr"
     echo -e "${blue}--------------${reset}"
   } >&2
 else
-  echo -n "$output"
+  cat $stdout
 fi
 
 exit $status

yarn-project/txe/src/bin/index.ts

@@ -10,6 +10,16 @@ import { createTXERpcServer } from '../index.js';
 async function main() {
   const { TXE_PORT = 8080 } = process.env;
 
+  process.on('SIGTERM', () => {
+    logger.info('Received SIGTERM.');
+    process.exit(0);
+  });
+
+  process.on('SIGINT', () => {
+    logger.info('Received SIGTERM.');
+    process.exit(0);
+  });
+
   const logger = createLogger('txe:service');
   logger.info(`Setting up TXE...`);