diff --git a/.github/workflows/gpu-hvd-tests.yml b/.github/workflows/gpu-hvd-tests.yml
index fe904366300..2017cf8acda 100644
--- a/.github/workflows/gpu-hvd-tests.yml
+++ b/.github/workflows/gpu-hvd-tests.yml
@@ -152,7 +152,7 @@ jobs:
           set -xe
 
           bash tests/run_gpu_tests.sh 2 hvd
-          CUDA_VISIBLE_DEVICES="" pytest --cov ignite --cov-append --cov-report term-missing --cov-report xml -vvv tests/ignite/ignite/ -m distributed -k hvd
+          CUDA_VISIBLE_DEVICES="" pytest --cov ignite --cov-append --cov-report term-missing --cov-report xml -vvv tests/ignite -m distributed -k hvd
 
           EOF
           )
diff --git a/tests/ignite/conftest.py b/tests/ignite/conftest.py
index c0e1785dd61..d182ff29d00 100644
--- a/tests/ignite/conftest.py
+++ b/tests/ignite/conftest.py
@@ -1,8 +1,10 @@
 import functools
 import os
 import shutil
+import signal
 import sys
 import tempfile
+import threading
 import time
 from pathlib import Path
 
@@ -34,6 +36,20 @@ def pytest_addoption(parser):
     )
 
 
+@pytest.fixture(scope="session", autouse=True)
+def term_handler():
+    # This allows the pytest session to be terminated upon retries on CI. It may
+    # be worth using this fixture solely in that context. For a discussion on
+    # whether sigterm should be ignored see:
+    # https://github.com/pytest-dev/pytest/issues/5243
+    if threading.current_thread() is threading.main_thread() and hasattr(signal, "SIGTERM"):
+        orig = signal.signal(signal.SIGTERM, signal.getsignal(signal.SIGINT))
+        yield
+        signal.signal(signal.SIGTERM, orig)
+    else:
+        yield  # Just pass through if SIGTERM isn't supported or we are not in the main thread
+
+
 @pytest.fixture(
     params=[
         "cpu",
diff --git a/tests/run_gpu_tests.sh b/tests/run_gpu_tests.sh
index a1b7eef70a0..371c70aee37 100644
--- a/tests/run_gpu_tests.sh
+++ b/tests/run_gpu_tests.sh
@@ -27,7 +27,7 @@ if [ "${skip_distrib_tests}" -eq "1" ]; then
 fi
 
 run_tests \
-    --core_args "-vvv -m distributed tests" \
+    --core_args "-vvv -m distributed tests/ignite" \
     --cache_dir ".gpu-distrib" \
     --skip_distrib_tests 0 \
     --use_coverage 1 \
@@ -37,7 +37,7 @@ run_tests \
 
 if [ ${ngpus} -gt 1 ]; then
     run_tests \
-        --core_args "-vvv -m distributed tests" \
+        --core_args "-vvv -m distributed tests/ignite" \
         --world_size "${ngpus}" \
         --cache_dir ".gpu-distrib-multi" \
         --skip_distrib_tests 0 \
diff --git a/tests/run_tpu_tests.sh b/tests/run_tpu_tests.sh
index 61e71b2ef1f..5ea0f993173 100644
--- a/tests/run_tpu_tests.sh
+++ b/tests/run_tpu_tests.sh
@@ -4,7 +4,7 @@ set -xeu
 use_last_failed=${USE_LAST_FAILED:-0}
 
 run_tests \
-    --core_args "-vvv -m tpu tests" \
+    --core_args "-vvv -m tpu tests/ignite" \
     --cache_dir ".tpu" \
     --use_coverage 1 \
     --use_last_failed ${use_last_failed}
@@ -13,7 +13,7 @@ run_tests \
 if [ -z ${NUM_TPU_WORKERS+x} ]; then
     export NUM_TPU_WORKERS=1
     run_tests \
-        --core_args "-vvv -m tpu tests" \
+        --core_args "-vvv -m tpu tests/ignite" \
         --cache_dir ".tpu-multi" \
         --use_coverage 1 \
         --use_last_failed ${use_last_failed}