add timeout for validators

bin/config.py

@@ -47,6 +47,10 @@
 
 SEED_DEPENDENCY_RETRIES = 10
 
+# The default timeout used for generators, visualizer etc.
+DEFAULT_TIMEOUT = 30
+DEFAULT_INTERACTION_TIMEOUT = 60
+
 # The root directory of the BAPCtools repository.
 tools_root = Path(__file__).resolve().parent.parent
 
@@ -64,17 +68,18 @@
     'jobs': os.cpu_count() // 2,
     'time': 600,  # Used for `bt fuzz`
     'verbose': 0,
+    'timeout': DEFAULT_TIMEOUT,
 }
 
 
 # The list of arguments below is generated using the following command:
 """
 for cmd in $(bapctools --help | grep '^  {' | sed 's/  {//;s/}//;s/,/ /g') ; do bapctools $cmd --help ; done |& \
 grep '^  [^ ]' | sed 's/^  //' | cut -d ' ' -f 1 | sed -E 's/,//;s/^-?-?//;s/-/_/g' | sort -u | \
-grep -Ev '^(h|jobs|time|verbose)$' | sed "s/^/'/;s/$/',/" | tr '\n' ' ' | sed 's/^/args_list = [/;s/, $/]\n/'
+grep -Ev '^(h|jobs|time|verbose|timeout)$' | sed "s/^/'/;s/$/',/" | tr '\n' ' ' | sed 's/^/args_list = [/;s/, $/]\n/'
 """
 # fmt: off
-args_list = ['1', 'add_manual', 'all', 'api', 'author', 'check_deterministic', 'clean', 'clean_generated', 'cleanup_generated', 'contest', 'contest_id', 'contestname', 'cp', 'cpp_flags', 'default_solution', 'directory', 'error', 'force', 'force_build', 'ignore_validators', 'input', 'interaction', 'interactive', 'kattis', 'memory', 'move_manual', 'move_to', 'no_bar', 'no_generate', 'no_solutions', 'no_timelimit', 'order', 'order_from_ccs', 'output', 'password', 'problem', 'problemname', 'remove', 'samples', 'scoreboard_repo', 'skel', 'skip', 'skip_solution', 'skip_testcase_sanity_checks', 'skip_visualizer', 'submissions', 'table', 'testcases', 'timelimit', 'timeout', 'username', 'validation', 'watch', 'web']
+args_list = ['1', 'add_manual', 'all', 'api', 'author', 'check_deterministic', 'clean', 'clean_generated', 'cleanup_generated', 'contest', 'contest_id', 'contestname', 'cp', 'cpp_flags', 'default_solution', 'directory', 'error', 'force', 'force_build', 'ignore_validators', 'input', 'interaction', 'interactive', 'kattis', 'memory', 'move_manual', 'move_to', 'no_bar', 'no_generate', 'no_solutions', 'no_timelimit', 'order', 'order_from_ccs', 'output', 'password', 'problem', 'problemname', 'remove', 'samples', 'scoreboard_repo', 'skel', 'skip', 'skip_solution', 'skip_testcase_sanity_checks', 'skip_visualizer', 'submissions', 'table', 'testcases', 'timelimit', 'username', 'validation', 'watch', 'web']
 # fmt: on
 
 

bin/interactive.py

@@ -37,7 +37,7 @@ def run_interactive_testcase(
     output_validator = output_validators[0]
 
     # Set limits
-    validator_timeout = 60
+    validator_timeout = config.DEFAULT_INTERACTION_TIMEOUT
 
     memory_limit = get_memory_limit()
     timelimit = run.problem.settings.timelimit

bin/program.py

@@ -5,6 +5,7 @@
 import threading
 
 from util import *
+from colorama import Fore
 
 EXTRA_LANGUAGES = '''
 checktestdata:
@@ -464,7 +465,7 @@ def run(self, bar, cwd, name, args=[]):
             else:
                 f.unlink()
 
-        timeout = config.args.timeout or 30
+        timeout = config.args.timeout
 
         with stdout_path.open('w') as stdout_file:
             result = exec_command(
@@ -475,7 +476,7 @@ def run(self, bar, cwd, name, args=[]):
 
         if result.ok == -9:
             # Timeout -> stop retrying and fail.
-            bar.error(f'TIMEOUT after {timeout}s')
+            bar.log(f'TIMEOUT after {timeout}s',color=Fore.RED)
             return result
 
         if result.ok is not True:
@@ -490,7 +491,7 @@ def run(self, bar, cwd, name, args=[]):
                 stdout_path.rename(in_path)
         else:
             if not in_path.is_file():
-                bar.error(f'Did not write {name}.in and stdout is empty!')
+                bar.log(f'Did not write {name}.in and stdout is empty!',color=Fore.RED)
                 result.ok = False
                 return result
 
@@ -504,4 +505,4 @@ class Visualizer(Program):
     # Stdin and stdout are not used.
     def run(self, cwd, args=[]):
         assert self.run_command is not None
-        return exec_command(self.run_command + args, timeout=config.args.timeout or 30, cwd=cwd)
+        return exec_command(self.run_command + args, timeout=config.args.timeout, cwd=cwd)

bin/tools.py

@@ -441,6 +441,9 @@ def build_parser():
         action='store_true',
         help='Skip sanity checks on testcases.',
     )
+    validate_parser.add_argument(
+        '--timeout', '-t', type=int, help='Override the default timeout. Default: 30.'
+    )
 
     # constraints validation
     constraintsparser = subparsers.add_parser(

bin/util.py

@@ -792,7 +792,7 @@ def exec_command(command, expect=0, crop=True, **kwargs):
             print(' < ', kwargs['stdin'].name, end='', file=sys.stderr)
         print(file=sys.stderr)
 
-    timeout = 30
+    timeout = config.DEFAULT_TIMEOUT
     if 'timeout' in kwargs:
         if kwargs['timeout'] is None:
             timeout = None

bin/validate.py

@@ -98,6 +98,7 @@ def run(self, testcase, constraints=None, args=None):
                 expect=config.RTV_WA if testcase.bad_input else config.RTV_AC,
                 stdin=in_file,
                 cwd=cwd,
+                timeout=config.args.timeout,
             )
 
         if constraints is not None:
@@ -150,6 +151,7 @@ def run(self, testcase, run=None, constraints=None, args=None):
                     expect=config.RTV_WA if testcase.bad_output else config.RTV_AC,
                     stdin=ans_file,
                     cwd=cwd,
+                    timeout=config.args.timeout,
                 )
 
             if constraints is not None:
@@ -171,4 +173,5 @@ def run(self, testcase, run=None, constraints=None, args=None):
                 expect=config.RTV_AC,
                 stdin=out_file,
                 cwd=run.feedbackdir,
+                timeout=self.problem.settings.timeout,
             )