Skip to content

Commit

Permalink
Server explains the reasons to the worker if no tasks are available
Browse files Browse the repository at this point in the history
  • Loading branch information
maximmasiutin committed Mar 19, 2023
1 parent be4894b commit 0762816
Show file tree
Hide file tree
Showing 2 changed files with 48 additions and 2 deletions.
28 changes: 27 additions & 1 deletion server/fishtest/rundb.py
Original file line number Diff line number Diff line change
Expand Up @@ -766,6 +766,13 @@ def priority(run): # lower is better

run_found = False

worker_low_memory = False
worker_too_many_threads = False
worker_too_few_threads = False
worker_no_binary = False
worker_tc_too_short = False
worker_core_limit_reached = False

for run in self.task_runs:
if run["finished"]:
continue
Expand All @@ -774,9 +781,11 @@ def priority(run): # lower is better
continue

if run["args"]["threads"] > max_threads:
worker_too_many_threads = true
continue

if run["args"]["threads"] < min_threads:
worker_too_few_threads = true
continue

# Check if there aren't already enough workers
Expand Down Expand Up @@ -811,6 +820,7 @@ def priority(run): # lower is better
)

if need_base + need_tt > max_memory:
worker_low_memory = True
continue

# Github API limit...
Expand All @@ -820,6 +830,7 @@ def priority(run): # lower is better
and run["_id"] in self.worker_runs[unique_key]
)
if not have_binary:
worker_no_binary = True
continue

# To avoid time losses in the case of large concurrency and short TC,
Expand All @@ -837,6 +848,7 @@ def priority(run): # lower is better
< 1.0
)
if tc_too_short:
worker_tc_too_short = True
continue

# Limit the number of cores.
Expand All @@ -856,6 +868,7 @@ def priority(run): # lower is better
break

if core_limit_reached:
worker_core_limit_reached = True
continue

# If we make it here, it means we have found a run
Expand All @@ -865,7 +878,20 @@ def priority(run): # lower is better

# If there is no suitable run, tell the worker.
if not run_found:
return {"task_waiting": False}
ret = {"task_waiting": False}
if worker_low_memory is True:
ret["worker_low_memory"] = True
if worker_too_many_threads is True:
ret["worker_too_many_threads"] = True
if worker_too_few_threads is True:
ret["worker_too_few_threads"] = True
if worker_no_binary is True:
ret["worker_no_binary"] = True
if worker_tc_too_short is True:
ret["worker_tc_too_short"] = True
if worker_core_limit_reached is True:
ret["worker_core_limit_reached"] = True
return ret

# Now we create a new task for this run.
opening_offset = 0
Expand Down
22 changes: 21 additions & 1 deletion worker/worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -717,6 +717,7 @@ def _get_help_string(self, action):
"-t",
"--min_threads",
dest="min_threads",
metavar="MIN_THREADS",
default=config.getint("parameters", "min_threads"),
type=int,
help="do not accept tasks with fewer threads than MIN_THREADS",
Expand Down Expand Up @@ -1336,7 +1337,26 @@ def fetch_and_handle_task(worker_info, password, remote, lock_file, current_stat

# No tasks ready for us yet, just wait...
if "task_waiting" in req:
print("No tasks available at this time, waiting...")
reasons = []
if "worker_low_memory" in req:
reasons.append("The MAX_MEMORY parameter is too low")
if "worker_too_many_threads" in req:
reasons.append("The CONCURRENCY parameter is too high")
if "worker_too_few_threads" in req:
reasons.append("The MIN_THREADS parameter is too low")
if "worker_no_binary" in req:
reasons.append("No binary or near Github API limit")
if "worker_tc_too_short" in req:
reasons.append("The TC is too short")
if "worker_core_limit_reached" in req:
reasons.append("Exceeded the limit of cores set by the server")
if len(reasons) == 0:
print("No tasks available at this time, waiting...")
else:
print("No suitable tasks available for the worker at this time, reason(s):")
for reason in reasons:
print(" - {}".format(reason))
print("Waiting...")
return False

run, task_id = req["run"], req["task_id"]
Expand Down

0 comments on commit 0762816

Please sign in to comment.