From 06cdc9b8a90a1adfc1d6ffd08fd8b42d51d25679 Mon Sep 17 00:00:00 2001 From: Michael Schmuki Date: Mon, 27 May 2024 14:12:59 +0200 Subject: [PATCH 1/2] Execute "after_docker_exception" on DOCKER_ENGINE_SIGKILL (this ensures status of related deltas gets updated) --- docker-app/worker_wrapper/wrapper.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-app/worker_wrapper/wrapper.py b/docker-app/worker_wrapper/wrapper.py index 3c727de48..6ca48a9ee 100644 --- a/docker-app/worker_wrapper/wrapper.py +++ b/docker-app/worker_wrapper/wrapper.py @@ -170,8 +170,6 @@ def run(self): "Failed to update job status, probably does not exist in the database.", exc_info=err, ) - # No further action required, probably received by wrapper's autoclean mechanism when the `Project` is deleted - return elif exit_code == TIMEOUT_ERROR_EXIT_CODE: feedback["error"] = "Worker timeout error." feedback["error_type"] = "TIMEOUT" From 190cb6af54d1f5e84dc936133cf2116710e6551f Mon Sep 17 00:00:00 2001 From: Michael Schmuki Date: Mon, 27 May 2024 15:52:40 +0200 Subject: [PATCH 2/2] Ensure no "after_docker_exception" when a SIGKILLed job doesn't exist in the db (anymore) --- docker-app/worker_wrapper/wrapper.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/docker-app/worker_wrapper/wrapper.py b/docker-app/worker_wrapper/wrapper.py index 6ca48a9ee..4f0108226 100644 --- a/docker-app/worker_wrapper/wrapper.py +++ b/docker-app/worker_wrapper/wrapper.py @@ -157,19 +157,14 @@ def run(self): feedback["error_stack"] = "" try: - logger.info( - "Set job status to `failed` due to being killed by the docker engine.", - ) - - self.job.output = output.decode("utf-8") - self.job.feedback = feedback - self.job.status = Job.Status.FAILED - self.job.save(update_fields=["output", "feedback", "status"]) + self.job.refresh_from_db() except Exception as err: logger.error( "Failed to update job status, probably does not exist in the database.", exc_info=err, ) + # No further action required, probably received by wrapper's autoclean mechanism when the `Project` is deleted + return elif exit_code == TIMEOUT_ERROR_EXIT_CODE: feedback["error"] = "Worker timeout error." feedback["error_type"] = "TIMEOUT"