From 7485c496cf61a0a6e069efd4633a1f6a5e0dd54a Mon Sep 17 00:00:00 2001 From: Ric Evans Date: Mon, 11 Sep 2023 10:46:10 -0500 Subject: [PATCH] Fix Jammed Backlog --- skydriver/config.py | 9 +++++++++ skydriver/k8s/scan_backlog.py | 2 ++ 2 files changed, 11 insertions(+) diff --git a/skydriver/config.py b/skydriver/config.py index d14ec87c..2cb3b159 100644 --- a/skydriver/config.py +++ b/skydriver/config.py @@ -79,6 +79,15 @@ def __post_init__(self) -> None: "Missing required environment variable: 'CLIENTMANAGER_IMAGE_WITH_TAG'" ) + if self.SCAN_BACKLOG_RUNNER_SHORT_DELAY > self.SCAN_BACKLOG_RUNNER_DELAY: + raise RuntimeError( + "'SCAN_BACKLOG_RUNNER_SHORT_DELAY' cannot be greater than 'SCAN_BACKLOG_RUNNER_DELAY'" + ) + if self.SCAN_BACKLOG_RUNNER_DELAY > self.SCAN_BACKLOG_PENDING_ENTRY_TTL_REVIVE: + raise RuntimeError( + "'SCAN_BACKLOG_RUNNER_DELAY' cannot be greater than 'SCAN_BACKLOG_PENDING_ENTRY_TTL_REVIVE'" + ) + ENV = from_environment_as_dataclass(EnvConfig) diff --git a/skydriver/k8s/scan_backlog.py b/skydriver/k8s/scan_backlog.py index 69751deb..8e821fcd 100644 --- a/skydriver/k8s/scan_backlog.py +++ b/skydriver/k8s/scan_backlog.py @@ -83,6 +83,7 @@ async def startup( job_obj = pickle.loads(entry.pickled_k8s_job) except Exception as e: LOGGER.exception(e) + short_sleep = True # don't wait long b/c nothing was started continue LOGGER.info( @@ -97,6 +98,7 @@ async def startup( except kubernetes.client.exceptions.ApiException as e: # job (entry) will be revived & restarted in future iteration LOGGER.exception(e) + short_sleep = True # don't wait long b/c nothing was started continue # remove from backlog now that startup succeeded