Fix Jammed Backlog

skydriver/config.py

@@ -79,6 +79,15 @@ def __post_init__(self) -> None:
                 "Missing required environment variable: 'CLIENTMANAGER_IMAGE_WITH_TAG'"
             )
 
+        if self.SCAN_BACKLOG_RUNNER_SHORT_DELAY > self.SCAN_BACKLOG_RUNNER_DELAY:
+            raise RuntimeError(
+                "'SCAN_BACKLOG_RUNNER_SHORT_DELAY' cannot be greater than 'SCAN_BACKLOG_RUNNER_DELAY'"
+            )
+        if self.SCAN_BACKLOG_RUNNER_DELAY > self.SCAN_BACKLOG_PENDING_ENTRY_TTL_REVIVE:
+            raise RuntimeError(
+                "'SCAN_BACKLOG_RUNNER_DELAY' cannot be greater than 'SCAN_BACKLOG_PENDING_ENTRY_TTL_REVIVE'"
+            )
+
 
 ENV = from_environment_as_dataclass(EnvConfig)
 

skydriver/k8s/scan_backlog.py

@@ -83,6 +83,7 @@ async def startup(
             job_obj = pickle.loads(entry.pickled_k8s_job)
         except Exception as e:
             LOGGER.exception(e)
+            short_sleep = True  # don't wait long b/c nothing was started
             continue
 
         LOGGER.info(
@@ -97,6 +98,7 @@ async def startup(
         except kubernetes.client.exceptions.ApiException as e:
             # job (entry) will be revived & restarted in future iteration
             LOGGER.exception(e)
+            short_sleep = True  # don't wait long b/c nothing was started
             continue
 
         # remove from backlog now that startup succeeded