Add SkyDriver-Triggered Kill Switch for Central Server [minor] (#239)

Co-authored-by: wipacdevbot <[email protected]>

dependencies-from-Dockerfile.log

@@ -39,7 +39,7 @@ fonttools==4.29.1
 fs==2.4.12
 gast==0.5.2
 healpy==1.15.0
-htchirp==2.0
+htchirp==3.0
 htcondor==23.1.0
 html5lib==1.1
 humanfriendly==10.0
@@ -138,7 +138,7 @@ urllib3==2.1.0
 urwid==2.1.2
 wcwidth==0.2.5
 webencodings==0.5.1
-wipac-dev-tools==1.7.1
+wipac-dev-tools==1.8.1
 wipac-rest-tools==1.6.0
 xlwt==1.3.0
 zipp==1.0.0
@@ -249,10 +249,10 @@ setuptools==59.6.0
 six==1.16.0
 skymap-scanner
 ├── ewms-pilot [required: Any, installed: 0.17.5]
-│   ├── htchirp [required: Any, installed: 2.0]
+│   ├── htchirp [required: Any, installed: 3.0]
 │   ├── htcondor [required: Any, installed: 23.1.0]
 │   └── oms-mqclient [required: Any, installed: 2.4.9]
-│       └── wipac-dev-tools [required: Any, installed: 1.7.1]
+│       └── wipac-dev-tools [required: Any, installed: 1.8.1]
 │           ├── requests [required: Any, installed: 2.25.1]
 │           └── typing-extensions [required: Any, installed: 4.8.0]
 ├── healpy [required: Any, installed: 1.15.0]
@@ -263,17 +263,17 @@ skymap-scanner
 │   ├── meander [required: Any, installed: 0.0.3]
 │   ├── numpy [required: Any, installed: 1.21.5]
 │   ├── pandas [required: Any, installed: 1.3.5]
-│   └── wipac-dev-tools [required: Any, installed: 1.7.1]
+│   └── wipac-dev-tools [required: Any, installed: 1.8.1]
 │       ├── requests [required: Any, installed: 2.25.1]
 │       └── typing-extensions [required: Any, installed: 4.8.0]
 ├── iminuit [required: Any, installed: 2.24.0]
 │   └── numpy [required: >=1.21, installed: 1.21.5]
 ├── numpy [required: Any, installed: 1.21.5]
 ├── oms-mqclient [required: Any, installed: 2.4.9]
-│   └── wipac-dev-tools [required: Any, installed: 1.7.1]
+│   └── wipac-dev-tools [required: Any, installed: 1.8.1]
 │       ├── requests [required: Any, installed: 2.25.1]
 │       └── typing-extensions [required: Any, installed: 4.8.0]
-├── wipac-dev-tools [required: Any, installed: 1.7.1]
+├── wipac-dev-tools [required: Any, installed: 1.8.1]
 │   ├── requests [required: Any, installed: 2.25.1]
 │   └── typing-extensions [required: Any, installed: 4.8.0]
 └── wipac-rest-tools [required: Any, installed: 1.6.0]
@@ -287,7 +287,7 @@ skymap-scanner
     │   └── requests [required: >=1.2.0, installed: 2.25.1]
     ├── tornado [required: Any, installed: 6.3.3]
     ├── urllib3 [required: >=2.0.4, installed: 2.1.0]
-    └── wipac-dev-tools [required: Any, installed: 1.7.1]
+    └── wipac-dev-tools [required: Any, installed: 1.8.1]
         ├── requests [required: Any, installed: 2.25.1]
         └── typing-extensions [required: Any, installed: 4.8.0]
 sortedcontainers==2.1.0

dependencies-from-Dockerfile_no_cvmfs.log

@@ -39,7 +39,7 @@ fonttools==4.29.1
 fs==2.4.12
 gast==0.5.2
 healpy==1.15.0
-htchirp==2.0
+htchirp==3.0
 htcondor==23.1.0
 html5lib==1.1
 humanfriendly==10.0
@@ -138,7 +138,7 @@ urllib3==2.1.0
 urwid==2.1.2
 wcwidth==0.2.5
 webencodings==0.5.1
-wipac-dev-tools==1.7.1
+wipac-dev-tools==1.8.1
 wipac-rest-tools==1.6.0
 xlwt==1.3.0
 zipp==1.0.0
@@ -249,10 +249,10 @@ setuptools==59.6.0
 six==1.16.0
 skymap-scanner
 ├── ewms-pilot [required: Any, installed: 0.17.5]
-│   ├── htchirp [required: Any, installed: 2.0]
+│   ├── htchirp [required: Any, installed: 3.0]
 │   ├── htcondor [required: Any, installed: 23.1.0]
 │   └── oms-mqclient [required: Any, installed: 2.4.9]
-│       └── wipac-dev-tools [required: Any, installed: 1.7.1]
+│       └── wipac-dev-tools [required: Any, installed: 1.8.1]
 │           ├── requests [required: Any, installed: 2.25.1]
 │           └── typing-extensions [required: Any, installed: 4.8.0]
 ├── healpy [required: Any, installed: 1.15.0]
@@ -263,17 +263,17 @@ skymap-scanner
 │   ├── meander [required: Any, installed: 0.0.3]
 │   ├── numpy [required: Any, installed: 1.21.5]
 │   ├── pandas [required: Any, installed: 1.3.5]
-│   └── wipac-dev-tools [required: Any, installed: 1.7.1]
+│   └── wipac-dev-tools [required: Any, installed: 1.8.1]
 │       ├── requests [required: Any, installed: 2.25.1]
 │       └── typing-extensions [required: Any, installed: 4.8.0]
 ├── iminuit [required: Any, installed: 2.24.0]
 │   └── numpy [required: >=1.21, installed: 1.21.5]
 ├── numpy [required: Any, installed: 1.21.5]
 ├── oms-mqclient [required: Any, installed: 2.4.9]
-│   └── wipac-dev-tools [required: Any, installed: 1.7.1]
+│   └── wipac-dev-tools [required: Any, installed: 1.8.1]
 │       ├── requests [required: Any, installed: 2.25.1]
 │       └── typing-extensions [required: Any, installed: 4.8.0]
-├── wipac-dev-tools [required: Any, installed: 1.7.1]
+├── wipac-dev-tools [required: Any, installed: 1.8.1]
 │   ├── requests [required: Any, installed: 2.25.1]
 │   └── typing-extensions [required: Any, installed: 4.8.0]
 └── wipac-rest-tools [required: Any, installed: 1.6.0]
@@ -287,7 +287,7 @@ skymap-scanner
     │   └── requests [required: >=1.2.0, installed: 2.25.1]
     ├── tornado [required: Any, installed: 6.3.3]
     ├── urllib3 [required: >=2.0.4, installed: 2.1.0]
-    └── wipac-dev-tools [required: Any, installed: 1.7.1]
+    └── wipac-dev-tools [required: Any, installed: 1.8.1]
         ├── requests [required: Any, installed: 2.25.1]
         └── typing-extensions [required: Any, installed: 4.8.0]
 sortedcontainers==2.1.0

dependencies-from-Dockerfile_pulsar.log

@@ -39,7 +39,7 @@ fonttools==4.29.1
 fs==2.4.12
 gast==0.5.2
 healpy==1.15.0
-htchirp==2.0
+htchirp==3.0
 htcondor==23.1.0
 html5lib==1.1
 humanfriendly==10.0
@@ -139,7 +139,7 @@ urllib3==2.1.0
 urwid==2.1.2
 wcwidth==0.2.5
 webencodings==0.5.1
-wipac-dev-tools==1.7.1
+wipac-dev-tools==1.8.1
 wipac-rest-tools==1.6.0
 xlwt==1.3.0
 zipp==1.0.0
@@ -251,10 +251,10 @@ setuptools==59.6.0
 six==1.16.0
 skymap-scanner
 ├── ewms-pilot [required: Any, installed: 0.17.5]
-│   ├── htchirp [required: Any, installed: 2.0]
+│   ├── htchirp [required: Any, installed: 3.0]
 │   ├── htcondor [required: Any, installed: 23.1.0]
 │   └── oms-mqclient [required: Any, installed: 2.4.9]
-│       └── wipac-dev-tools [required: Any, installed: 1.7.1]
+│       └── wipac-dev-tools [required: Any, installed: 1.8.1]
 │           ├── requests [required: Any, installed: 2.25.1]
 │           └── typing-extensions [required: Any, installed: 4.8.0]
 ├── healpy [required: Any, installed: 1.15.0]
@@ -265,17 +265,17 @@ skymap-scanner
 │   ├── meander [required: Any, installed: 0.0.3]
 │   ├── numpy [required: Any, installed: 1.21.5]
 │   ├── pandas [required: Any, installed: 1.3.5]
-│   └── wipac-dev-tools [required: Any, installed: 1.7.1]
+│   └── wipac-dev-tools [required: Any, installed: 1.8.1]
 │       ├── requests [required: Any, installed: 2.25.1]
 │       └── typing-extensions [required: Any, installed: 4.8.0]
 ├── iminuit [required: Any, installed: 2.24.0]
 │   └── numpy [required: >=1.21, installed: 1.21.5]
 ├── numpy [required: Any, installed: 1.21.5]
 ├── oms-mqclient [required: Any, installed: 2.4.9]
-│   └── wipac-dev-tools [required: Any, installed: 1.7.1]
+│   └── wipac-dev-tools [required: Any, installed: 1.8.1]
 │       ├── requests [required: Any, installed: 2.25.1]
 │       └── typing-extensions [required: Any, installed: 4.8.0]
-├── wipac-dev-tools [required: Any, installed: 1.7.1]
+├── wipac-dev-tools [required: Any, installed: 1.8.1]
 │   ├── requests [required: Any, installed: 2.25.1]
 │   └── typing-extensions [required: Any, installed: 4.8.0]
 └── wipac-rest-tools [required: Any, installed: 1.6.0]
@@ -289,7 +289,7 @@ skymap-scanner
     │   └── requests [required: >=1.2.0, installed: 2.25.1]
     ├── tornado [required: Any, installed: 6.3.3]
     ├── urllib3 [required: >=2.0.4, installed: 2.1.0]
-    └── wipac-dev-tools [required: Any, installed: 1.7.1]
+    └── wipac-dev-tools [required: Any, installed: 1.8.1]
         ├── requests [required: Any, installed: 2.25.1]
         └── typing-extensions [required: Any, installed: 4.8.0]
 sortedcontainers==2.1.0

skymap_scanner/config.py

@@ -94,6 +94,8 @@ class EnvConfig:
     SKYSCAN_PROGRESS_INTERVAL_SEC: int = 1 * 60
     SKYSCAN_RESULT_INTERVAL_SEC: int = 2 * 60
 
+    SKYSCAN_KILL_SWITCH_CHECK_INTERVAL: int = 5 * 60
+
     # BROKER/MQ VARS
     SKYSCAN_BROKER_CLIENT: str = "rabbitmq"
     SKYSCAN_BROKER_ADDRESS: str = ""  # broker / mq address

skymap_scanner/server/start_scan.py

@@ -8,21 +8,20 @@
 import json
 import logging
 import random
+import threading
 import time
 from pathlib import Path
 from typing import Any, Dict, Iterator, List, Optional, Set, Tuple
 
 import healpy  # type: ignore[import]
 import mqclient as mq
 import numpy
-from icecube.icetray import I3Units  # type: ignore[import]
 from icecube import (  # type: ignore[import]
     astro,
     dataclasses,
     full_event_followup,
     icetray,
 )
-from rest_tools.client import RestClient
 from skyreader import EventMetadata
 from wipac_dev_tools import argparse_tools, logging_tools
 
@@ -45,6 +44,7 @@
     NSideProgression,
     fetch_event_contents_from_file,
     fetch_event_contents_from_skydriver,
+    kill_switch_check_from_skydriver,
 )
 
 StrDict = Dict[str, Any]
@@ -89,9 +89,9 @@ def __init__(
         self.input_pos_name = input_pos_name
         self.input_time_name = input_time_name
         self.output_particle_name = output_particle_name
-        
+
         RecoAlgo: type[RecoInterface] = recos.get_reco_interface_object(reco_algo)
-        
+
         self.reco: RecoInterface = RecoAlgo()
 
         self.pos_variations = self.reco.get_vertex_variations()
@@ -120,7 +120,7 @@ def __init__(
 
         # The HLC pulse mask should have been been created in prepare_frames().
         self.pulseseries_hlc = dataclasses.I3RecoPulseSeriesMap.from_frame(p_frame,cfg.INPUT_PULSES_NAME+'HLC')
-        
+
         self.omgeo = g_frame["I3Geometry"].omgeo
 
 
@@ -183,7 +183,7 @@ def i3particle(self, position, direction, energy, time):
         particle.fit_status = dataclasses.I3Particle.FitStatus.OK
         particle.pos = position
         particle.dir = direction
-        
+
         if self.reco.refine_time:
             LOGGER.debug(f"Reco_algo is {self.reco.name}, refining time")
             # given direction and vertex position, calculate time from CAD
@@ -234,7 +234,7 @@ def _gen_pframes(
         else:
             coarser_nside = nside
             while True:
-                # Look up the first available coarser NSIDE by iteratively dividing by two the current nside. 
+                # Look up the first available coarser NSIDE by iteratively dividing by two the current nside.
                 # NOTE (v3): this guesswork could be avoided using the NSIDE progression.
                 coarser_nside = coarser_nside/2
                 coarser_pixel = healpy.ang2pix(int(coarser_nside), numpy.pi/2-dec, ra)
@@ -267,7 +267,7 @@ def _gen_pframes(
                     time = self.nsides_dict[coarser_nside][coarser_pixel].time
                     energy = self.nsides_dict[coarser_nside][coarser_pixel].energy
 
-        # Now generate the vertex seed position variations according to the reco-specific logic. 
+        # Now generate the vertex seed position variations according to the reco-specific logic.
 
         LOGGER.debug(f"Generating {len(self.pos_variations)} position variations.")
 
@@ -730,6 +730,13 @@ def _nside_and_pixelextension(val: str) -> Tuple[int, int]:
         timeout=cfg.ENV.SKYSCAN_MQ_TIMEOUT_FROM_CLIENTS,
     )
 
+    # create background thread for checking whether to abort -- fire & forget
+    threading.Thread(
+        target=asyncio.run,
+        args=(kill_switch_check_from_skydriver(),),
+        daemon=True,
+    ).start()
+
     # go!
     asyncio.run(
         scan(

skymap_scanner/server/utils.py

@@ -1,8 +1,11 @@
 """Server-specific utils."""
 
 
+import asyncio
 import json
+import logging
 import pickle
+import sys
 from collections import OrderedDict
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
@@ -43,6 +46,29 @@ async def nonurgent_request(rc: RestClient, args: dict[str, Any]) -> Any:
         return None
 
 
+async def kill_switch_check_from_skydriver() -> None:
+    """Routinely check SkyDriver whether to continue the scan."""
+    if not cfg.ENV.SKYSCAN_SKYDRIVER_ADDRESS:
+        return
+
+    logger = logging.getLogger("skyscan.kill_switch")
+
+    skydriver_rc = connect_to_skydriver(urgent=False)
+
+    while True:
+        await asyncio.sleep(cfg.ENV.SKYSCAN_KILL_SWITCH_CHECK_INTERVAL)
+
+        status = await skydriver_rc.request(
+            "GET", f"/scan/{cfg.ENV.SKYSCAN_SKYDRIVER_SCAN_ID}/status"
+        )
+
+        if status["scan_state"].startswith("STOPPED__"):
+            logger.critical(
+                f"Kill switch triggered by SkyDriver scan state: {status['scan_state']}"
+            )
+            sys.exit(1)
+
+
 async def fetch_event_contents_from_skydriver() -> Any:
     """Fetch event contents from SkyDriver."""
     skydriver_rc = connect_to_skydriver(urgent=True)