New pulse series name

.github/workflows/tests.yml

@@ -378,7 +378,8 @@ jobs:
         eventfile: [
           hese_event_01.json,
           run00136766-evt000007637140-GOLD.pkl,
-          run00136662-evt000035405932-BRONZE.pkl
+          run00136662-evt000035405932-BRONZE.pkl,
+          138632_31747601.json
         ]
         exclude:
         # splinempe should not run on HESE

dependencies-from-Dockerfile.log

@@ -130,7 +130,7 @@ tables==3.7.0
 toml==0.10.2
 tornado==6.4
 traitlets==5.1.1
-typing_extensions==4.8.0
+typing_extensions==4.9.0
 ufoLib2==0.13.1
 unicodedata2==14.0.0
 UNKNOWN @ file:///root/nuflux-2.0.4
@@ -254,7 +254,7 @@ skymap-scanner
 │   └── oms-mqclient [required: Any, installed: 2.4.10]
 │       └── wipac-dev-tools [required: Any, installed: 1.8.2]
 │           ├── requests [required: Any, installed: 2.25.1]
-│           └── typing-extensions [required: Any, installed: 4.8.0]
+│           └── typing-extensions [required: Any, installed: 4.9.0]
 ├── healpy [required: Any, installed: 1.15.0]
 ├── icecube-skyreader [required: Any, installed: 1.2.8]
 │   ├── astropy [required: Any, installed: 5.0.2]
@@ -265,31 +265,31 @@ skymap-scanner
 │   ├── pandas [required: Any, installed: 1.3.5]
 │   └── wipac-dev-tools [required: Any, installed: 1.8.2]
 │       ├── requests [required: Any, installed: 2.25.1]
-│       └── typing-extensions [required: Any, installed: 4.8.0]
+│       └── typing-extensions [required: Any, installed: 4.9.0]
 ├── iminuit [required: Any, installed: 2.24.0]
 │   └── numpy [required: >=1.21, installed: 1.21.5]
 ├── numpy [required: Any, installed: 1.21.5]
 ├── oms-mqclient [required: Any, installed: 2.4.10]
 │   └── wipac-dev-tools [required: Any, installed: 1.8.2]
 │       ├── requests [required: Any, installed: 2.25.1]
-│       └── typing-extensions [required: Any, installed: 4.8.0]
+│       └── typing-extensions [required: Any, installed: 4.9.0]
 ├── wipac-dev-tools [required: Any, installed: 1.8.2]
 │   ├── requests [required: Any, installed: 2.25.1]
-│   └── typing-extensions [required: Any, installed: 4.8.0]
+│   └── typing-extensions [required: Any, installed: 4.9.0]
 └── wipac-rest-tools [required: Any, installed: 1.6.0]
     ├── cachetools [required: Any, installed: 5.3.2]
     ├── PyJWT [required: !=2.6.0, installed: 2.8.0]
     ├── qrcode [required: Any, installed: 7.4.2]
     │   ├── pypng [required: Any, installed: 0.20220715.0]
-    │   └── typing-extensions [required: Any, installed: 4.8.0]
+    │   └── typing-extensions [required: Any, installed: 4.9.0]
     ├── requests [required: Any, installed: 2.25.1]
     ├── requests-futures [required: Any, installed: 1.0.1]
     │   └── requests [required: >=1.2.0, installed: 2.25.1]
     ├── tornado [required: Any, installed: 6.4]
     ├── urllib3 [required: >=2.0.4, installed: 2.1.0]
     └── wipac-dev-tools [required: Any, installed: 1.8.2]
         ├── requests [required: Any, installed: 2.25.1]
-        └── typing-extensions [required: Any, installed: 4.8.0]
+        └── typing-extensions [required: Any, installed: 4.9.0]
 sortedcontainers==2.1.0
 soupsieve==2.3.1
 sympy==1.9

dependencies-from-Dockerfile_no_cvmfs.log

@@ -130,7 +130,7 @@ tables==3.7.0
 toml==0.10.2
 tornado==6.4
 traitlets==5.1.1
-typing_extensions==4.8.0
+typing_extensions==4.9.0
 ufoLib2==0.13.1
 unicodedata2==14.0.0
 UNKNOWN @ file:///root/nuflux-2.0.4
@@ -254,7 +254,7 @@ skymap-scanner
 │   └── oms-mqclient [required: Any, installed: 2.4.10]
 │       └── wipac-dev-tools [required: Any, installed: 1.8.2]
 │           ├── requests [required: Any, installed: 2.25.1]
-│           └── typing-extensions [required: Any, installed: 4.8.0]
+│           └── typing-extensions [required: Any, installed: 4.9.0]
 ├── healpy [required: Any, installed: 1.15.0]
 ├── icecube-skyreader [required: Any, installed: 1.2.8]
 │   ├── astropy [required: Any, installed: 5.0.2]
@@ -265,31 +265,31 @@ skymap-scanner
 │   ├── pandas [required: Any, installed: 1.3.5]
 │   └── wipac-dev-tools [required: Any, installed: 1.8.2]
 │       ├── requests [required: Any, installed: 2.25.1]
-│       └── typing-extensions [required: Any, installed: 4.8.0]
+│       └── typing-extensions [required: Any, installed: 4.9.0]
 ├── iminuit [required: Any, installed: 2.24.0]
 │   └── numpy [required: >=1.21, installed: 1.21.5]
 ├── numpy [required: Any, installed: 1.21.5]
 ├── oms-mqclient [required: Any, installed: 2.4.10]
 │   └── wipac-dev-tools [required: Any, installed: 1.8.2]
 │       ├── requests [required: Any, installed: 2.25.1]
-│       └── typing-extensions [required: Any, installed: 4.8.0]
+│       └── typing-extensions [required: Any, installed: 4.9.0]
 ├── wipac-dev-tools [required: Any, installed: 1.8.2]
 │   ├── requests [required: Any, installed: 2.25.1]
-│   └── typing-extensions [required: Any, installed: 4.8.0]
+│   └── typing-extensions [required: Any, installed: 4.9.0]
 └── wipac-rest-tools [required: Any, installed: 1.6.0]
     ├── cachetools [required: Any, installed: 5.3.2]
     ├── PyJWT [required: !=2.6.0, installed: 2.8.0]
     ├── qrcode [required: Any, installed: 7.4.2]
     │   ├── pypng [required: Any, installed: 0.20220715.0]
-    │   └── typing-extensions [required: Any, installed: 4.8.0]
+    │   └── typing-extensions [required: Any, installed: 4.9.0]
     ├── requests [required: Any, installed: 2.25.1]
     ├── requests-futures [required: Any, installed: 1.0.1]
     │   └── requests [required: >=1.2.0, installed: 2.25.1]
     ├── tornado [required: Any, installed: 6.4]
     ├── urllib3 [required: >=2.0.4, installed: 2.1.0]
     └── wipac-dev-tools [required: Any, installed: 1.8.2]
         ├── requests [required: Any, installed: 2.25.1]
-        └── typing-extensions [required: Any, installed: 4.8.0]
+        └── typing-extensions [required: Any, installed: 4.9.0]
 sortedcontainers==2.1.0
 soupsieve==2.3.1
 sympy==1.9

dependencies-from-Dockerfile_pulsar.log

@@ -131,7 +131,7 @@ tables==3.7.0
 toml==0.10.2
 tornado==6.4
 traitlets==5.1.1
-typing_extensions==4.8.0
+typing_extensions==4.9.0
 ufoLib2==0.13.1
 unicodedata2==14.0.0
 UNKNOWN @ file:///root/nuflux-2.0.4
@@ -256,7 +256,7 @@ skymap-scanner
 │   └── oms-mqclient [required: Any, installed: 2.4.10]
 │       └── wipac-dev-tools [required: Any, installed: 1.8.2]
 │           ├── requests [required: Any, installed: 2.25.1]
-│           └── typing-extensions [required: Any, installed: 4.8.0]
+│           └── typing-extensions [required: Any, installed: 4.9.0]
 ├── healpy [required: Any, installed: 1.15.0]
 ├── icecube-skyreader [required: Any, installed: 1.2.8]
 │   ├── astropy [required: Any, installed: 5.0.2]
@@ -267,31 +267,31 @@ skymap-scanner
 │   ├── pandas [required: Any, installed: 1.3.5]
 │   └── wipac-dev-tools [required: Any, installed: 1.8.2]
 │       ├── requests [required: Any, installed: 2.25.1]
-│       └── typing-extensions [required: Any, installed: 4.8.0]
+│       └── typing-extensions [required: Any, installed: 4.9.0]
 ├── iminuit [required: Any, installed: 2.24.0]
 │   └── numpy [required: >=1.21, installed: 1.21.5]
 ├── numpy [required: Any, installed: 1.21.5]
 ├── oms-mqclient [required: Any, installed: 2.4.10]
 │   └── wipac-dev-tools [required: Any, installed: 1.8.2]
 │       ├── requests [required: Any, installed: 2.25.1]
-│       └── typing-extensions [required: Any, installed: 4.8.0]
+│       └── typing-extensions [required: Any, installed: 4.9.0]
 ├── wipac-dev-tools [required: Any, installed: 1.8.2]
 │   ├── requests [required: Any, installed: 2.25.1]
-│   └── typing-extensions [required: Any, installed: 4.8.0]
+│   └── typing-extensions [required: Any, installed: 4.9.0]
 └── wipac-rest-tools [required: Any, installed: 1.6.0]
     ├── cachetools [required: Any, installed: 5.3.2]
     ├── PyJWT [required: !=2.6.0, installed: 2.8.0]
     ├── qrcode [required: Any, installed: 7.4.2]
     │   ├── pypng [required: Any, installed: 0.20220715.0]
-    │   └── typing-extensions [required: Any, installed: 4.8.0]
+    │   └── typing-extensions [required: Any, installed: 4.9.0]
     ├── requests [required: Any, installed: 2.25.1]
     ├── requests-futures [required: Any, installed: 1.0.1]
     │   └── requests [required: >=1.2.0, installed: 2.25.1]
     ├── tornado [required: Any, installed: 6.4]
     ├── urllib3 [required: >=2.0.4, installed: 2.1.0]
     └── wipac-dev-tools [required: Any, installed: 1.8.2]
         ├── requests [required: Any, installed: 2.25.1]
-        └── typing-extensions [required: Any, installed: 4.8.0]
+        └── typing-extensions [required: Any, installed: 4.9.0]
 sortedcontainers==2.1.0
 soupsieve==2.3.1
 sympy==1.9

skymap_scanner/client/client.py

@@ -5,6 +5,7 @@
 import json
 import logging
 from pathlib import Path
+from ..utils.data_handling import get_gcd_datastager
 
 import ewms_pilot
 from wipac_dev_tools import argparse_tools, logging_tools
@@ -58,16 +59,24 @@ def main() -> None:
     with open("GCDQp_packet.json", "w") as f:
         json.dump(startup_json_dict["GCDQp_packet"], f)
 
+    datastager = get_gcd_datastager()
+
+    baseline_gcd_file = Path(startup_json_dict["baseline_GCD_file"])
+
+    datastager.stage_files([baseline_gcd_file.name])
+
+    baseline_gcd_file = Path(datastager.get_filepath(baseline_gcd_file.name))
+
     # check if baseline GCD file is reachable
-    if not Path(startup_json_dict["baseline_GCD_file"]).exists():
-        raise FileNotFoundError(startup_json_dict["baseline_GCD_file"])
+    if not baseline_gcd_file.exists():
+        raise FileNotFoundError(baseline_gcd_file)
 
     cmd = (
         "python -m skymap_scanner.client.reco_icetray "
         " --in-pkl {{INFILE}}"  # no f-string b/c want to preserve '{{..}}'
         " --out-pkl {{OUTFILE}}"  # ^^^
         " --gcdqp-packet-json GCDQp_packet.json"
-        f" --baseline-gcd-file {startup_json_dict['baseline_GCD_file']}"
+        f" --baseline-gcd-file {baseline_gcd_file}"
     )
 
     # go!

skymap_scanner/config.py

@@ -14,24 +14,43 @@
 # True constants
 #
 
-DEFAULT_GCD_DIR: Path = Path("/opt/i3-data/baseline_gcds")
 
 # Local data sources. These are assumed to be filesystem paths and are expected to have the same directory structure.
 LOCAL_DATA_SOURCES: Final[List[Path]] = [
     Path("/opt/i3-data"),
     Path("/cvmfs/icecube.opensciencegrid.org/data"),
 ]
-# Directory path under a local data source to fetch spline data from.
-LOCAL_SPLINE_SUBDIR: Final[str] = "photon-tables/splines"
 
 # HTTP source to download data from.
 REMOTE_DATA_SOURCE: Final[str] = "http://prod-exe.icecube.wisc.edu"
-REMOTE_SPLINE_SUBDIR: Final[str] = "spline-tables"
 
+# Local ephemeral directory to stage files.
 LOCAL_DATA_CACHE: Final[Path] = Path("./data-staging-cache")
 
+# Directory path under a local data source to fetch spline data from.
+LOCAL_SPLINE_SUBDIR: Final[str] = "photon-tables/splines"
+REMOTE_SPLINE_SUBDIR: Final[str] = "spline-tables"
+
+# GCD data sources.
+LOCAL_GCD_DATA_SOURCES: Final[List[Path]] = [
+    Path("/opt/i3-data/baseline_gcds"),
+    Path("/cvmfs/icecube.opensciencegrid.org/users/RealTime/GCD/PoleBaseGCDs"),
+]
+
+DEFAULT_GCD_DIR = LOCAL_GCD_DATA_SOURCES[0]
+
+# Since the container and CVFMS have GCD files in different subdirectories
+#   we put the complete path in LOCAL_GCD_DATA_SOURCES and use no subdir.
+LOCAL_GCD_SUBDIR = ""
+
+# This is temporary until it can be replaced by prod-exe.
+REMOTE_GCD_DATA_SOURCE: Final[
+    str
+] = "https://www-zeuthen.desy.de/~mlincett/baseline_gcds"
+
+
 # physics strings
-INPUT_PULSES_NAME: Final = "SplitUncleanedInIcePulses"
+INPUT_PULSES_NAME: Final = "SplitInIcePulses"
 INPUT_TIME_NAME: Final = "SeedVertexTime"
 INPUT_POS_NAME: Final = "SeedVertexPos"
 OUTPUT_PARTICLE_NAME: Final = "MillipedeSeedParticle"

skymap_scanner/server/start_scan.py

@@ -688,6 +688,7 @@ def _nside_and_pixelextension(val: str) -> Tuple[int, int]:
         event_contents = fetch_event_contents_from_file(args.event_file)
 
     # get inputs (load event_id + state_dict cache)
+    LOGGER.info("Extracting event...")
     event_metadata, state_dict = extract_json_message.extract_json_message(
         event_contents,
         reco_algo=args.reco_algo,
@@ -698,6 +699,7 @@ def _nside_and_pixelextension(val: str) -> Tuple[int, int]:
     )
 
     # write startup files for client-spawning
+    LOGGER.info("Writing startup JSON...")
     scan_id = write_startup_json(
         args.client_startup_json,
         event_metadata,

skymap_scanner/utils/data_handling.py

@@ -37,12 +37,12 @@ def stage_files(self, file_list: List[str]):
                     LOGGER.debug("File is available on staging path.")
                 else:
                     LOGGER.debug("Staging from HTTP source.")
-                    self.stage_file(basename)
+                    self.fetch_file(basename)
 
             else:
                 LOGGER.debug(f"File {basename} is available at {filepath}.")
 
-    def stage_file(self, basename: str):
+    def fetch_file(self, basename: str):
         """Retrieves a file from the HTTP source.
 
         Args:
@@ -115,3 +115,11 @@ def get_local_filepath(self, filename: str) -> str:
                 LOGGER.debug(f"-> fail.")
                 # File was not found in local paths.
         raise FileNotFoundError(f"File {filename} is not available on any local path.")
+
+
+def get_gcd_datastager() -> DataStager:
+    return DataStager(
+        local_paths=cfg.LOCAL_GCD_DATA_SOURCES,
+        local_subdir=cfg.LOCAL_GCD_SUBDIR,
+        remote_path=cfg.REMOTE_GCD_DATA_SOURCE,
+    )

skymap_scanner/utils/extract_json_message.py

@@ -21,6 +21,7 @@
     rewrite_frame_stop,
     save_GCD_frame_packet_to_file,
 )
+from .data_handling import get_gcd_datastager
 
 
 def extract_GCD_diff_base_filename(frame_packet):
@@ -62,8 +63,10 @@ def extract_json_message(
         raise RuntimeError("cache directory \"{0}\" is not a directory.".format(cache_dir))
 
     # extract the packet
+    LOGGER.info("Extracting JSON to frame packet")
     frame_packet = full_event_followup.i3live_json_to_frame_packet(json.dumps(json_data), pnf_framing=True)
 
+    LOGGER.info("Assembling event frames")
     _, event_metadata, state_dict = __extract_frame_packet(
         frame_packet,
         reco_algo=reco_algo,
@@ -73,6 +76,7 @@ def extract_json_message(
         pulsesName=pulsesName
     )
 
+    LOGGER.info("Load scan state...")
     # try to load existing pixels if there are any
     state_dict = load_scan_state(event_metadata, state_dict, reco_algo, cache_dir=cache_dir)
     return event_metadata, state_dict
@@ -154,20 +158,27 @@ def __extract_frame_packet(
     # - look up baseline GCD in GCD_dir based on run number
     # - assemble GCDQp from baseline GCD
 
+    LOGGER.info("Retrieving GCD...")
     LOGGER.debug(f"Extracted GCD_diff_base_filename = {baseline_GCD}.")
     LOGGER.debug(f"GCD dir is set to = {GCD_dir}.")
 
     #=====================
     # GCD-diff framepacket
     #=====================
     if baseline_GCD is None:
-        LOGGER.debug("Packet does not need a GCD diff.")
+        LOGGER.info("Packet does not need a GCD diff.")
     else:
+        LOGGER.info("Running GCD uncompress logic...")
+        datastager = get_gcd_datastager()
         # assume GCD dir is always valid
         baseline_GCD_file = os.path.join(GCD_dir, baseline_GCD)
+
         LOGGER.debug(f"Trying GCD file: {baseline_GCD_file}")
+        datastager.stage_files([baseline_GCD])
+        baseline_GCD_file = datastager.get_filepath(baseline_GCD)
+
         if not os.path.isfile(baseline_GCD_file):
-            raise RuntimeError("Baseline GCD file not available!")
+            raise RuntimeError(f"Baseline GCD file {baseline_GCD_file} not available!")
         # NOTE: logic allowing GCD_dir to point to a file, in order to directly override the GCD has been removed.
 
         cached_baseline_GCD_file = os.path.join(event_cache_dir, cfg.BASELINE_GCD_FILENAME)
@@ -203,7 +214,7 @@ def __extract_frame_packet(
     # GCD-less framepacket
     #=====================
     if ("I3Geometry" not in frame_packet[0]) and ("I3GeometryDiff" not in frame_packet[0]):
-        LOGGER.debug("Packet with empty GCD frames. Need to load baseline GCD")
+        LOGGER.info("Packet with empty GCD frames. Need to load baseline GCD")
         # If no GCD is specified, work out correct one from run number
         available_GCDs = sorted([x for x in os.listdir(GCD_dir) if ".i3" in x])
         run = float(header.run_id)