Add Cluster-watching to Starter (#88)

Co-authored-by: github-actions <[email protected]>
WIPACrepo · Oct 30, 2023 · 46f6502 · 46f6502
1 parent 38e9464
commit 46f6502
Show file tree

Hide file tree

Showing 3 changed files with 68 additions and 10 deletions.
diff --git a/clientmanager/condor/act.py b/clientmanager/condor/act.py
@@ -7,7 +7,7 @@
 
 from .. import utils
 from ..config import ENV, LOGGER
-from . import condor_tools, starter, stopper
+from . import condor_tools, starter, stopper, watcher
 
 
 def act(args: argparse.Namespace) -> None:
@@ -73,6 +73,12 @@ def _act(args: argparse.Namespace, schedd_obj: htcondor.Schedd) -> None:
                 starter_info=submit_dict,
             )
             LOGGER.info("Sent cluster info to SkyDriver")
+            watcher.watch(
+                args.collector,
+                args.schedd,
+                args.cluster_id,
+                schedd_obj,
+            )
         case "stop":
             stopper.stop(
                 args.collector,

diff --git a/clientmanager/condor/watcher.py b/clientmanager/condor/watcher.py
@@ -0,0 +1,52 @@
+"""For watching Skymap Scanner clients on an HTCondor cluster."""
+
+
+import time
+
+import htcondor  # type: ignore[import]
+
+from ..config import LOGGER
+
+
+def watch(
+    collector: str,
+    schedd: str,
+    cluster_id: str,
+    schedd_obj: htcondor.Schedd,
+) -> None:
+    """Main logic."""
+    LOGGER.info(
+        f"Watching Skymap Scanner client workers on {cluster_id} / {collector} / {schedd}"
+    )
+    start = time.time()
+
+    while time.time() - start < 60 * 60:  # only go for 1 hour -- TODO smarten
+        # class ad
+        ads = schedd_obj.query(
+            f"ClusterId == {cluster_id}",
+            # ["list", "of", "desired", "attributes"],
+        )
+        for i, ad in enumerate(ads):
+            LOGGER.debug(f"class ad #{i}")
+            LOGGER.debug(ad)
+
+        # histories
+        histories = schedd_obj.history(
+            f"ClusterId == {cluster_id}",
+            # ["list", "of", "desired", "attributes"],
+        )
+        for i, history in enumerate(histories):
+            LOGGER.debug(f"history #{i}")
+            LOGGER.debug(history)
+
+        # jobEpochHistory
+        histories = schedd_obj.jobEpochHistory(
+            f"ClusterId == {cluster_id}",
+            # ["list", "of", "desired", "attributes"],
+        )
+        for i, history in enumerate(histories):
+            LOGGER.debug(f"jobEpochHistory #{i}")
+            LOGGER.debug(history)
+
+        time.sleep(60)
+        LOGGER.info("requesting again...")
diff --git a/dependencies-from-Dockerfile.log b/dependencies-from-Dockerfile.log
@@ -7,8 +7,8 @@
 #  pip freeze
 ########################################################################
 backoff==2.2.1
-boto3==1.28.70
-botocore==1.31.70
+boto3==1.28.74
+botocore==1.31.74
 cachetools==5.3.2
 certifi==2023.7.22
 cffi==1.16.0
@@ -60,7 +60,7 @@ typeguard==4.1.5
 typing_extensions==4.8.0
 urllib3==1.26.18
 websocket-client==1.6.4
-wipac-dev-tools==1.7.0
+wipac-dev-tools==1.7.1
 wipac-rest-tools==1.5.2
 wipac-telemetry==0.3.0
 wrapt==1.15.0
@@ -75,15 +75,15 @@ pip==23.2.1
 pipdeptree==2.13.0
 setuptools==65.5.1
 skydriver-clientmanager
-├── boto3 [required: Any, installed: 1.28.70]
-│   ├── botocore [required: >=1.31.70,<1.32.0, installed: 1.31.70]
+├── boto3 [required: Any, installed: 1.28.74]
+│   ├── botocore [required: >=1.31.74,<1.32.0, installed: 1.31.74]
 │   │   ├── jmespath [required: >=0.7.1,<2.0.0, installed: 1.0.1]
 │   │   ├── python-dateutil [required: >=2.1,<3.0.0, installed: 2.8.2]
 │   │   │   └── six [required: >=1.5, installed: 1.16.0]
 │   │   └── urllib3 [required: >=1.25.4,<2.1, installed: 1.26.18]
 │   ├── jmespath [required: >=0.7.1,<2.0.0, installed: 1.0.1]
 │   └── s3transfer [required: >=0.7.0,<0.8.0, installed: 0.7.0]
-│       └── botocore [required: >=1.12.36,<2.0a.0, installed: 1.31.70]
+│       └── botocore [required: >=1.12.36,<2.0a.0, installed: 1.31.74]
 │           ├── jmespath [required: >=0.7.1,<2.0.0, installed: 1.0.1]
 │           ├── python-dateutil [required: >=2.1,<3.0.0, installed: 2.8.2]
 │           │   └── six [required: >=1.5, installed: 1.16.0]
@@ -132,7 +132,7 @@ skydriver-clientmanager
 ├── tornado [required: Any, installed: 6.3.3]
 ├── typeguard [required: Any, installed: 4.1.5]
 │   └── typing-extensions [required: >=4.7.0, installed: 4.8.0]
-├── wipac-dev-tools [required: Any, installed: 1.7.0]
+├── wipac-dev-tools [required: Any, installed: 1.7.1]
 │   ├── requests [required: Any, installed: 2.31.0]
 │   │   ├── certifi [required: >=2017.4.17, installed: 2023.7.22]
 │   │   ├── charset-normalizer [required: >=2,<4, installed: 3.3.1]
@@ -157,7 +157,7 @@ skydriver-clientmanager
     │       ├── idna [required: >=2.5,<4, installed: 3.4]
     │       └── urllib3 [required: >=1.21.1,<3, installed: 1.26.18]
     ├── tornado [required: Any, installed: 6.3.3]
-    └── wipac-dev-tools [required: Any, installed: 1.7.0]
+    └── wipac-dev-tools [required: Any, installed: 1.7.1]
         ├── requests [required: Any, installed: 2.31.0]
         │   ├── certifi [required: >=2017.4.17, installed: 2023.7.22]
         │   ├── charset-normalizer [required: >=2,<4, installed: 3.3.1]
@@ -247,7 +247,7 @@ wipac-telemetry==0.3.0
 │   └── typing-extensions [required: >=3.7.4, installed: 4.8.0]
 ├── protobuf [required: Any, installed: 3.20.3]
 ├── typing-extensions [required: Any, installed: 4.8.0]
-└── wipac-dev-tools [required: Any, installed: 1.7.0]
+└── wipac-dev-tools [required: Any, installed: 1.7.1]
     ├── requests [required: Any, installed: 2.31.0]
     │   ├── certifi [required: >=2017.4.17, installed: 2023.7.22]
     │   ├── charset-normalizer [required: >=2,<4, installed: 3.3.1]