From 46f6502f143bd39af9698934b5085486811f88c3 Mon Sep 17 00:00:00 2001 From: Ric Evans <19216225+ric-evans@users.noreply.github.com> Date: Mon, 30 Oct 2023 15:47:33 -0500 Subject: [PATCH] Add Cluster-watching to Starter (#88) Co-authored-by: github-actions --- clientmanager/condor/act.py | 8 ++++- clientmanager/condor/watcher.py | 52 ++++++++++++++++++++++++++++++++ dependencies-from-Dockerfile.log | 18 +++++------ 3 files changed, 68 insertions(+), 10 deletions(-) create mode 100644 clientmanager/condor/watcher.py diff --git a/clientmanager/condor/act.py b/clientmanager/condor/act.py index 7999915c..63cbf3e9 100644 --- a/clientmanager/condor/act.py +++ b/clientmanager/condor/act.py @@ -7,7 +7,7 @@ from .. import utils from ..config import ENV, LOGGER -from . import condor_tools, starter, stopper +from . import condor_tools, starter, stopper, watcher def act(args: argparse.Namespace) -> None: @@ -73,6 +73,12 @@ def _act(args: argparse.Namespace, schedd_obj: htcondor.Schedd) -> None: starter_info=submit_dict, ) LOGGER.info("Sent cluster info to SkyDriver") + watcher.watch( + args.collector, + args.schedd, + args.cluster_id, + schedd_obj, + ) case "stop": stopper.stop( args.collector, diff --git a/clientmanager/condor/watcher.py b/clientmanager/condor/watcher.py new file mode 100644 index 00000000..7ab90619 --- /dev/null +++ b/clientmanager/condor/watcher.py @@ -0,0 +1,52 @@ +"""For watching Skymap Scanner clients on an HTCondor cluster.""" + + +import time + +import htcondor # type: ignore[import] + +from ..config import LOGGER + + +def watch( + collector: str, + schedd: str, + cluster_id: str, + schedd_obj: htcondor.Schedd, +) -> None: + """Main logic.""" + LOGGER.info( + f"Watching Skymap Scanner client workers on {cluster_id} / {collector} / {schedd}" + ) + start = time.time() + + while time.time() - start < 60 * 60: # only go for 1 hour -- TODO smarten + # class ad + ads = schedd_obj.query( + f"ClusterId == {cluster_id}", + # ["list", "of", "desired", "attributes"], + ) + for i, ad in enumerate(ads): + LOGGER.debug(f"class ad #{i}") + LOGGER.debug(ad) + + # histories + histories = schedd_obj.history( + f"ClusterId == {cluster_id}", + # ["list", "of", "desired", "attributes"], + ) + for i, history in enumerate(histories): + LOGGER.debug(f"history #{i}") + LOGGER.debug(history) + + # jobEpochHistory + histories = schedd_obj.jobEpochHistory( + f"ClusterId == {cluster_id}", + # ["list", "of", "desired", "attributes"], + ) + for i, history in enumerate(histories): + LOGGER.debug(f"jobEpochHistory #{i}") + LOGGER.debug(history) + + time.sleep(60) + LOGGER.info("requesting again...") diff --git a/dependencies-from-Dockerfile.log b/dependencies-from-Dockerfile.log index 6a7bfc1a..0cbe41fe 100644 --- a/dependencies-from-Dockerfile.log +++ b/dependencies-from-Dockerfile.log @@ -7,8 +7,8 @@ # pip freeze ######################################################################## backoff==2.2.1 -boto3==1.28.70 -botocore==1.31.70 +boto3==1.28.74 +botocore==1.31.74 cachetools==5.3.2 certifi==2023.7.22 cffi==1.16.0 @@ -60,7 +60,7 @@ typeguard==4.1.5 typing_extensions==4.8.0 urllib3==1.26.18 websocket-client==1.6.4 -wipac-dev-tools==1.7.0 +wipac-dev-tools==1.7.1 wipac-rest-tools==1.5.2 wipac-telemetry==0.3.0 wrapt==1.15.0 @@ -75,15 +75,15 @@ pip==23.2.1 pipdeptree==2.13.0 setuptools==65.5.1 skydriver-clientmanager -├── boto3 [required: Any, installed: 1.28.70] -│ ├── botocore [required: >=1.31.70,<1.32.0, installed: 1.31.70] +├── boto3 [required: Any, installed: 1.28.74] +│ ├── botocore [required: >=1.31.74,<1.32.0, installed: 1.31.74] │ │ ├── jmespath [required: >=0.7.1,<2.0.0, installed: 1.0.1] │ │ ├── python-dateutil [required: >=2.1,<3.0.0, installed: 2.8.2] │ │ │ └── six [required: >=1.5, installed: 1.16.0] │ │ └── urllib3 [required: >=1.25.4,<2.1, installed: 1.26.18] │ ├── jmespath [required: >=0.7.1,<2.0.0, installed: 1.0.1] │ └── s3transfer [required: >=0.7.0,<0.8.0, installed: 0.7.0] -│ └── botocore [required: >=1.12.36,<2.0a.0, installed: 1.31.70] +│ └── botocore [required: >=1.12.36,<2.0a.0, installed: 1.31.74] │ ├── jmespath [required: >=0.7.1,<2.0.0, installed: 1.0.1] │ ├── python-dateutil [required: >=2.1,<3.0.0, installed: 2.8.2] │ │ └── six [required: >=1.5, installed: 1.16.0] @@ -132,7 +132,7 @@ skydriver-clientmanager ├── tornado [required: Any, installed: 6.3.3] ├── typeguard [required: Any, installed: 4.1.5] │ └── typing-extensions [required: >=4.7.0, installed: 4.8.0] -├── wipac-dev-tools [required: Any, installed: 1.7.0] +├── wipac-dev-tools [required: Any, installed: 1.7.1] │ ├── requests [required: Any, installed: 2.31.0] │ │ ├── certifi [required: >=2017.4.17, installed: 2023.7.22] │ │ ├── charset-normalizer [required: >=2,<4, installed: 3.3.1] @@ -157,7 +157,7 @@ skydriver-clientmanager │ ├── idna [required: >=2.5,<4, installed: 3.4] │ └── urllib3 [required: >=1.21.1,<3, installed: 1.26.18] ├── tornado [required: Any, installed: 6.3.3] - └── wipac-dev-tools [required: Any, installed: 1.7.0] + └── wipac-dev-tools [required: Any, installed: 1.7.1] ├── requests [required: Any, installed: 2.31.0] │ ├── certifi [required: >=2017.4.17, installed: 2023.7.22] │ ├── charset-normalizer [required: >=2,<4, installed: 3.3.1] @@ -247,7 +247,7 @@ wipac-telemetry==0.3.0 │ └── typing-extensions [required: >=3.7.4, installed: 4.8.0] ├── protobuf [required: Any, installed: 3.20.3] ├── typing-extensions [required: Any, installed: 4.8.0] -└── wipac-dev-tools [required: Any, installed: 1.7.0] +└── wipac-dev-tools [required: Any, installed: 1.7.1] ├── requests [required: Any, installed: 2.31.0] │ ├── certifi [required: >=2017.4.17, installed: 2023.7.22] │ ├── charset-normalizer [required: >=2,<4, installed: 3.3.1]