Merge branch 'tickets/DM-48449'

lsst-dm · Jan 17, 2025 · a937a79 · a937a79
2 parents 6929bc4 + 65e4d12
commit a937a79
Show file tree

Hide file tree

Showing 7 changed files with 24 additions and 28 deletions.
diff --git a/docker/ingestd/Dockerfile b/docker/ingestd/Dockerfile
@@ -8,10 +8,6 @@ RUN git clone https://github.com/lsst-dm/ctrl_ingestd && \
     cd ctrl_ingestd && \
     git checkout ${CTRL_INGESTD_GIT_VERSION}
 
-ARG CONFLUENT_KAFKA_VERSION
-RUN source /opt/lsst/software/stack/loadLSST.bash && \
-    mamba install python-confluent-kafka=${CONFLUENT_KAFKA_VERSION}
-
 RUN source /opt/lsst/software/stack/loadLSST.bash && \
     setup sconsUtils && \
     setup utils && \

diff --git a/docker/testenv-ingestd/Dockerfile b/docker/testenv-ingestd/Dockerfile
@@ -6,9 +6,6 @@ USER root
 WORKDIR /home/lsst
 
 USER lsst
-ARG CONFLUENT_KAFKA_VERSION
-RUN source /opt/lsst/software/stack/loadLSST.bash && \
-    pip install confluent_kafka==${CONFLUENT_KAFKA_VERSION}
 
 COPY run.sh /home/lsst/run.sh
 

diff --git a/docker/versions.env b/docker/versions.env
@@ -1,4 +1,4 @@
-CTRL_INGESTD_VERSION=1.4
-CTRL_INGESTD_GIT_VERSION=tickets/DM-47437
-LSST_STACK_VERSION=7-stack-lsst_distrib-w_2024_22
+CTRL_INGESTD_VERSION=1.5
+CTRL_INGESTD_GIT_VERSION=tickets/DM-48449
+LSST_STACK_VERSION=7-stack-lsst_distrib-w_2025_02
 CONFLUENT_KAFKA_VERSION=2.2.0
diff --git a/python/lsst/ctrl/ingestd/rseButler.py b/python/lsst/ctrl/ingestd/rseButler.py
@@ -40,7 +40,7 @@ def __init__(self, repo: str):
 
         self.butler = Butler(repo, writeable=True)
 
-    def create_entry(self, butler_file: str, sidecar: dict) -> FileDataset:
+    def create_entry(self, butler_file: str, sidecar: str) -> FileDataset:
         """Create a FileDatset with sidecar information
 
         Parameters

diff --git a/tests/data/message.json b/tests/data/message.json
@@ -1 +1 @@
-{"id": "cdcb2ee0cebe471aa36196e28de33db4", "created_at": "datetime.datetime(2023, 11, 29, 20, 19, 3, 275833)", "event_type": "transfer-done", "services": "kafka", "payload": {"activity": "User Subscriptions", "request-id": "a50f0e3ad52a4b1ea7f072a719b5141f", "duration": 8, "checksum-adler": "95935a72", "checksum-md5": "None", "file-size": 57980160, "bytes": 57980160, "guid": "None", "previous-request-id": "None", "protocol": "root", "scope": "test", "name": "srp/data/calexp_HSC_y_HSC-Y_330_1_54_HSC_runs_RC2_w_2023_32_DM-40356_20230812T080035Z.fits", "dataset": "None", "datasetScope": "None", "src-type": "DISK", "src-rse": "XRD1", "src-url": "root://xrd1:1094//rucio/test/srp/data/calexp_HSC_y_HSC-Y_330_1_54_HSC_runs_RC2_w_2023_32_DM-40356_20230812T080035Z.fits", "dst-type": "DISK", "dst-rse": "XRD2", "dst-url": "root://xrd2:1095//rucio/test/srp/data/calexp_HSC_y_HSC-Y_330_1_54_HSC_runs_RC2_w_2023_32_DM-40356_20230812T080035Z.fits", "reason": "", "transfer-endpoint": "https://fts:8446", "transfer-id": "6dfb5aaa-8ef4-11ee-9688-0242ac16000b", "transfer-link": "None", "created_at": "2023-11-29 20:18:04.425276", "submitted_at": "2023-11-29 20:18:16.351754", "started_at": "2023-11-29 20:18:31", "transferred_at": "2023-11-29 20:18:39", "tool-id": "rucio-conveyor", "account": "root", "datatype": "None", "transfer_link": "https://fts:8449/fts3/ftsmon/#/job/6dfb5aaa-8ef4-11ee-9688-0242ac16000b", "rubin_butler": 1, "rubin_sidecar": {"id": "00a86e99-7661-4f14-ae0d-93d3d4162e26", "datasetType": {"name": "calexp", "storageClass": "ExposureF", "dimensions": {"names": ["band", "instrument", "detector", "physical_filter", "visit"]}}, "dataId": {"dataId": {"band": "y", "instrument": "HSC", "detector": 1, "physical_filter": "HSC-Y", "visit": 330}}, "run": "HSC/runs/RC2/w_2023_32/DM-40356/20230812T080035Z"}}}
+{"event_type": "transfer-done", "payload": {"activity": "User Subscriptions", "request-id": "217d5d321a394d349b82829cbb410b3f", "duration": 3, "checksum-adler": "3210cb3f", "checksum-md5": "7018315389da34f8e4d1d2d3c57b014a", "file-size": 1365120, "bytes": 1365120, "guid": "None", "previous-request-id": "None", "protocol": "root", "scope": "test", "name": "something/visitSummary_HSC_y_HSC-Y_328_HSC_runs_RC2_w_2023_32_DM-40356_20230814T170253Z.fits", "dataset": "rubin_dataset", "datasetScope": "test", "src-type": "DISK", "src-rse": "XRD1", "src-url": "root://xrd1:1094//rucio/test/something/visitSummary_HSC_y_HSC-Y_328_HSC_runs_RC2_w_2023_32_DM-40356_20230814T170253Z.fits", "dst-type": "DISK", "dst-rse": "XRD3", "dst-url": "root://xrd3:1096//rucio/test/something/visitSummary_HSC_y_HSC-Y_328_HSC_runs_RC2_w_2023_32_DM-40356_20230814T170253Z.fits", "reason": "", "transfer-endpoint": "https://fts:8446", "transfer-id": "602975dc-d445-11ef-b2f8-0242ac130013", "transfer-link": "None", "created_at": "2025-01-16 20:05:52.814896", "submitted_at": "2025-01-16 20:06:30.300257", "started_at": "2025-01-16 20:06:41", "transferred_at": "2025-01-16 20:06:44", "tool-id": "rucio-conveyor", "account": "root", "datatype": "None", "transfer_link": "https://fts:8449/fts3/ftsmon/#/job/602975dc-d445-11ef-b2f8-0242ac130013", "rubin_butler": 1, "rubin_sidecar": {"id":"0ef08762-b0dd-4a02-8b1c-e09b1544992d","datasetType":{"name":"visitSummary","storageClass":"ExposureCatalog","dimensions":["instrument","visit"]},"dataId":{"dataId":{"instrument":"HSC","visit":328,"band":"y","physical_filter":"HSC-Y"}},"run":"HSC/runs/RC2/w_2023_32/DM-40356/20230814T170253Z"}}, "created_at": "2025-01-16 20:07:16.564403"}
diff --git a/tests/test_message.py b/tests/test_message.py
@@ -46,27 +46,31 @@ def configure(self, json_name):
 
     def testAttributes(self):
         self.configure("message.json")
-        self.assertEqual(self.msg.get_dst_rse(), "XRD2")
+        self.assertEqual(self.msg.get_dst_rse(), "XRD3")
         self.assertEqual(
             self.msg.get_dst_url(),
             (
-                "root://xrd2:1095//rucio/test/srp/data/"
-                "calexp_HSC_y_HSC-Y_330_1_54_HSC_runs_RC2_w_2023_32_"
-                "DM-40356_20230812T080035Z.fits"
+                "root://xrd3:1096//rucio/test/something/"
+                "visitSummary_HSC_y_HSC-Y_328_HSC_runs_"
+                "RC2_w_2023_32_DM-40356_20230814T170253Z.fits"
             ),
         )
         self.assertEqual(self.msg.get_rubin_butler(), 1)
         sidecar_str = self.msg.get_rubin_sidecar_str()
+        print(f"{sidecar_str=}")
         self.assertEqual(
             sidecar_str,
             (
-                '{"id": "00a86e99-7661-4f14-ae0d-93d3d4162e26",'
-                ' "datasetType": {"name": "calexp", "storageClass": "ExposureF",'
-                ' "dimensions": {"names": ["band", "instrument", "detector",'
-                ' "physical_filter", "visit"]}}, "dataId": {"dataId":'
-                ' {"band": "y", "instrument": "HSC", "detector": 1,'
-                ' "physical_filter": "HSC-Y", "visit": 330}}, "run":'
-                ' "HSC/runs/RC2/w_2023_32/DM-40356/20230812T080035Z"}'
+                '{"id": "0ef08762-b0dd-4a02-8b1c-e09b1544992d",'
+                ' "datasetType": {"name": "visitSummary",'
+                ' "storageClass": "ExposureCatalog",'
+                ' "dimensions": ["instrument",'
+                ' "visit"]},'
+                ' "dataId": {"dataId": {"instrument": "HSC",'
+                ' "visit": 328,'
+                ' "band": "y",'
+                ' "physical_filter": "HSC-Y"}}, "run":'
+                ' "HSC/runs/RC2/w_2023_32/DM-40356/20230814T170253Z"}'
             ),
         )
 

diff --git a/tests/test_rseButler.py b/tests/test_rseButler.py
@@ -61,11 +61,10 @@ def testRseButler(self):
         instr.register(butler.butler.registry)
         butler.butler.import_(filename=prep_file)
 
-        self.temp_file = tempfile.NamedTemporaryFile()
-
-        sidecar_str = self.msg.get_rubin_sidecar_str()
-        fds = butler.create_entry(self.temp_file.name, sidecar_str)
-        butler.ingest([fds])
+        with tempfile.NamedTemporaryFile() as temp_file:
+            sidecar_str = self.msg.get_rubin_sidecar_str()
+            fds = butler.create_entry(temp_file.name, sidecar_str)
+            butler.ingest([fds])
 
 
 class MemoryTester(lsst.utils.tests.MemoryTestCase):
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		{"id": "cdcb2ee0cebe471aa36196e28de33db4", "created_at": "datetime.datetime(2023, 11, 29, 20, 19, 3, 275833)", "event_type": "transfer-done", "services": "kafka", "payload": {"activity": "User Subscriptions", "request-id": "a50f0e3ad52a4b1ea7f072a719b5141f", "duration": 8, "checksum-adler": "95935a72", "checksum-md5": "None", "file-size": 57980160, "bytes": 57980160, "guid": "None", "previous-request-id": "None", "protocol": "root", "scope": "test", "name": "srp/data/calexp_HSC_y_HSC-Y_330_1_54_HSC_runs_RC2_w_2023_32_DM-40356_20230812T080035Z.fits", "dataset": "None", "datasetScope": "None", "src-type": "DISK", "src-rse": "XRD1", "src-url": "root://xrd1:1094//rucio/test/srp/data/calexp_HSC_y_HSC-Y_330_1_54_HSC_runs_RC2_w_2023_32_DM-40356_20230812T080035Z.fits", "dst-type": "DISK", "dst-rse": "XRD2", "dst-url": "root://xrd2:1095//rucio/test/srp/data/calexp_HSC_y_HSC-Y_330_1_54_HSC_runs_RC2_w_2023_32_DM-40356_20230812T080035Z.fits", "reason": "", "transfer-endpoint": "https://fts:8446", "transfer-id": "6dfb5aaa-8ef4-11ee-9688-0242ac16000b", "transfer-link": "None", "created_at": "2023-11-29 20:18:04.425276", "submitted_at": "2023-11-29 20:18:16.351754", "started_at": "2023-11-29 20:18:31", "transferred_at": "2023-11-29 20:18:39", "tool-id": "rucio-conveyor", "account": "root", "datatype": "None", "transfer_link": "https://fts:8449/fts3/ftsmon/#/job/6dfb5aaa-8ef4-11ee-9688-0242ac16000b", "rubin_butler": 1, "rubin_sidecar": {"id": "00a86e99-7661-4f14-ae0d-93d3d4162e26", "datasetType": {"name": "calexp", "storageClass": "ExposureF", "dimensions": {"names": ["band", "instrument", "detector", "physical_filter", "visit"]}}, "dataId": {"dataId": {"band": "y", "instrument": "HSC", "detector": 1, "physical_filter": "HSC-Y", "visit": 330}}, "run": "HSC/runs/RC2/w_2023_32/DM-40356/20230812T080035Z"}}}
		{"event_type": "transfer-done", "payload": {"activity": "User Subscriptions", "request-id": "217d5d321a394d349b82829cbb410b3f", "duration": 3, "checksum-adler": "3210cb3f", "checksum-md5": "7018315389da34f8e4d1d2d3c57b014a", "file-size": 1365120, "bytes": 1365120, "guid": "None", "previous-request-id": "None", "protocol": "root", "scope": "test", "name": "something/visitSummary_HSC_y_HSC-Y_328_HSC_runs_RC2_w_2023_32_DM-40356_20230814T170253Z.fits", "dataset": "rubin_dataset", "datasetScope": "test", "src-type": "DISK", "src-rse": "XRD1", "src-url": "root://xrd1:1094//rucio/test/something/visitSummary_HSC_y_HSC-Y_328_HSC_runs_RC2_w_2023_32_DM-40356_20230814T170253Z.fits", "dst-type": "DISK", "dst-rse": "XRD3", "dst-url": "root://xrd3:1096//rucio/test/something/visitSummary_HSC_y_HSC-Y_328_HSC_runs_RC2_w_2023_32_DM-40356_20230814T170253Z.fits", "reason": "", "transfer-endpoint": "https://fts:8446", "transfer-id": "602975dc-d445-11ef-b2f8-0242ac130013", "transfer-link": "None", "created_at": "2025-01-16 20:05:52.814896", "submitted_at": "2025-01-16 20:06:30.300257", "started_at": "2025-01-16 20:06:41", "transferred_at": "2025-01-16 20:06:44", "tool-id": "rucio-conveyor", "account": "root", "datatype": "None", "transfer_link": "https://fts:8449/fts3/ftsmon/#/job/602975dc-d445-11ef-b2f8-0242ac130013", "rubin_butler": 1, "rubin_sidecar": {"id":"0ef08762-b0dd-4a02-8b1c-e09b1544992d","datasetType":{"name":"visitSummary","storageClass":"ExposureCatalog","dimensions":["instrument","visit"]},"dataId":{"dataId":{"instrument":"HSC","visit":328,"band":"y","physical_filter":"HSC-Y"}},"run":"HSC/runs/RC2/w_2023_32/DM-40356/20230814T170253Z"}}, "created_at": "2025-01-16 20:07:16.564403"}