From fe0638fa98cd1b9ef7eea8065f66c303d6e3f7fa Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 10 Oct 2024 14:08:39 +0200 Subject: [PATCH 1/2] Update for restructured NeXus base workflow from `ess.reduce.nexus` --- src/ess/dream/io/geant4.py | 44 +++++++++++++++++++--------- src/ess/dream/workflow.py | 16 +++++----- src/ess/powder/types.py | 11 +++---- tests/dream/geant4_reduction_test.py | 16 +++++----- tests/dream/io/geant4_test.py | 16 ++++------ tests/dream/io/nexus_test.py | 2 +- 6 files changed, 56 insertions(+), 49 deletions(-) diff --git a/src/ess/dream/io/geant4.py b/src/ess/dream/io/geant4.py index aba34e5..702f8c7 100644 --- a/src/ess/dream/io/geant4.py +++ b/src/ess/dream/io/geant4.py @@ -4,6 +4,7 @@ import numpy as np import sciline import scipp as sc +import scippnexus as snx from ess.powder.types import ( CalibratedDetector, @@ -11,12 +12,12 @@ CalibrationFilename, DetectorData, Filename, - NeXusDetector, + NeXusComponent, NeXusDetectorName, - NeXusSample, - NeXusSource, + Position, RunType, ) +from ess.reduce.nexus.types import CalibratedBeamline from ess.reduce.nexus.workflow import GenericNeXusWorkflow MANTLE_DETECTOR_ID = sc.index(7) @@ -60,9 +61,23 @@ def load_geant4_csv(file_path: Filename[RunType]) -> AllRawDetectors[RunType]: def extract_geant4_detector( detectors: AllRawDetectors[RunType], detector_name: NeXusDetectorName -) -> NeXusDetector[RunType]: +) -> NeXusComponent[snx.NXdetector, RunType]: """Extract a single detector from a loaded GEANT4 simulation.""" - return NeXusDetector[RunType](detectors["instrument"][detector_name]) + return NeXusComponent[snx.NXdetector, RunType]( + detectors["instrument"][detector_name] + ) + + +def get_calibrated_geant4_detector( + detector: NeXusComponent[snx.NXdetector, RunType], +) -> CalibratedDetector[RunType]: + """ + Replacement for :py:func:`ess.reduce.nexus.workflow.get_calibrated_detector`. + + Since the Geant4 detectors already have computed positions as well as logical shape, + this just extracts the relevant event data. + """ + return detector['events'].copy(deep=False) def _load_raw_events(file_path: str) -> sc.DataArray: @@ -171,21 +186,21 @@ def geant4_load_calibration(filename: CalibrationFilename) -> CalibrationData: def dummy_assemble_detector_data( - detector: CalibratedDetector[RunType], + detector: CalibratedBeamline[RunType], ) -> DetectorData[RunType]: """Dummy assembly of detector data, detector already contains neutron data.""" return DetectorData[RunType](detector) -def dummy_source() -> NeXusSource[RunType]: - return NeXusSource[RunType]( - sc.DataGroup(position=sc.vector([np.nan, np.nan, np.nan], unit="mm")) +def dummy_source_position() -> Position[snx.NXsource, RunType]: + return Position[snx.NXsource, RunType]( + sc.vector([np.nan, np.nan, np.nan], unit="mm") ) -def dummy_sample() -> NeXusSample[RunType]: - return NeXusSample[RunType]( - sc.DataGroup(position=sc.vector([np.nan, np.nan, np.nan], unit="mm")) +def dummy_sample_position() -> Position[snx.NXsample, RunType]: + return Position[snx.NXsample, RunType]( + sc.vector([np.nan, np.nan, np.nan], unit="mm") ) @@ -197,7 +212,8 @@ def LoadGeant4Workflow() -> sciline.Pipeline: wf.insert(extract_geant4_detector) wf.insert(load_geant4_csv) wf.insert(geant4_load_calibration) + wf.insert(get_calibrated_geant4_detector) wf.insert(dummy_assemble_detector_data) - wf.insert(dummy_source) - wf.insert(dummy_sample) + wf.insert(dummy_source_position) + wf.insert(dummy_sample_position) return wf diff --git a/src/ess/dream/workflow.py b/src/ess/dream/workflow.py index d959c9c..790fad3 100644 --- a/src/ess/dream/workflow.py +++ b/src/ess/dream/workflow.py @@ -5,12 +5,12 @@ import sciline import scipp as sc +import scippnexus as snx from ess.powder import providers as powder_providers from ess.powder.types import ( AccumulatedProtonCharge, - NeXusSample, - NeXusSource, + Position, SampleRun, VanadiumRun, ) @@ -23,14 +23,14 @@ def default_parameters() -> dict: # Quantities not available in the simulated data - sample = sc.DataGroup(position=sc.vector([0.0, 0.0, 0.0], unit="mm")) - source = sc.DataGroup(position=sc.vector([-3.478, 0.0, -76550], unit="mm")) + sample_position = sc.vector([0.0, 0.0, 0.0], unit="mm") + source_position = sc.vector([-3.478, 0.0, -76550], unit="mm") charge = sc.scalar(1.0, unit="µAh") return { - NeXusSample[SampleRun]: sample, - NeXusSample[VanadiumRun]: sample, - NeXusSource[SampleRun]: source, - NeXusSource[VanadiumRun]: source, + Position[snx.NXsample, SampleRun]: sample_position, + Position[snx.NXsample, VanadiumRun]: sample_position, + Position[snx.NXsource, SampleRun]: source_position, + Position[snx.NXsource, VanadiumRun]: source_position, AccumulatedProtonCharge[SampleRun]: charge, AccumulatedProtonCharge[VanadiumRun]: charge, CIFAuthors: CIFAuthors([]), diff --git a/src/ess/powder/types.py b/src/ess/powder/types.py index 2339403..4f9e190 100644 --- a/src/ess/powder/types.py +++ b/src/ess/powder/types.py @@ -29,15 +29,12 @@ MonitorData = reduce_t.MonitorData MonitorPositionOffset = reduce_t.MonitorPositionOffset MonitorType = reduce_t.MonitorType -NeXusMonitorName = reduce_t.NeXusMonitorName -NeXusDetector = reduce_t.NeXusDetector -NeXusMonitor = reduce_t.NeXusMonitor -NeXusSample = reduce_t.NeXusSample -NeXusSource = reduce_t.NeXusSource +NeXusDetectorName = reduce_t.NeXusDetectorName +NeXusMonitorName = reduce_t.NeXusName +NeXusComponent = reduce_t.NeXusComponent RunType = reduce_t.RunType SampleRun = reduce_t.SampleRun -SamplePosition = reduce_t.SamplePosition -SourcePosition = reduce_t.SourcePosition +Position = reduce_t.Position VanadiumRun = reduce_t.VanadiumRun DetectorBankSizes = reduce_t.DetectorBankSizes diff --git a/tests/dream/geant4_reduction_test.py b/tests/dream/geant4_reduction_test.py index 0fd0874..d2cabed 100644 --- a/tests/dream/geant4_reduction_test.py +++ b/tests/dream/geant4_reduction_test.py @@ -7,6 +7,7 @@ import sciline import scipp as sc import scipp.testing +import scippnexus as snx from scippneutron.io.cif import Author import ess.dream.data # noqa: F401 @@ -22,9 +23,8 @@ IofDspacingTwoTheta, MaskedData, NeXusDetectorName, - NeXusSample, - NeXusSource, NormalizedByProtonCharge, + Position, ReducedDspacingCIF, SampleRun, TofMask, @@ -35,8 +35,8 @@ WavelengthMask, ) -sample = sc.DataGroup(position=sc.vector([0.0, 0.0, 0.0], unit='mm')) -source = sc.DataGroup(position=sc.vector([-3.478, 0.0, -76550], unit='mm')) +sample = sc.vector([0.0, 0.0, 0.0], unit='mm') +source = sc.vector([-3.478, 0.0, -76550], unit='mm') charge = sc.scalar(1.0, unit='µAh') params = { @@ -48,10 +48,10 @@ DspacingBins: sc.linspace('dspacing', 0.0, 2.3434, 201, unit='angstrom'), TofMask: lambda x: (x < sc.scalar(0.0, unit='ns')) | (x > sc.scalar(86e6, unit='ns')), - NeXusSample[SampleRun]: sample, - NeXusSample[VanadiumRun]: sample, - NeXusSource[SampleRun]: source, - NeXusSource[VanadiumRun]: source, + Position[snx.NXsample, SampleRun]: sample, + Position[snx.NXsample, VanadiumRun]: sample, + Position[snx.NXsource, SampleRun]: source, + Position[snx.NXsource, VanadiumRun]: source, AccumulatedProtonCharge[SampleRun]: charge, AccumulatedProtonCharge[VanadiumRun]: charge, TwoThetaMask: None, diff --git a/tests/dream/io/geant4_test.py b/tests/dream/io/geant4_test.py index 43e6b58..81aaff8 100644 --- a/tests/dream/io/geant4_test.py +++ b/tests/dream/io/geant4_test.py @@ -8,17 +8,11 @@ import pytest import scipp as sc import scipp.testing +import scippnexus as snx from ess.dream import data, load_geant4_csv from ess.dream.io.geant4 import LoadGeant4Workflow -from ess.powder.types import ( - Filename, - NeXusDetector, - NeXusDetectorName, - NeXusSample, - NeXusSource, - SampleRun, -) +from ess.powder.types import Filename, NeXusComponent, NeXusDetectorName, SampleRun @pytest.fixture(scope="module") @@ -186,13 +180,13 @@ def test_geant4_in_pipeline(file_path, file): pipeline = LoadGeant4Workflow() pipeline[Filename[SampleRun]] = file_path pipeline[NeXusDetectorName] = NeXusDetectorName("mantle") - pipeline[NeXusSample[SampleRun]] = sc.DataGroup( + pipeline[NeXusComponent[snx.NXsample, SampleRun]] = sc.DataGroup( position=sc.vector([0.0, 0.0, 0.0], unit="mm") ) - pipeline[NeXusSource[SampleRun]] = sc.DataGroup( + pipeline[NeXusComponent[snx.NXsource, SampleRun]] = sc.DataGroup( position=sc.vector([-3.478, 0.0, -76550], unit="mm") ) - detector = pipeline.compute(NeXusDetector[SampleRun])['events'] + detector = pipeline.compute(NeXusComponent[snx.NXdetector, SampleRun])['events'] expected = load_geant4_csv(file)["instrument"]["mantle"]["events"] sc.testing.assert_identical(detector, expected) diff --git a/tests/dream/io/nexus_test.py b/tests/dream/io/nexus_test.py index c1a0b60..f8f9a3e 100644 --- a/tests/dream/io/nexus_test.py +++ b/tests/dream/io/nexus_test.py @@ -13,9 +13,9 @@ Filename, Monitor1, NeXusDetectorName, - NeXusMonitorName, SampleRun, ) +from ess.reduce.nexus.types import NeXusName as NeXusMonitorName bank_dims = {'wire', 'module', 'segment', 'strip', 'counter'} hr_sans_dims = {'strip', 'other'} From 42534de8f8a9d5dd7938b6036a16fb00aca541fa Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Fri, 11 Oct 2024 12:38:16 +0200 Subject: [PATCH 2/2] Bump essreduce --- pyproject.toml | 2 +- requirements/base.in | 2 +- requirements/base.txt | 16 ++++++++-------- requirements/basetest.txt | 6 +++--- requirements/ci.txt | 8 ++++---- requirements/dev.txt | 4 ++-- requirements/docs.txt | 14 +++++++------- requirements/nightly.txt | 8 ++++---- requirements/static.txt | 4 ++-- requirements/wheels.txt | 4 ++-- 10 files changed, 34 insertions(+), 34 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 661d1d0..3dfa61f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ requires-python = ">=3.10" # Make sure to list one dependency per line. dependencies = [ "dask", - "essreduce>=24.10.0", + "essreduce>=24.10.2", "graphviz", "numpy", "plopp", diff --git a/requirements/base.in b/requirements/base.in index 98e782f..248b776 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -3,7 +3,7 @@ # --- END OF CUSTOM SECTION --- # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! dask -essreduce>=24.10.0 +essreduce>=24.10.2 graphviz numpy plopp diff --git a/requirements/base.txt b/requirements/base.txt index b8b7bf5..4b1aa9d 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:3edca7ef7f02d8df072e91ce62fa7acd26cc0825 +# SHA1:86f9288f7221f3a8506d1cc40db8a4584011b0e0 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -15,7 +15,7 @@ comm==0.2.2 # via ipywidgets contourpy==1.3.0 # via matplotlib -cyclebane==24.6.0 +cyclebane==24.10.0 # via sciline cycler==0.12.1 # via matplotlib @@ -23,7 +23,7 @@ dask==2024.9.1 # via -r base.in decorator==5.1.1 # via ipython -essreduce==24.10.0 +essreduce==24.10.2 # via -r base.in exceptiongroup==1.2.2 # via ipython @@ -43,7 +43,7 @@ importlib-metadata==8.5.0 # via dask ipydatawidgets==4.3.5 # via pythreejs -ipython==8.27.0 +ipython==8.28.0 # via ipywidgets ipywidgets==8.1.5 # via @@ -65,9 +65,9 @@ matplotlib-inline==0.1.7 # via ipython mpltoolbox==24.5.1 # via scippneutron -networkx==3.3 +networkx==3.4 # via cyclebane -numpy==2.1.1 +numpy==2.1.2 # via # -r base.in # contourpy @@ -125,7 +125,7 @@ scipp==24.9.1 # scippnexus scippneutron==24.9.0 # via -r base.in -scippnexus==24.9.1 +scippnexus==24.10.0 # via # -r base.in # essreduce @@ -140,7 +140,7 @@ six==1.16.0 # python-dateutil stack-data==0.6.3 # via ipython -toolz==0.12.1 +toolz==1.0.0 # via # dask # partd diff --git a/requirements/basetest.txt b/requirements/basetest.txt index f5c7f8b..f726a66 100644 --- a/requirements/basetest.txt +++ b/requirements/basetest.txt @@ -7,7 +7,7 @@ # certifi==2024.8.30 # via requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests exceptiongroup==1.2.2 # via pytest @@ -15,7 +15,7 @@ idna==3.10 # via requests iniconfig==2.0.0 # via pytest -numpy==2.1.1 +numpy==2.1.2 # via pandas packaging==24.1 # via @@ -39,7 +39,7 @@ requests==2.32.3 # via pooch six==1.16.0 # via python-dateutil -tomli==2.0.1 +tomli==2.0.2 # via pytest tzdata==2024.2 # via pandas diff --git a/requirements/ci.txt b/requirements/ci.txt index 72003c0..aaa8fd2 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -11,11 +11,11 @@ certifi==2024.8.30 # via requests chardet==5.2.0 # via tox -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests colorama==0.4.6 # via tox -distlib==0.3.8 +distlib==0.3.9 # via virtualenv filelock==3.16.1 # via @@ -44,11 +44,11 @@ requests==2.32.3 # via -r ci.in smmap==5.0.1 # via gitdb -tomli==2.0.1 +tomli==2.0.2 # via # pyproject-api # tox -tox==4.21.0 +tox==4.21.2 # via -r ci.in typing-extensions==4.12.2 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 183e0b1..19ce3c1 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -81,7 +81,7 @@ pip-compile-multi==2.6.4 # via -r dev.in pip-tools==7.4.1 # via pip-compile-multi -plumbum==1.8.3 +plumbum==1.9.0 # via copier prometheus-client==0.21.0 # via jupyter-server @@ -115,7 +115,7 @@ terminado==0.18.1 # jupyter-server-terminals toposort==1.10 # via pip-compile-multi -types-python-dateutil==2.9.0.20240906 +types-python-dateutil==2.9.0.20241003 # via arrow uri-template==1.3.0 # via jsonschema diff --git a/requirements/docs.txt b/requirements/docs.txt index 15e373e..c66d0e5 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -26,9 +26,9 @@ bleach==6.1.0 # via nbconvert certifi==2024.8.30 # via requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests -debugpy==1.8.6 +debugpy==1.8.7 # via ipykernel defusedxml==0.7.1 # via nbconvert @@ -60,7 +60,7 @@ jinja2==3.1.4 # sphinx jsonschema==4.23.0 # via nbformat -jsonschema-specifications==2023.12.1 +jsonschema-specifications==2024.10.1 # via jsonschema jupyter-client==8.6.3 # via @@ -81,7 +81,7 @@ markdown-it-py==3.0.0 # via # mdit-py-plugins # myst-parser -markupsafe==2.1.5 +markupsafe==3.0.1 # via # jinja2 # nbconvert @@ -150,7 +150,7 @@ snowballstemmer==2.2.0 # via sphinx soupsieve==2.6 # via beautifulsoup4 -sphinx==8.0.2 +sphinx==8.1.0 # via # -r docs.in # myst-parser @@ -160,7 +160,7 @@ sphinx==8.0.2 # sphinx-copybutton # sphinx-design # sphinxcontrib-bibtex -sphinx-autodoc-typehints==2.4.4 +sphinx-autodoc-typehints==2.5.0 # via -r docs.in sphinx-copybutton==0.5.2 # via -r docs.in @@ -182,7 +182,7 @@ sphinxcontrib-serializinghtml==2.0.0 # via sphinx tinycss2==1.3.0 # via nbconvert -tomli==2.0.1 +tomli==2.0.2 # via sphinx tornado==6.4.1 # via diff --git a/requirements/nightly.txt b/requirements/nightly.txt index 05a9404..a7bdfc6 100644 --- a/requirements/nightly.txt +++ b/requirements/nightly.txt @@ -16,7 +16,7 @@ comm==0.2.2 # via ipywidgets contourpy==1.3.0 # via matplotlib -cyclebane==24.6.0 +cyclebane==24.10.0 # via sciline cycler==0.12.1 # via matplotlib @@ -42,7 +42,7 @@ importlib-metadata==8.5.0 # via dask ipydatawidgets==4.3.5 # via pythreejs -ipython==8.27.0 +ipython==8.28.0 # via ipywidgets ipywidgets==8.1.5 # via @@ -64,7 +64,7 @@ matplotlib-inline==0.1.7 # via ipython mpltoolbox==24.5.1 # via scippneutron -networkx==3.3 +networkx==3.4 # via cyclebane parso==0.8.4 # via jedi @@ -115,7 +115,7 @@ scipy==1.14.1 # scippnexus stack-data==0.6.3 # via ipython -toolz==0.12.1 +toolz==1.0.0 # via # dask # partd diff --git a/requirements/static.txt b/requirements/static.txt index 6f2ecf5..6d65fbd 100644 --- a/requirements/static.txt +++ b/requirements/static.txt @@ -7,7 +7,7 @@ # cfgv==3.4.0 # via pre-commit -distlib==0.3.8 +distlib==0.3.9 # via virtualenv filelock==3.16.1 # via virtualenv @@ -17,7 +17,7 @@ nodeenv==1.9.1 # via pre-commit platformdirs==4.3.6 # via virtualenv -pre-commit==3.8.0 +pre-commit==4.0.1 # via -r static.in pyyaml==6.0.2 # via pre-commit diff --git a/requirements/wheels.txt b/requirements/wheels.txt index e9a7ea9..d1c1063 100644 --- a/requirements/wheels.txt +++ b/requirements/wheels.txt @@ -5,11 +5,11 @@ # # pip-compile-multi # -build==1.2.2 +build==1.2.2.post1 # via -r wheels.in packaging==24.1 # via build pyproject-hooks==1.2.0 # via build -tomli==2.0.1 +tomli==2.0.2 # via build