From 81145a0dab993e87e1b6803fc4f7f71694bb5e51 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 5 Sep 2024 14:12:26 +0200 Subject: [PATCH 01/19] Do not auto-compute component position unused arg Compute component pos later Fix for non-nexus component "groups" --- src/ess/reduce/nexus/__init__.py | 4 ++-- src/ess/reduce/nexus/_nexus_loader.py | 9 ++++++-- src/ess/reduce/nexus/workflow.py | 14 ++++++++---- tests/nexus/nexus_loader_test.py | 32 ++++++++++++++++++--------- tests/nexus/workflow_test.py | 27 ++++++++++++---------- 5 files changed, 56 insertions(+), 30 deletions(-) diff --git a/src/ess/reduce/nexus/__init__.py b/src/ess/reduce/nexus/__init__.py index 66dbbeaa..335c8e90 100644 --- a/src/ess/reduce/nexus/__init__.py +++ b/src/ess/reduce/nexus/__init__.py @@ -16,7 +16,7 @@ group_event_data, load_component, compute_component_position, - extract_events_or_histogram, + extract_signal_data_array, ) __all__ = [ @@ -25,5 +25,5 @@ 'load_data', 'load_component', 'compute_component_position', - 'extract_events_or_histogram', + 'extract_signal_data_array', ] diff --git a/src/ess/reduce/nexus/_nexus_loader.py b/src/ess/reduce/nexus/_nexus_loader.py index 149272ec..35243b0e 100644 --- a/src/ess/reduce/nexus/_nexus_loader.py +++ b/src/ess/reduce/nexus/_nexus_loader.py @@ -41,10 +41,15 @@ def load_component( component = _unique_child_group(instrument, nx_class, group_name) loaded = cast(sc.DataGroup, component[selection]) loaded['nexus_component_name'] = component.name.split('/')[-1] - return compute_component_position(loaded) + return loaded def compute_component_position(dg: sc.DataGroup) -> sc.DataGroup: + # In some downstream packages we use some of the Nexus components which attempt + # to compute positions without having actual Nexus data defining depends_on chains. + # We assume positions have been set in the non-Nexus input somehow and return early. + if 'depends_on' not in dg: + return dg transform_out_name = 'transform' if transform_out_name in dg: raise RuntimeError( @@ -115,7 +120,7 @@ def _contains_nx_class(group: snx.Group, nx_class: type[snx.NXobject]) -> bool: return False -def extract_events_or_histogram(dg: sc.DataGroup) -> sc.DataArray: +def extract_signal_data_array(dg: sc.DataGroup) -> sc.DataArray: event_data_arrays = { key: value for key, value in dg.items() diff --git a/src/ess/reduce/nexus/workflow.py b/src/ess/reduce/nexus/workflow.py index 12d87aff..46fff776 100644 --- a/src/ess/reduce/nexus/workflow.py +++ b/src/ess/reduce/nexus/workflow.py @@ -292,6 +292,8 @@ def load_nexus_monitor( location: Location spec for the monitor group. """ + definitions = snx.base_definitions() + definitions["NXmonitor"] = _StrippedMonitor return NeXusMonitor[RunType, MonitorType]( nexus.load_component(location, nx_class=snx.NXmonitor, definitions=definitions) ) @@ -348,7 +350,8 @@ def get_source_position(source: NeXusSource[RunType]) -> SourcePosition[RunType] source: NeXus source group. """ - return SourcePosition[RunType](source["position"]) + dg = nexus.compute_component_position(source) + return SourcePosition[RunType](dg["position"]) def get_sample_position(sample: NeXusSample[RunType]) -> SamplePosition[RunType]: @@ -362,7 +365,8 @@ def get_sample_position(sample: NeXusSample[RunType]) -> SamplePosition[RunType] sample: NeXus sample group. """ - return SamplePosition[RunType](sample.get("position", origin)) + dg = nexus.compute_component_position(sample) + return SamplePosition[RunType](dg.get("position", origin)) def get_calibrated_detector( @@ -397,7 +401,8 @@ def get_calibrated_detector( bank_sizes: Dictionary of detector bank sizes. """ - da = nexus.extract_events_or_histogram(detector) + detector = nexus.compute_component_position(detector) + da = nexus.extract_signal_data_array(detector) if ( sizes := (bank_sizes or {}).get(detector.get('nexus_component_name')) ) is not None: @@ -459,8 +464,9 @@ def get_calibrated_monitor( source_position: Position of the neutron source. """ + monitor = nexus.compute_component_position(monitor) return CalibratedMonitor[RunType, MonitorType]( - nexus.extract_events_or_histogram(monitor).assign_coords( + nexus.extract_signal_data_array(monitor).assign_coords( position=monitor['position'] + offset.to(unit=monitor['position'].unit), source_position=source_position, ) diff --git a/tests/nexus/nexus_loader_test.py b/tests/nexus/nexus_loader_test.py index 1a204d16..29f9a71e 100644 --- a/tests/nexus/nexus_loader_test.py +++ b/tests/nexus/nexus_loader_test.py @@ -251,6 +251,7 @@ def test_load_detector(nexus_file, expected_bank12, entry_name, selection): if selection is not None: loc.selection = selection detector = nexus.load_component(loc, nx_class=snx.NXdetector) + detector = nexus.compute_component_position(detector) if selection: expected = expected_bank12.bins[selection] expected.coords.pop(selection[0]) @@ -282,7 +283,8 @@ def test_load_and_group_event_data_consistent_with_load_via_detector( ) if selection: loc.selection = selection - detector = nexus.load_component(loc, nx_class=snx.NXdetector)['bank12_events'] + detector = nexus.load_component(loc, nx_class=snx.NXdetector) + detector = nexus.compute_component_position(detector)['bank12_events'] events = nexus.load_data( nexus_file, selection=selection, @@ -300,7 +302,8 @@ def test_group_event_data_does_not_modify_input(nexus_file): filename=nexus_file, component_name=nexus.types.NeXusDetectorName('bank12'), ) - detector = nexus.load_component(loc, nx_class=snx.NXdetector)['bank12_events'] + detector = nexus.load_component(loc, nx_class=snx.NXdetector) + detector = nexus.compute_component_position(detector)['bank12_events'] events = nexus.load_data( nexus_file, component_name=nexus.types.NeXusDetectorName('bank12'), @@ -328,7 +331,7 @@ def test_load_detector_open_file_with_new_definitions_raises(nexus_file): nexus.load_component(loc, nx_class=snx.NXdetector, definitions={}) -def test_load_detector_new_definitions_applied(nexus_file, expected_bank12): +def test_load_detector_new_definitions_applied(nexus_file): if not isinstance(nexus_file, snx.Group): new_definition_used = False @@ -381,6 +384,7 @@ def test_load_detector_select_entry_if_not_unique(nexus_file, expected_bank12): entry_name=nexus.types.NeXusEntryName('entry-001'), ) detector = nexus.load_component(loc, nx_class=snx.NXdetector) + detector = nexus.compute_component_position(detector) sc.testing.assert_identical(detector['bank12_events'], expected_bank12) @@ -402,6 +406,7 @@ def test_load_monitor(nexus_file, expected_monitor, entry_name, selection): if selection is not None: loc.selection = selection monitor = nexus.load_component(loc, nx_class=snx.NXmonitor) + monitor = nexus.compute_component_position(monitor) expected = expected_monitor[selection] if selection else expected_monitor sc.testing.assert_identical(monitor['data'], expected) @@ -415,6 +420,7 @@ def test_load_source(nexus_file, expected_source, entry_name, source_name): component_name=source_name, ) source = nexus.load_component(loc, nx_class=snx.NXsource) + source = nexus.compute_component_position(source) # NeXus details that we don't need to test as long as the positions are ok: del source['depends_on'] del source['transformations'] @@ -460,7 +466,7 @@ def test_extract_detector_data(): ' _': sc.linspace('xx', 2, 3, 10), } ) - data = nexus.extract_events_or_histogram(detector) + data = nexus.extract_signal_data_array(detector) sc.testing.assert_identical(data, detector['jdl2ab']) @@ -472,7 +478,7 @@ def test_extract_monitor_data(): ' _': sc.linspace('xx', 2, 3, 10), } ) - data = nexus.extract_events_or_histogram(monitor) + data = nexus.extract_signal_data_array(monitor) sc.testing.assert_identical(data, monitor['(eed)']) @@ -488,17 +494,17 @@ def test_extract_detector_data_requires_unique_dense_data(): with pytest.raises( ValueError, match="Cannot uniquely identify the data to extract" ): - nexus.extract_events_or_histogram(detector) + nexus.extract_signal_data_array(detector) def test_extract_detector_data_ignores_position_data_array(): detector = sc.DataGroup(jdl2ab=sc.data.data_xy(), position=sc.data.data_xy()) - nexus.extract_events_or_histogram(detector) + nexus.extract_signal_data_array(detector) def test_extract_detector_data_ignores_transform_data_array(): detector = sc.DataGroup(jdl2ab=sc.data.data_xy(), transform=sc.data.data_xy()) - nexus.extract_events_or_histogram(detector) + nexus.extract_signal_data_array(detector) def test_extract_detector_data_requires_unique_event_data(): @@ -513,7 +519,7 @@ def test_extract_detector_data_requires_unique_event_data(): with pytest.raises( ValueError, match="Cannot uniquely identify the data to extract" ): - nexus.extract_events_or_histogram(detector) + nexus.extract_signal_data_array(detector) def test_extract_detector_data_favors_event_data_over_histogram_data(): @@ -525,5 +531,11 @@ def test_extract_detector_data_favors_event_data_over_histogram_data(): ' _': sc.linspace('xx', 2, 3, 10), } ) - data = nexus.extract_events_or_histogram(detector) + data = nexus.extract_signal_data_array(detector) sc.testing.assert_identical(data, detector['lob']) + + +def compute_component_position_returns_input_if_no_depends_on() -> None: + dg = sc.DataGroup(position=sc.vector([1, 2, 3], unit='m')) + result = nexus.compute_component_position(dg) + assert result is dg diff --git a/tests/nexus/workflow_test.py b/tests/nexus/workflow_test.py index 41add63f..67226292 100644 --- a/tests/nexus/workflow_test.py +++ b/tests/nexus/workflow_test.py @@ -5,7 +5,7 @@ from scipp.testing import assert_identical from ess.reduce import data -from ess.reduce.nexus import workflow +from ess.reduce.nexus import compute_component_position, workflow from ess.reduce.nexus.types import ( DetectorData, Filename, @@ -28,6 +28,7 @@ def group_with_no_position(request) -> workflow.NeXusSample[SampleRun]: def test_sample_position_returns_position_of_group() -> None: + depends_on = sc.spatial.translation(value=[1.0, 2.0, 3.0], unit='m') position = sc.vector([1.0, 2.0, 3.0], unit='m') sample_group = workflow.NeXusSample[SampleRun](sc.DataGroup(position=position)) assert_identical(workflow.get_sample_position(sample_group), position) @@ -42,6 +43,7 @@ def test_get_sample_position_returns_origin_if_position_not_found( def test_get_source_position_returns_position_of_group() -> None: + depends_on = sc.spatial.translation(value=[1.0, 2.0, 3.0], unit='m') position = sc.vector([1.0, 2.0, 3.0], unit='m') source_group = workflow.NeXusSource[SampleRun](sc.DataGroup(position=position)) assert_identical(workflow.get_source_position(source_group), position) @@ -57,22 +59,17 @@ def test_get_source_position_raises_exception_if_position_not_found( @pytest.fixture() def nexus_detector() -> workflow.NeXusDetector[SampleRun]: detector_number = sc.arange('detector_number', 6, unit=None) - x = sc.linspace('detector_number', 0, 1, num=6, unit='m') - position = sc.spatial.as_vectors(x, sc.zeros_like(x), sc.zeros_like(x)) data = sc.DataArray( sc.empty_like(detector_number), coords={ 'detector_number': detector_number, - 'position': position, + 'x_pixel_offset': sc.linspace('detector_number', 0, 1, num=6, unit='m'), }, ) return workflow.NeXusDetector[SampleRun]( sc.DataGroup( data=data, - # Note that this position (the overall detector position) will be ignored, - # only the pixel positions (typically defined relative to this in an - # actual NeXus file) will be used. - position=sc.vector([1.0, 2.0, 3.0], unit='m'), + depends_on=sc.spatial.translation(value=[1.0, 2.0, 3.0], unit='m'), nexus_component_name='detector1', ) ) @@ -97,7 +94,7 @@ def test_get_calibrated_detector_extracts_data_field_from_nexus_detector( ) assert_identical( detector.drop_coords(('sample_position', 'source_position', 'gravity')), - nexus_detector['data'], + compute_component_position(nexus_detector)['data'], ) @@ -146,7 +143,9 @@ def test_get_calibrated_detector_adds_offset_to_position( gravity=workflow.gravity_vector_neg_y(), bank_sizes={}, ) - position = nexus_detector['data'].coords['position'] + offset + position = ( + compute_component_position(nexus_detector)['data'].coords['position'] + offset + ) assert detector.coords['position'].sizes == {'detector_number': 6} assert_identical(detector.coords['position'], position) @@ -264,7 +263,10 @@ def test_assemble_detector_preserves_masks(calibrated_detector, detector_event_d def nexus_monitor() -> workflow.NeXusMonitor[SampleRun, Monitor1]: data = sc.DataArray(sc.scalar(1.2), coords={'something': sc.scalar(13)}) return workflow.NeXusMonitor[SampleRun, Monitor1]( - sc.DataGroup(data=data, position=sc.vector([1.0, 2.0, 3.0], unit='m')) + sc.DataGroup( + data=data, + depends_on=sc.spatial.translation(value=[1.0, 2.0, 3.0], unit='m'), + ) ) @@ -277,7 +279,8 @@ def test_get_calibrated_monitor_extracts_data_field_from_nexus_monitor( source_position=sc.vector([0.0, 0.0, -10.0], unit='m'), ) assert_identical( - monitor.drop_coords(('position', 'source_position')), nexus_monitor['data'] + monitor.drop_coords(('position', 'source_position')), + compute_component_position(nexus_monitor)['data'], ) From 596cf26d48f67f8bd61d4c55ce1f2f97c59eba3e Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 2 Oct 2024 09:18:45 +0200 Subject: [PATCH 02/19] Add example of extrating position via separate TransformationChain --- src/ess/reduce/nexus/types.py | 17 ++++++++++ src/ess/reduce/nexus/workflow.py | 35 ++++++++++++++++----- tests/nexus/workflow_test.py | 54 +++++++++++++++++++++----------- 3 files changed, 80 insertions(+), 26 deletions(-) diff --git a/src/ess/reduce/nexus/types.py b/src/ess/reduce/nexus/types.py index 2933aa02..e1377212 100644 --- a/src/ess/reduce/nexus/types.py +++ b/src/ess/reduce/nexus/types.py @@ -237,3 +237,20 @@ class NeXusMonitorDataLocationSpec( NeXusComponentLocationSpec[snx.NXevent_data, RunType], Generic[RunType, MonitorType] ): """NeXus filename and parameters to identify (parts of) monitor data to load.""" + + +T = TypeVar('T', bound='NeXusTransformationChain') + + +@dataclass +class NeXusTransformationChain(snx.TransformationChain, Generic[Component, RunType]): + @classmethod + def from_base(cls: type[T], base: snx.TransformationChain) -> T: + return cls( + parent=base.parent, + value=base.value, + transformations=base.transformations, + ) + + def compute_position(self) -> sc.Variable | sc.DataArray: + return self.compute() * sc.vector([0, 0, 0], unit='m') diff --git a/src/ess/reduce/nexus/workflow.py b/src/ess/reduce/nexus/workflow.py index 46fff776..7694ad82 100644 --- a/src/ess/reduce/nexus/workflow.py +++ b/src/ess/reduce/nexus/workflow.py @@ -36,6 +36,7 @@ NeXusMonitorName, NeXusSample, NeXusSource, + NeXusTransformationChain, PreopenNeXusFile, PulseSelection, RunType, @@ -341,17 +342,33 @@ def load_nexus_monitor_data( ) -def get_source_position(source: NeXusSource[RunType]) -> SourcePosition[RunType]: +def get_source_transformation_chain( + source: NeXusSource[RunType], +) -> NeXusTransformationChain[snx.NXsource, RunType]: """ - Extract the source position from a NeXus source group. + Extract the transformation chain from a NeXus source group. Parameters ---------- source: NeXus source group. """ - dg = nexus.compute_component_position(source) - return SourcePosition[RunType](dg["position"]) + chain = source['depends_on'] + return NeXusTransformationChain[snx.NXsource, RunType].from_base(chain) + + +def get_source_position( + transformations: NeXusTransformationChain[snx.NXsource, RunType], +) -> SourcePosition[RunType]: + """ + Extract the source position of a NeXus source group. + + Parameters + ---------- + transformations: + NeXus transformation chain of the source group. + """ + return SourcePosition[RunType](transformations.compute_position()) def get_sample_position(sample: NeXusSample[RunType]) -> SamplePosition[RunType]: @@ -558,7 +575,13 @@ def _add_variances(da: sc.DataArray) -> sc.DataArray: definitions["NXmonitor"] = _StrippedMonitor -_common_providers = (gravity_vector_neg_y, file_path_to_file_spec, all_pulses) +_common_providers = ( + gravity_vector_neg_y, + file_path_to_file_spec, + all_pulses, + get_source_transformation_chain, + get_source_position, +) _monitor_providers = ( no_monitor_position_offset, @@ -568,7 +591,6 @@ def _add_variances(da: sc.DataArray) -> sc.DataArray: load_nexus_monitor, load_nexus_monitor_data, load_nexus_source, - get_source_position, get_calibrated_monitor, assemble_monitor_data, ) @@ -583,7 +605,6 @@ def _add_variances(da: sc.DataArray) -> sc.DataArray: load_nexus_detector_data, load_nexus_source, load_nexus_sample, - get_source_position, get_sample_position, get_calibrated_detector, assemble_detector_data, diff --git a/tests/nexus/workflow_test.py b/tests/nexus/workflow_test.py index 67226292..da30115c 100644 --- a/tests/nexus/workflow_test.py +++ b/tests/nexus/workflow_test.py @@ -2,6 +2,7 @@ # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) import pytest import scipp as sc +import scippnexus as snx from scipp.testing import assert_identical from ess.reduce import data @@ -27,8 +28,24 @@ def group_with_no_position(request) -> workflow.NeXusSample[SampleRun]: return workflow.NeXusSample[SampleRun](sc.DataGroup(request.param)) +@pytest.fixture() +def depends_on() -> snx.TransformationChain: + translation = snx.nxtransformations.Transform( + name='/entry/instrument/comp1/transformations/trans1', + transformation_type='translation', + value=sc.scalar(1.0, unit='m'), + vector=sc.vector(value=[1.0, 2.0, 3.0], unit=''), + depends_on=snx.DependsOn(parent='', value='.'), + offset=None, + ) + return snx.TransformationChain( + parent='/entry/instrument/comp1', + value='transformations/trans1', + transformations={translation.name: translation}, + ) + + def test_sample_position_returns_position_of_group() -> None: - depends_on = sc.spatial.translation(value=[1.0, 2.0, 3.0], unit='m') position = sc.vector([1.0, 2.0, 3.0], unit='m') sample_group = workflow.NeXusSample[SampleRun](sc.DataGroup(position=position)) assert_identical(workflow.get_sample_position(sample_group), position) @@ -42,22 +59,26 @@ def test_get_sample_position_returns_origin_if_position_not_found( ) -def test_get_source_position_returns_position_of_group() -> None: - depends_on = sc.spatial.translation(value=[1.0, 2.0, 3.0], unit='m') +def test_get_source_position_returns_position_of_group( + depends_on: snx.TransformationChain, +) -> None: position = sc.vector([1.0, 2.0, 3.0], unit='m') - source_group = workflow.NeXusSource[SampleRun](sc.DataGroup(position=position)) - assert_identical(workflow.get_source_position(source_group), position) + source_group = workflow.NeXusSource[SampleRun](sc.DataGroup(depends_on=depends_on)) + chain = workflow.get_source_transformation_chain(source_group) + assert_identical(workflow.get_source_position(chain), position) -def test_get_source_position_raises_exception_if_position_not_found( +def test_get_source_transformation_chain_raises_exception_if_position_not_found( group_with_no_position, ) -> None: - with pytest.raises(KeyError, match='position'): - workflow.get_source_position(group_with_no_position) + with pytest.raises(KeyError, match='depends_on'): + workflow.get_source_transformation_chain(group_with_no_position) @pytest.fixture() -def nexus_detector() -> workflow.NeXusDetector[SampleRun]: +def nexus_detector( + depends_on: snx.TransformationChain, +) -> workflow.NeXusDetector[SampleRun]: detector_number = sc.arange('detector_number', 6, unit=None) data = sc.DataArray( sc.empty_like(detector_number), @@ -67,11 +88,7 @@ def nexus_detector() -> workflow.NeXusDetector[SampleRun]: }, ) return workflow.NeXusDetector[SampleRun]( - sc.DataGroup( - data=data, - depends_on=sc.spatial.translation(value=[1.0, 2.0, 3.0], unit='m'), - nexus_component_name='detector1', - ) + sc.DataGroup(data=data, depends_on=depends_on, nexus_component_name='detector1') ) @@ -260,13 +277,12 @@ def test_assemble_detector_preserves_masks(calibrated_detector, detector_event_d @pytest.fixture() -def nexus_monitor() -> workflow.NeXusMonitor[SampleRun, Monitor1]: +def nexus_monitor( + depends_on: snx.TransformationChain, +) -> workflow.NeXusMonitor[SampleRun, Monitor1]: data = sc.DataArray(sc.scalar(1.2), coords={'something': sc.scalar(13)}) return workflow.NeXusMonitor[SampleRun, Monitor1]( - sc.DataGroup( - data=data, - depends_on=sc.spatial.translation(value=[1.0, 2.0, 3.0], unit='m'), - ) + sc.DataGroup(data=data, depends_on=depends_on) ) From 7c634acb67931722c10abd634fcffd0436c7f754 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 2 Oct 2024 15:37:04 +0200 Subject: [PATCH 03/19] Remove unnecessary added code --- src/ess/reduce/nexus/workflow.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/ess/reduce/nexus/workflow.py b/src/ess/reduce/nexus/workflow.py index 7694ad82..f58c4811 100644 --- a/src/ess/reduce/nexus/workflow.py +++ b/src/ess/reduce/nexus/workflow.py @@ -293,8 +293,6 @@ def load_nexus_monitor( location: Location spec for the monitor group. """ - definitions = snx.base_definitions() - definitions["NXmonitor"] = _StrippedMonitor return NeXusMonitor[RunType, MonitorType]( nexus.load_component(location, nx_class=snx.NXmonitor, definitions=definitions) ) From 61d9e9682425e00aea8ba208cbf8ac2e41158797 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 3 Oct 2024 11:16:27 +0200 Subject: [PATCH 04/19] Move source/sample position assign to separate later step Previously we introduced false dependencies of detector ids on these. --- src/ess/reduce/nexus/types.py | 4 ++++ src/ess/reduce/nexus/workflow.py | 22 +++++++++++++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/ess/reduce/nexus/types.py b/src/ess/reduce/nexus/types.py index e1377212..e1acd6cc 100644 --- a/src/ess/reduce/nexus/types.py +++ b/src/ess/reduce/nexus/types.py @@ -166,6 +166,10 @@ class CalibratedDetector(sciline.Scope[RunType, sc.DataArray], sc.DataArray): """Calibrated data from a detector.""" +class CalibratedBeamline(sciline.Scope[RunType, sc.DataArray], sc.DataArray): + """Calibrated beamline with detector and other components.""" + + class CalibratedMonitor( sciline.ScopeTwoParams[RunType, MonitorType, sc.DataArray], sc.DataArray ): diff --git a/src/ess/reduce/nexus/workflow.py b/src/ess/reduce/nexus/workflow.py index f58c4811..bd3f8a3c 100644 --- a/src/ess/reduce/nexus/workflow.py +++ b/src/ess/reduce/nexus/workflow.py @@ -13,6 +13,7 @@ from . import _nexus_loader as nexus from .types import ( + CalibratedBeamline, CalibratedDetector, CalibratedMonitor, DetectorBankSizes, @@ -388,9 +389,6 @@ def get_calibrated_detector( detector: NeXusDetector[RunType], *, offset: DetectorPositionOffset[RunType], - source_position: SourcePosition[RunType], - sample_position: SamplePosition[RunType], - gravity: GravityVector, bank_sizes: DetectorBankSizes, ) -> CalibratedDetector[RunType]: """ @@ -427,6 +425,18 @@ def get_calibrated_detector( return CalibratedDetector[RunType]( da.assign_coords( position=da.coords['position'] + offset.to(unit=da.coords['position'].unit), + ) + ) + + +def assemble_beamline( + detector: CalibratedDetector[RunType], + source_position: SourcePosition[RunType], + sample_position: SamplePosition[RunType], + gravity: GravityVector, +) -> CalibratedBeamline[RunType]: + return CalibratedBeamline[RunType]( + detector.assign_coords( source_position=source_position, sample_position=sample_position, gravity=gravity, @@ -435,10 +445,11 @@ def get_calibrated_detector( def assemble_detector_data( - detector: CalibratedDetector[RunType], event_data: NeXusDetectorData[RunType] + detector: CalibratedBeamline[RunType], + event_data: NeXusDetectorData[RunType], ) -> DetectorData[RunType]: """ - Assemble a detector data array with event data and source- and sample-position. + Assemble a detector data array with event data. Also adds variances to the event data if they are missing. @@ -605,6 +616,7 @@ def _add_variances(da: sc.DataArray) -> sc.DataArray: load_nexus_sample, get_sample_position, get_calibrated_detector, + assemble_beamline, assemble_detector_data, ) From 33a2b5f66c9b57c942c007f9fde4a430913a9a56 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 3 Oct 2024 13:32:32 +0200 Subject: [PATCH 05/19] Try to unify component handling code --- src/ess/reduce/nexus/types.py | 39 ++++++---- src/ess/reduce/nexus/workflow.py | 121 ++++++++++++------------------- 2 files changed, 70 insertions(+), 90 deletions(-) diff --git a/src/ess/reduce/nexus/types.py b/src/ess/reduce/nexus/types.py index e1acd6cc..657eebcd 100644 --- a/src/ess/reduce/nexus/types.py +++ b/src/ess/reduce/nexus/types.py @@ -62,6 +62,8 @@ VanadiumRun, ) +ComponentType = TypeVar('ComponentType', snx.NXdetector, snx.NXsample, snx.NXsource) + class TransmissionRun(Generic[ScatteringRunType]): """ @@ -116,22 +118,16 @@ class NeXusMonitorName(sciline.Scope[MonitorType, str], str): """Name of a monitor in a NeXus file.""" -class NeXusDetector(sciline.Scope[RunType, sc.DataGroup], sc.DataGroup): - """Full raw data from a NeXus detector.""" - - class NeXusMonitor( sciline.ScopeTwoParams[RunType, MonitorType, sc.DataGroup], sc.DataGroup ): """Full raw data from a NeXus monitor.""" -class NeXusSample(sciline.Scope[RunType, sc.DataGroup], sc.DataGroup): - """Raw data from a NeXus sample.""" - - -class NeXusSource(sciline.Scope[RunType, sc.DataGroup], sc.DataGroup): - """Raw data from a NeXus source.""" +class NeXusComponent( + sciline.ScopeTwoParams[ComponentType, RunType, sc.DataGroup], sc.DataGroup +): + """Raw data from a NeXus component.""" class NeXusDetectorData(sciline.Scope[RunType, sc.DataArray], sc.DataArray): @@ -144,14 +140,12 @@ class NeXusMonitorData( """Data array loaded from an NXevent_data or NXdata group within an NXmonitor.""" -class SourcePosition(sciline.Scope[RunType, sc.Variable], sc.Variable): +class ComponentPosition( + sciline.ScopeTwoParams[ComponentType, RunType, sc.Variable], sc.Variable +): """Position of the neutron source.""" -class SamplePosition(sciline.Scope[RunType, sc.Variable], sc.Variable): - """Position of the sample.""" - - class DetectorPositionOffset(sciline.Scope[RunType, sc.Variable], sc.Variable): """Offset for the detector position, added to base position.""" @@ -258,3 +252,18 @@ def from_base(cls: type[T], base: snx.TransformationChain) -> T: def compute_position(self) -> sc.Variable | sc.DataArray: return self.compute() * sc.vector([0, 0, 0], unit='m') + + +@dataclass +class NeXusTransformation(Generic[Component, RunType]): + value: sc.Variable + + @staticmethod + def from_chain( + chain: NeXusTransformationChain[Component, RunType], + # TODO can add filter options here + ) -> 'NeXusTransformation[Component, RunType]': + transform = chain.compute() + if transform.ndim == 0: + return NeXusTransformation(value=transform) + raise ValueError(f"Expected scalar transformation, got {transform}") diff --git a/src/ess/reduce/nexus/workflow.py b/src/ess/reduce/nexus/workflow.py index bd3f8a3c..0ef2998a 100644 --- a/src/ess/reduce/nexus/workflow.py +++ b/src/ess/reduce/nexus/workflow.py @@ -16,6 +16,8 @@ CalibratedBeamline, CalibratedDetector, CalibratedMonitor, + ComponentPosition, + ComponentType, DetectorBankSizes, DetectorData, DetectorPositionOffset, @@ -24,8 +26,8 @@ MonitorData, MonitorPositionOffset, MonitorType, + NeXusComponent, NeXusComponentLocationSpec, - NeXusDetector, NeXusDetectorData, NeXusDetectorDataLocationSpec, NeXusDetectorName, @@ -35,14 +37,11 @@ NeXusMonitorDataLocationSpec, NeXusMonitorLocationSpec, NeXusMonitorName, - NeXusSample, - NeXusSource, + NeXusTransformation, NeXusTransformationChain, PreopenNeXusFile, PulseSelection, RunType, - SamplePosition, - SourcePosition, ) origin = sc.vector([0, 0, 0], unit="m") @@ -79,32 +78,18 @@ def gravity_vector_neg_y() -> GravityVector: return GravityVector(sc.vector(value=[0, -1, 0]) * g) -def unique_sample_spec( +def unique_component_spec( filename: NeXusFileSpec[RunType], -) -> NeXusComponentLocationSpec[snx.NXsample, RunType]: +) -> NeXusComponentLocationSpec[ComponentType, RunType]: """ - Create a location spec for a unique sample group in a NeXus file. + Create a location spec for a unique component group in a NeXus file. Parameters ---------- filename: NeXus file to use for the location spec. """ - return NeXusComponentLocationSpec[snx.NXsample, RunType](filename=filename.value) - - -def unique_source_spec( - filename: NeXusFileSpec[RunType], -) -> NeXusComponentLocationSpec[snx.NXsource, RunType]: - """ - Create a location spec for a unique source group in a NeXus file. - - Parameters - ---------- - filename: - NeXus file to use for the location spec. - """ - return NeXusComponentLocationSpec[snx.NXsource, RunType](filename=filename.value) + return NeXusComponentLocationSpec[ComponentType, RunType](filename=filename.value) def monitor_by_name( @@ -189,7 +174,7 @@ def detector_data_by_name( def load_nexus_sample( location: NeXusComponentLocationSpec[snx.NXsample, RunType], -) -> NeXusSample[RunType]: +) -> NeXusComponent[snx.NXsample, RunType]: """ Load a NeXus sample group from a file. @@ -206,13 +191,13 @@ def load_nexus_sample( try: dg = nexus.load_component(location, nx_class=snx.NXsample) except ValueError: - dg = sc.DataGroup() - return NeXusSample[RunType](dg) + dg = sc.DataGroup(depends_on=snx.TransformationChain(parent='', value='.')) + return NeXusComponent[snx.NXsample, RunType](dg) def load_nexus_source( location: NeXusComponentLocationSpec[snx.NXsource, RunType], -) -> NeXusSource[RunType]: +) -> NeXusComponent[snx.NXsource, RunType]: """ Load a NeXus source group from a file. @@ -221,12 +206,14 @@ def load_nexus_source( location: Location spec for the source group. """ - return NeXusSource[RunType](nexus.load_component(location, nx_class=snx.NXsource)) + return NeXusComponent[snx.NXsource, RunType]( + nexus.load_component(location, nx_class=snx.NXsource) + ) def load_nexus_detector( location: NeXusComponentLocationSpec[snx.NXdetector, RunType], -) -> NeXusDetector[RunType]: +) -> NeXusComponent[snx.NXdetector, RunType]: """ Load detector from NeXus, but with event data replaced by placeholders. @@ -258,7 +245,7 @@ def load_nexus_detector( # The selection is only used for selecting a range of event data. location = replace(location, selection=()) - return NeXusDetector[RunType]( + return NeXusComponent[snx.NXdetector, RunType]( nexus.load_component(location, nx_class=snx.NXdetector, definitions=definitions) ) @@ -341,53 +328,37 @@ def load_nexus_monitor_data( ) -def get_source_transformation_chain( - source: NeXusSource[RunType], -) -> NeXusTransformationChain[snx.NXsource, RunType]: - """ - Extract the transformation chain from a NeXus source group. - - Parameters - ---------- - source: - NeXus source group. - """ - chain = source['depends_on'] - return NeXusTransformationChain[snx.NXsource, RunType].from_base(chain) - - -def get_source_position( - transformations: NeXusTransformationChain[snx.NXsource, RunType], -) -> SourcePosition[RunType]: +def get_transformation_chain( + detector: NeXusComponent[ComponentType, RunType], +) -> NeXusTransformationChain[ComponentType, RunType]: """ - Extract the source position of a NeXus source group. + Extract the transformation chain from a NeXus detector group. Parameters ---------- - transformations: - NeXus transformation chain of the source group. + detector: + NeXus detector group. """ - return SourcePosition[RunType](transformations.compute_position()) + chain = detector['depends_on'] + return NeXusTransformationChain[ComponentType, RunType].from_base(chain) -def get_sample_position(sample: NeXusSample[RunType]) -> SamplePosition[RunType]: - """ - Extract the sample position from a NeXus sample group. +def to_transformation( + chain: NeXusTransformationChain[ComponentType, RunType], +) -> NeXusTransformation[ComponentType, RunType]: + return NeXusTransformation[ComponentType, RunType].from_chain(chain) - Defaults to the origin if the sample group does not contain a position field. - Parameters - ---------- - sample: - NeXus sample group. - """ - dg = nexus.compute_component_position(sample) - return SamplePosition[RunType](dg.get("position", origin)) +def compute_position( + transformation: NeXusTransformation[ComponentType, RunType], +) -> ComponentPosition[ComponentType, RunType]: + return ComponentPosition[ComponentType, RunType](transformation.value * origin) def get_calibrated_detector( - detector: NeXusDetector[RunType], + detector: NeXusComponent[snx.NXdetector, RunType], *, + transform: NeXusTransformation[snx.NXdetector, RunType], offset: DetectorPositionOffset[RunType], bank_sizes: DetectorBankSizes, ) -> CalibratedDetector[RunType]: @@ -414,7 +385,7 @@ def get_calibrated_detector( bank_sizes: Dictionary of detector bank sizes. """ - detector = nexus.compute_component_position(detector) + # detector = nexus.compute_component_position(detector) da = nexus.extract_signal_data_array(detector) if ( sizes := (bank_sizes or {}).get(detector.get('nexus_component_name')) @@ -422,17 +393,19 @@ def get_calibrated_detector( da = da.fold(dim="detector_number", sizes=sizes) # Note: We apply offset as early as possible, i.e., right in this function # the detector array from the raw loader NeXus group, to prevent a source of bugs. + base_pos = snx.zip_pixel_offsets(da.coords) + position = transform.value * base_pos return CalibratedDetector[RunType]( da.assign_coords( - position=da.coords['position'] + offset.to(unit=da.coords['position'].unit), + position=position + offset.to(unit=position.unit), ) ) def assemble_beamline( detector: CalibratedDetector[RunType], - source_position: SourcePosition[RunType], - sample_position: SamplePosition[RunType], + source_position: ComponentPosition[snx.NXsource, RunType], + sample_position: ComponentPosition[snx.NXsample, RunType], gravity: GravityVector, ) -> CalibratedBeamline[RunType]: return CalibratedBeamline[RunType]( @@ -473,7 +446,7 @@ def assemble_detector_data( def get_calibrated_monitor( monitor: NeXusMonitor[RunType, MonitorType], offset: MonitorPositionOffset[RunType, MonitorType], - source_position: SourcePosition[RunType], + source_position: ComponentPosition[snx.NXsource, RunType], ) -> CalibratedMonitor[RunType, MonitorType]: """ Extract the data array corresponding to a monitor's signal field. @@ -588,13 +561,14 @@ def _add_variances(da: sc.DataArray) -> sc.DataArray: gravity_vector_neg_y, file_path_to_file_spec, all_pulses, - get_source_transformation_chain, - get_source_position, + unique_component_spec, + get_transformation_chain, + to_transformation, + compute_position, ) _monitor_providers = ( no_monitor_position_offset, - unique_source_spec, monitor_by_name, monitor_data_by_name, load_nexus_monitor, @@ -606,15 +580,12 @@ def _add_variances(da: sc.DataArray) -> sc.DataArray: _detector_providers = ( no_detector_position_offset, - unique_source_spec, - unique_sample_spec, detector_by_name, detector_data_by_name, load_nexus_detector, load_nexus_detector_data, load_nexus_source, load_nexus_sample, - get_sample_position, get_calibrated_detector, assemble_beamline, assemble_detector_data, From bb3837c754ce9629d0f1a5355366c3712c7a36f2 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 3 Oct 2024 14:04:47 +0200 Subject: [PATCH 06/19] Unify some more code --- src/ess/reduce/nexus/types.py | 68 ++++++++++++++----------------- src/ess/reduce/nexus/workflow.py | 69 ++++++++++---------------------- 2 files changed, 51 insertions(+), 86 deletions(-) diff --git a/src/ess/reduce/nexus/types.py b/src/ess/reduce/nexus/types.py index 657eebcd..7bb48f59 100644 --- a/src/ess/reduce/nexus/types.py +++ b/src/ess/reduce/nexus/types.py @@ -62,8 +62,6 @@ VanadiumRun, ) -ComponentType = TypeVar('ComponentType', snx.NXdetector, snx.NXsample, snx.NXsource) - class TransmissionRun(Generic[ScatteringRunType]): """ @@ -113,37 +111,45 @@ class TransmissionRun(Generic[ScatteringRunType]): ) """TypeVar used for specifying the monitor type such as Incident or Transmission""" +ComponentType = TypeVar( + 'ComponentType', + snx.NXdetector, + snx.NXsample, + snx.NXsource, + Monitor1, + Monitor2, + Monitor3, + Monitor4, + Monitor5, + Incident, + Transmission, +) + class NeXusMonitorName(sciline.Scope[MonitorType, str], str): """Name of a monitor in a NeXus file.""" -class NeXusMonitor( - sciline.ScopeTwoParams[RunType, MonitorType, sc.DataGroup], sc.DataGroup -): - """Full raw data from a NeXus monitor.""" - - class NeXusComponent( sciline.ScopeTwoParams[ComponentType, RunType, sc.DataGroup], sc.DataGroup ): """Raw data from a NeXus component.""" -class NeXusDetectorData(sciline.Scope[RunType, sc.DataArray], sc.DataArray): - """Data array loaded from an NXevent_data or NXdata group within an NXdetector.""" - - -class NeXusMonitorData( - sciline.ScopeTwoParams[RunType, MonitorType, sc.DataArray], sc.DataArray +class NeXusData( + sciline.ScopeTwoParams[ComponentType, RunType, sc.DataArray], sc.DataArray ): - """Data array loaded from an NXevent_data or NXdata group within an NXmonitor.""" + """ + Data array loaded from an NXevent_data or NXdata group. + + This must be contained in an NXmonitor or NXdetector group. + """ class ComponentPosition( sciline.ScopeTwoParams[ComponentType, RunType, sc.Variable], sc.Variable ): - """Position of the neutron source.""" + """Position of a component such as source, sample, monitor, or detector.""" class DetectorPositionOffset(sciline.Scope[RunType, sc.Variable], sc.Variable): @@ -215,33 +221,17 @@ class NeXusComponentLocationSpec(NeXusLocationSpec, Generic[Component, RunType]) @dataclass -class NeXusMonitorLocationSpec( - NeXusComponentLocationSpec[snx.NXmonitor, RunType], Generic[RunType, MonitorType] -): - """ - NeXus filename and optional parameters to identify (parts of) a monitor to load. - """ - - -@dataclass -class NeXusDetectorDataLocationSpec( - NeXusComponentLocationSpec[snx.NXevent_data, RunType], Generic[RunType] -): +class NeXusDataLocationSpec(NeXusLocationSpec, Generic[ComponentType, RunType]): """NeXus filename and parameters to identify (parts of) detector data to load.""" -@dataclass -class NeXusMonitorDataLocationSpec( - NeXusComponentLocationSpec[snx.NXevent_data, RunType], Generic[RunType, MonitorType] -): - """NeXus filename and parameters to identify (parts of) monitor data to load.""" - - T = TypeVar('T', bound='NeXusTransformationChain') @dataclass -class NeXusTransformationChain(snx.TransformationChain, Generic[Component, RunType]): +class NeXusTransformationChain( + snx.TransformationChain, Generic[ComponentType, RunType] +): @classmethod def from_base(cls: type[T], base: snx.TransformationChain) -> T: return cls( @@ -255,14 +245,14 @@ def compute_position(self) -> sc.Variable | sc.DataArray: @dataclass -class NeXusTransformation(Generic[Component, RunType]): +class NeXusTransformation(Generic[ComponentType, RunType]): value: sc.Variable @staticmethod def from_chain( - chain: NeXusTransformationChain[Component, RunType], + chain: NeXusTransformationChain[ComponentType, RunType], # TODO can add filter options here - ) -> 'NeXusTransformation[Component, RunType]': + ) -> 'NeXusTransformation[ComponentType, RunType]': transform = chain.compute() if transform.ndim == 0: return NeXusTransformation(value=transform) diff --git a/src/ess/reduce/nexus/workflow.py b/src/ess/reduce/nexus/workflow.py index 0ef2998a..40f849fb 100644 --- a/src/ess/reduce/nexus/workflow.py +++ b/src/ess/reduce/nexus/workflow.py @@ -28,14 +28,10 @@ MonitorType, NeXusComponent, NeXusComponentLocationSpec, - NeXusDetectorData, - NeXusDetectorDataLocationSpec, + NeXusData, + NeXusDataLocationSpec, NeXusDetectorName, NeXusFileSpec, - NeXusMonitor, - NeXusMonitorData, - NeXusMonitorDataLocationSpec, - NeXusMonitorLocationSpec, NeXusMonitorName, NeXusTransformation, NeXusTransformationChain, @@ -94,7 +90,7 @@ def unique_component_spec( def monitor_by_name( filename: NeXusFileSpec[RunType], name: NeXusMonitorName[MonitorType] -) -> NeXusMonitorLocationSpec[RunType, MonitorType]: +) -> NeXusComponentLocationSpec[MonitorType, RunType]: """ Create a location spec for a monitor group in a NeXus file. @@ -105,7 +101,7 @@ def monitor_by_name( name: Name of the monitor group. """ - return NeXusMonitorLocationSpec[RunType, MonitorType]( + return NeXusComponentLocationSpec[MonitorType, RunType]( filename=filename.value, component_name=name ) @@ -114,7 +110,7 @@ def monitor_data_by_name( filename: NeXusFileSpec[RunType], name: NeXusMonitorName[MonitorType], selection: PulseSelection[RunType], -) -> NeXusMonitorDataLocationSpec[RunType, MonitorType]: +) -> NeXusDataLocationSpec[MonitorType, RunType]: """ Create a location spec for monitor data in a NeXus file. @@ -127,7 +123,7 @@ def monitor_data_by_name( selection: Selection (start and stop as a Python slice object) for the monitor data. """ - return NeXusMonitorDataLocationSpec[RunType, MonitorType]( + return NeXusDataLocationSpec[MonitorType, RunType]( filename=filename.value, component_name=name, selection=selection.value ) @@ -154,7 +150,7 @@ def detector_data_by_name( filename: NeXusFileSpec[RunType], name: NeXusDetectorName, selection: PulseSelection[RunType], -) -> NeXusDetectorDataLocationSpec[RunType]: +) -> NeXusDataLocationSpec[snx.NXdetector, RunType]: """ Create a location spec for detector data in a NeXus file. @@ -167,7 +163,7 @@ def detector_data_by_name( selection: Selection (start and stop as a Python slice object) for the detector data. """ - return NeXusDetectorDataLocationSpec[RunType]( + return NeXusDataLocationSpec[snx.NXdetector, RunType]( filename=filename.value, component_name=name, selection=selection.value ) @@ -251,8 +247,8 @@ def load_nexus_detector( def load_nexus_monitor( - location: NeXusMonitorLocationSpec[RunType, MonitorType], -) -> NeXusMonitor[RunType, MonitorType]: + location: NeXusComponentLocationSpec[MonitorType, RunType], +) -> NeXusComponent[MonitorType, RunType]: """ Load monitor from NeXus, but with event data replaced by placeholders. @@ -270,7 +266,7 @@ def load_nexus_monitor( Loading thus proceeds in three steps: 1. This function loads the monitor, but replaces the event data with placeholders. - 2. :py:func:`get_calbirated_monitor` drops the additional information, returning + 2. :py:func:`get_calibrated_monitor` drops the additional information, returning only the contained scipp.DataArray. This will generally contain coordinates as well as pixel masks. 3. :py:func:`assemble_monitor_data` replaces placeholder data values with the @@ -281,14 +277,14 @@ def load_nexus_monitor( location: Location spec for the monitor group. """ - return NeXusMonitor[RunType, MonitorType]( + return NeXusComponent[MonitorType, RunType]( nexus.load_component(location, nx_class=snx.NXmonitor, definitions=definitions) ) -def load_nexus_detector_data( - location: NeXusDetectorDataLocationSpec[RunType], -) -> NeXusDetectorData[RunType]: +def load_nexus_data( + location: NeXusDataLocationSpec[ComponentType, RunType], +) -> NeXusData[ComponentType, RunType]: """ Load event or histogram data from a NeXus detector group. @@ -297,28 +293,7 @@ def load_nexus_detector_data( location: Location spec for the detector group. """ - return NeXusDetectorData[RunType]( - nexus.load_data( - file_path=location.filename, - entry_name=location.entry_name, - selection=location.selection, - component_name=location.component_name, - ) - ) - - -def load_nexus_monitor_data( - location: NeXusMonitorDataLocationSpec[RunType, MonitorType], -) -> NeXusMonitorData[RunType, MonitorType]: - """ - Load event or histogram data from a NeXus monitor group. - - Parameters - ---------- - location: - Location spec for the monitor group. - """ - return NeXusMonitorData[RunType, MonitorType]( + return NeXusData[ComponentType, RunType]( nexus.load_data( file_path=location.filename, entry_name=location.entry_name, @@ -346,12 +321,14 @@ def get_transformation_chain( def to_transformation( chain: NeXusTransformationChain[ComponentType, RunType], ) -> NeXusTransformation[ComponentType, RunType]: + """Convert transformation chain into a single transformation matrix.""" return NeXusTransformation[ComponentType, RunType].from_chain(chain) def compute_position( transformation: NeXusTransformation[ComponentType, RunType], ) -> ComponentPosition[ComponentType, RunType]: + """Compute the position of a component from a transformation matrix.""" return ComponentPosition[ComponentType, RunType](transformation.value * origin) @@ -385,7 +362,6 @@ def get_calibrated_detector( bank_sizes: Dictionary of detector bank sizes. """ - # detector = nexus.compute_component_position(detector) da = nexus.extract_signal_data_array(detector) if ( sizes := (bank_sizes or {}).get(detector.get('nexus_component_name')) @@ -419,7 +395,7 @@ def assemble_beamline( def assemble_detector_data( detector: CalibratedBeamline[RunType], - event_data: NeXusDetectorData[RunType], + event_data: NeXusData[snx.NXdetector, RunType], ) -> DetectorData[RunType]: """ Assemble a detector data array with event data. @@ -444,7 +420,7 @@ def assemble_detector_data( def get_calibrated_monitor( - monitor: NeXusMonitor[RunType, MonitorType], + monitor: NeXusComponent[MonitorType, RunType], offset: MonitorPositionOffset[RunType, MonitorType], source_position: ComponentPosition[snx.NXsource, RunType], ) -> CalibratedMonitor[RunType, MonitorType]: @@ -474,7 +450,7 @@ def get_calibrated_monitor( def assemble_monitor_data( monitor: CalibratedMonitor[RunType, MonitorType], - data: NeXusMonitorData[RunType, MonitorType], + data: NeXusData[MonitorType, RunType], ) -> MonitorData[RunType, MonitorType]: """ Assemble a monitor data array with event data. @@ -565,6 +541,7 @@ def _add_variances(da: sc.DataArray) -> sc.DataArray: get_transformation_chain, to_transformation, compute_position, + load_nexus_data, ) _monitor_providers = ( @@ -572,7 +549,6 @@ def _add_variances(da: sc.DataArray) -> sc.DataArray: monitor_by_name, monitor_data_by_name, load_nexus_monitor, - load_nexus_monitor_data, load_nexus_source, get_calibrated_monitor, assemble_monitor_data, @@ -583,7 +559,6 @@ def _add_variances(da: sc.DataArray) -> sc.DataArray: detector_by_name, detector_data_by_name, load_nexus_detector, - load_nexus_detector_data, load_nexus_source, load_nexus_sample, get_calibrated_detector, From ab4ffd9300d5ecf84a10e2277e96a16b2d04e925 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 3 Oct 2024 14:11:46 +0200 Subject: [PATCH 07/19] More unification --- src/ess/reduce/nexus/types.py | 10 ++++--- src/ess/reduce/nexus/workflow.py | 46 ++++++++------------------------ 2 files changed, 17 insertions(+), 39 deletions(-) diff --git a/src/ess/reduce/nexus/types.py b/src/ess/reduce/nexus/types.py index 7bb48f59..7ff999fb 100644 --- a/src/ess/reduce/nexus/types.py +++ b/src/ess/reduce/nexus/types.py @@ -22,8 +22,6 @@ NeXusGroup = NewType('NeXusGroup', snx.Group) """A ScippNexus group in an open file.""" -NeXusDetectorName = NewType('NeXusDetectorName', str) -"""Name of a detector (bank) in a NeXus file.""" NeXusEntryName = NewType('NeXusEntryName', str) """Name of an entry in a NeXus file.""" NeXusSourceName = NewType('NeXusSourceName', str) @@ -126,8 +124,12 @@ class TransmissionRun(Generic[ScatteringRunType]): ) -class NeXusMonitorName(sciline.Scope[MonitorType, str], str): - """Name of a monitor in a NeXus file.""" +class NeXusComponentName(sciline.Scope[ComponentType, str], str): + """Name of a monitor or detector component in a NeXus file.""" + + +NeXusDetectorName = NeXusComponentName[snx.NXdetector] +"""Name of a detector (bank) in a NeXus file.""" class NeXusComponent( diff --git a/src/ess/reduce/nexus/workflow.py b/src/ess/reduce/nexus/workflow.py index 40f849fb..c0049505 100644 --- a/src/ess/reduce/nexus/workflow.py +++ b/src/ess/reduce/nexus/workflow.py @@ -28,11 +28,10 @@ MonitorType, NeXusComponent, NeXusComponentLocationSpec, + NeXusComponentName, NeXusData, NeXusDataLocationSpec, - NeXusDetectorName, NeXusFileSpec, - NeXusMonitorName, NeXusTransformation, NeXusTransformationChain, PreopenNeXusFile, @@ -89,7 +88,7 @@ def unique_component_spec( def monitor_by_name( - filename: NeXusFileSpec[RunType], name: NeXusMonitorName[MonitorType] + filename: NeXusFileSpec[RunType], name: NeXusComponentName[MonitorType] ) -> NeXusComponentLocationSpec[MonitorType, RunType]: """ Create a location spec for a monitor group in a NeXus file. @@ -106,30 +105,8 @@ def monitor_by_name( ) -def monitor_data_by_name( - filename: NeXusFileSpec[RunType], - name: NeXusMonitorName[MonitorType], - selection: PulseSelection[RunType], -) -> NeXusDataLocationSpec[MonitorType, RunType]: - """ - Create a location spec for monitor data in a NeXus file. - - Parameters - ---------- - filename: - NeXus file to use for the location spec. - name: - Name of the monitor group. - selection: - Selection (start and stop as a Python slice object) for the monitor data. - """ - return NeXusDataLocationSpec[MonitorType, RunType]( - filename=filename.value, component_name=name, selection=selection.value - ) - - def detector_by_name( - filename: NeXusFileSpec[RunType], name: NeXusDetectorName + filename: NeXusFileSpec[RunType], name: NeXusComponentName[snx.NXdetector] ) -> NeXusComponentLocationSpec[snx.NXdetector, RunType]: """ Create a location spec for a detector group in a NeXus file. @@ -146,24 +123,24 @@ def detector_by_name( ) -def detector_data_by_name( +def data_by_name( filename: NeXusFileSpec[RunType], - name: NeXusDetectorName, + name: NeXusComponentName[ComponentType], selection: PulseSelection[RunType], -) -> NeXusDataLocationSpec[snx.NXdetector, RunType]: +) -> NeXusDataLocationSpec[ComponentType, RunType]: """ - Create a location spec for detector data in a NeXus file. + Create a location spec for monitor or detector data in a NeXus file. Parameters ---------- filename: NeXus file to use for the location spec. name: - Name of the detector group. + Name of the monitor or detector group. selection: - Selection (start and stop as a Python slice object) for the detector data. + Time range (start and stop as a Python slice object). """ - return NeXusDataLocationSpec[snx.NXdetector, RunType]( + return NeXusDataLocationSpec[ComponentType, RunType]( filename=filename.value, component_name=name, selection=selection.value ) @@ -542,12 +519,12 @@ def _add_variances(da: sc.DataArray) -> sc.DataArray: to_transformation, compute_position, load_nexus_data, + data_by_name, ) _monitor_providers = ( no_monitor_position_offset, monitor_by_name, - monitor_data_by_name, load_nexus_monitor, load_nexus_source, get_calibrated_monitor, @@ -557,7 +534,6 @@ def _add_variances(da: sc.DataArray) -> sc.DataArray: _detector_providers = ( no_detector_position_offset, detector_by_name, - detector_data_by_name, load_nexus_detector, load_nexus_source, load_nexus_sample, From 4ab8a2366490224910bcb0195a3e11fe409ddf41 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 3 Oct 2024 14:25:47 +0200 Subject: [PATCH 08/19] Drop selection also for monitor --- src/ess/reduce/nexus/workflow.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ess/reduce/nexus/workflow.py b/src/ess/reduce/nexus/workflow.py index c0049505..6c121cdb 100644 --- a/src/ess/reduce/nexus/workflow.py +++ b/src/ess/reduce/nexus/workflow.py @@ -254,6 +254,9 @@ def load_nexus_monitor( location: Location spec for the monitor group. """ + # The selection is only used for selecting a range of event data. + location = replace(location, selection=()) + return NeXusComponent[MonitorType, RunType]( nexus.load_component(location, nx_class=snx.NXmonitor, definitions=definitions) ) From 99bde33be686a1693f2d062274aa51ba4113c7e1 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 3 Oct 2024 15:00:16 +0200 Subject: [PATCH 09/19] Unify even more --- src/ess/reduce/nexus/types.py | 6 ++ src/ess/reduce/nexus/workflow.py | 164 +++++++++++-------------------- 2 files changed, 62 insertions(+), 108 deletions(-) diff --git a/src/ess/reduce/nexus/types.py b/src/ess/reduce/nexus/types.py index 7ff999fb..5de34455 100644 --- a/src/ess/reduce/nexus/types.py +++ b/src/ess/reduce/nexus/types.py @@ -122,12 +122,18 @@ class TransmissionRun(Generic[ScatteringRunType]): Incident, Transmission, ) +UniqueComponentType = TypeVar('UniqueComponentType', snx.NXsample, snx.NXsource) +"""Components that can be identified by their type as there will only be one.""" class NeXusComponentName(sciline.Scope[ComponentType, str], str): """Name of a monitor or detector component in a NeXus file.""" +class NeXusClassName(sciline.Scope[ComponentType, str], str): + """NX_class of a component in a NeXus file.""" + + NeXusDetectorName = NeXusComponentName[snx.NXdetector] """Name of a detector (bank) in a NeXus file.""" diff --git a/src/ess/reduce/nexus/workflow.py b/src/ess/reduce/nexus/workflow.py index 6c121cdb..464e5882 100644 --- a/src/ess/reduce/nexus/workflow.py +++ b/src/ess/reduce/nexus/workflow.py @@ -3,7 +3,6 @@ """Workflow and workflow components for interacting with NeXus files.""" -from dataclasses import replace from typing import Any import sciline @@ -26,6 +25,7 @@ MonitorData, MonitorPositionOffset, MonitorType, + NeXusClassName, NeXusComponent, NeXusComponentLocationSpec, NeXusComponentName, @@ -37,6 +37,7 @@ PreopenNeXusFile, PulseSelection, RunType, + UniqueComponentType, ) origin = sc.vector([0, 0, 0], unit="m") @@ -73,53 +74,37 @@ def gravity_vector_neg_y() -> GravityVector: return GravityVector(sc.vector(value=[0, -1, 0]) * g) -def unique_component_spec( - filename: NeXusFileSpec[RunType], +def component_spec_by_name( + filename: NeXusFileSpec[RunType], name: NeXusComponentName[ComponentType] ) -> NeXusComponentLocationSpec[ComponentType, RunType]: """ - Create a location spec for a unique component group in a NeXus file. - - Parameters - ---------- - filename: - NeXus file to use for the location spec. - """ - return NeXusComponentLocationSpec[ComponentType, RunType](filename=filename.value) - - -def monitor_by_name( - filename: NeXusFileSpec[RunType], name: NeXusComponentName[MonitorType] -) -> NeXusComponentLocationSpec[MonitorType, RunType]: - """ - Create a location spec for a monitor group in a NeXus file. + Create a location spec for a component group in a NeXus file. Parameters ---------- filename: NeXus file to use for the location spec. name: - Name of the monitor group. + Name of the component group. """ - return NeXusComponentLocationSpec[MonitorType, RunType]( + return NeXusComponentLocationSpec[ComponentType, RunType]( filename=filename.value, component_name=name ) -def detector_by_name( - filename: NeXusFileSpec[RunType], name: NeXusComponentName[snx.NXdetector] -) -> NeXusComponentLocationSpec[snx.NXdetector, RunType]: +def unique_component_spec( + filename: NeXusFileSpec[RunType], +) -> NeXusComponentLocationSpec[UniqueComponentType, RunType]: """ - Create a location spec for a detector group in a NeXus file. + Create a location spec for a unique component group in a NeXus file. Parameters ---------- filename: NeXus file to use for the location spec. - name: - Name of the detector group. """ - return NeXusComponentLocationSpec[snx.NXdetector, RunType]( - filename=filename.value, component_name=name + return NeXusComponentLocationSpec[UniqueComponentType, RunType]( + filename=filename.value ) @@ -168,97 +153,52 @@ def load_nexus_sample( return NeXusComponent[snx.NXsample, RunType](dg) -def load_nexus_source( - location: NeXusComponentLocationSpec[snx.NXsource, RunType], -) -> NeXusComponent[snx.NXsource, RunType]: - """ - Load a NeXus source group from a file. +def nx_class_for_monitor() -> NeXusClassName[MonitorType]: + return NeXusClassName[MonitorType](snx.NXmonitor) - Parameters - ---------- - location: - Location spec for the source group. - """ - return NeXusComponent[snx.NXsource, RunType]( - nexus.load_component(location, nx_class=snx.NXsource) - ) +def nx_class_for_detector() -> NeXusClassName[snx.NXdetector]: + return NeXusClassName[snx.NXdetector](snx.NXdetector) -def load_nexus_detector( - location: NeXusComponentLocationSpec[snx.NXdetector, RunType], -) -> NeXusComponent[snx.NXdetector, RunType]: - """ - Load detector from NeXus, but with event data replaced by placeholders. - As the event data can be large and is not needed at this stage, it is replaced by - a placeholder. A placeholder is used to allow for returning a scipp.DataArray, which - is what most downstream code will expect. - Currently the placeholder is the detector number, but this may change in the future. +def nx_class_for_source() -> NeXusClassName[snx.NXsource]: + return NeXusClassName[snx.NXsource](snx.NXsource) - The returned object is a scipp.DataGroup, as it may contain additional information - about the detector that cannot be represented as a single scipp.DataArray. Most - downstream code will only be interested in the contained scipp.DataArray so this - needs to be extracted. However, other processing steps may require the additional - information, so it is kept in the DataGroup. - Loading thus proceeds in three steps: +def nx_class_for_sample() -> NeXusClassName[snx.NXsample]: + return NeXusClassName[snx.NXsample](snx.NXsample) - 1. This function loads the detector, but replaces the event data with placeholders. - 2. :py:func:`get_calibrated_detector` drops the additional information, returning - only the contained scipp.DataArray, reshaped to the logical detector shape. - This will generally contain coordinates as well as pixel masks. - 3. :py:func:`assemble_detector_data` replaces placeholder data values with the - event data, and adds source and sample positions. - Parameters - ---------- - location: - Location spec for the detector group. +def load_nexus_component( + location: NeXusComponentLocationSpec[ComponentType, RunType], + nx_class: NeXusClassName[ComponentType], +) -> NeXusComponent[ComponentType, RunType]: """ - # The selection is only used for selecting a range of event data. - location = replace(location, selection=()) + Load a NeXus component group from a file. - return NeXusComponent[snx.NXdetector, RunType]( - nexus.load_component(location, nx_class=snx.NXdetector, definitions=definitions) - ) - - -def load_nexus_monitor( - location: NeXusComponentLocationSpec[MonitorType, RunType], -) -> NeXusComponent[MonitorType, RunType]: - """ - Load monitor from NeXus, but with event data replaced by placeholders. + When loading a detector or monitor, event data is replaced by placeholders. As the event data can be large and is not needed at this stage, it is replaced by a placeholder. A placeholder is used to allow for returning a scipp.DataArray, which is what most downstream code will expect. - Currently the placeholder is a size-0 array, but this may change in the future. + Currently the placeholder is the detector number (for detectors) or a size-0 array + (for monitors), but this may change in the future. The returned object is a scipp.DataGroup, as it may contain additional information - about the monitor that cannot be represented as a single scipp.DataArray. Most + about the detector that cannot be represented as a single scipp.DataArray. Most downstream code will only be interested in the contained scipp.DataArray so this needs to be extracted. However, other processing steps may require the additional information, so it is kept in the DataGroup. - Loading thus proceeds in three steps: - - 1. This function loads the monitor, but replaces the event data with placeholders. - 2. :py:func:`get_calibrated_monitor` drops the additional information, returning - only the contained scipp.DataArray. - This will generally contain coordinates as well as pixel masks. - 3. :py:func:`assemble_monitor_data` replaces placeholder data values with the - event data, and adds source and sample positions. - Parameters ---------- location: - Location spec for the monitor group. + Location spec for the source group. + nx_class: + NX_class to identify the component. """ - # The selection is only used for selecting a range of event data. - location = replace(location, selection=()) - - return NeXusComponent[MonitorType, RunType]( - nexus.load_component(location, nx_class=snx.NXmonitor, definitions=definitions) + return NeXusComponent[ComponentType, RunType]( + nexus.load_component(location, nx_class=nx_class, definitions=definitions) ) @@ -333,12 +273,6 @@ def get_calibrated_detector( NeXus detector group. offset: Offset to add to the detector position. - source_position: - Position of the neutron source. - sample_position: - Position of the sample. - gravity: - Gravity vector. bank_sizes: Dictionary of detector bank sizes. """ @@ -364,6 +298,20 @@ def assemble_beamline( sample_position: ComponentPosition[snx.NXsample, RunType], gravity: GravityVector, ) -> CalibratedBeamline[RunType]: + """ + Add beamline information (gravity vector, source- and sample-position) to detector. + + Parameters + ---------- + detector: + NeXus detector group. + source_position: + Position of the neutron source. + sample_position: + Position of the sample. + gravity: + Gravity vector. + """ return CalibratedBeamline[RunType]( detector.assign_coords( source_position=source_position, @@ -517,28 +465,28 @@ def _add_variances(da: sc.DataArray) -> sc.DataArray: gravity_vector_neg_y, file_path_to_file_spec, all_pulses, - unique_component_spec, + component_spec_by_name, + unique_component_spec, # after component_spec_by_name, partially overrides get_transformation_chain, to_transformation, compute_position, load_nexus_data, + load_nexus_component, data_by_name, + nx_class_for_detector, + nx_class_for_monitor, + nx_class_for_source, + nx_class_for_sample, ) _monitor_providers = ( no_monitor_position_offset, - monitor_by_name, - load_nexus_monitor, - load_nexus_source, get_calibrated_monitor, assemble_monitor_data, ) _detector_providers = ( no_detector_position_offset, - detector_by_name, - load_nexus_detector, - load_nexus_source, load_nexus_sample, get_calibrated_detector, assemble_beamline, From c7181c5f617ee96fb05b960e80ef762f431df1a5 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 3 Oct 2024 15:01:59 +0200 Subject: [PATCH 10/19] Cleanup --- src/ess/reduce/nexus/workflow.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/ess/reduce/nexus/workflow.py b/src/ess/reduce/nexus/workflow.py index 464e5882..a3c40d08 100644 --- a/src/ess/reduce/nexus/workflow.py +++ b/src/ess/reduce/nexus/workflow.py @@ -283,12 +283,9 @@ def get_calibrated_detector( da = da.fold(dim="detector_number", sizes=sizes) # Note: We apply offset as early as possible, i.e., right in this function # the detector array from the raw loader NeXus group, to prevent a source of bugs. - base_pos = snx.zip_pixel_offsets(da.coords) - position = transform.value * base_pos + position = transform.value * snx.zip_pixel_offsets(da.coords) return CalibratedDetector[RunType]( - da.assign_coords( - position=position + offset.to(unit=position.unit), - ) + da.assign_coords(position=position + offset.to(unit=position.unit)) ) From 0f68b0c2f58ebb2d46aee65cfa56c66b31f0d6a6 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 3 Oct 2024 15:08:11 +0200 Subject: [PATCH 11/19] Rename TypeVar ComponentType --- src/ess/reduce/nexus/types.py | 30 +++++++++------------ src/ess/reduce/nexus/workflow.py | 46 ++++++++++++++++---------------- 2 files changed, 35 insertions(+), 41 deletions(-) diff --git a/src/ess/reduce/nexus/types.py b/src/ess/reduce/nexus/types.py index 5de34455..ace5d98e 100644 --- a/src/ess/reduce/nexus/types.py +++ b/src/ess/reduce/nexus/types.py @@ -31,8 +31,6 @@ GravityVector = NewType('GravityVector', sc.Variable) -Component = TypeVar('Component', bound=snx.NXobject) - PreopenNeXusFile = NewType('PreopenNeXusFile', bool) """Whether to preopen NeXus files before passing them to the rest of the workflow.""" @@ -109,8 +107,8 @@ class TransmissionRun(Generic[ScatteringRunType]): ) """TypeVar used for specifying the monitor type such as Incident or Transmission""" -ComponentType = TypeVar( - 'ComponentType', +Component = TypeVar( + 'Component', snx.NXdetector, snx.NXsample, snx.NXsource, @@ -126,11 +124,11 @@ class TransmissionRun(Generic[ScatteringRunType]): """Components that can be identified by their type as there will only be one.""" -class NeXusComponentName(sciline.Scope[ComponentType, str], str): +class NeXusComponentName(sciline.Scope[Component, str], str): """Name of a monitor or detector component in a NeXus file.""" -class NeXusClassName(sciline.Scope[ComponentType, str], str): +class NeXusClassName(sciline.Scope[Component, str], str): """NX_class of a component in a NeXus file.""" @@ -139,14 +137,12 @@ class NeXusClassName(sciline.Scope[ComponentType, str], str): class NeXusComponent( - sciline.ScopeTwoParams[ComponentType, RunType, sc.DataGroup], sc.DataGroup + sciline.ScopeTwoParams[Component, RunType, sc.DataGroup], sc.DataGroup ): """Raw data from a NeXus component.""" -class NeXusData( - sciline.ScopeTwoParams[ComponentType, RunType, sc.DataArray], sc.DataArray -): +class NeXusData(sciline.ScopeTwoParams[Component, RunType, sc.DataArray], sc.DataArray): """ Data array loaded from an NXevent_data or NXdata group. @@ -155,7 +151,7 @@ class NeXusData( class ComponentPosition( - sciline.ScopeTwoParams[ComponentType, RunType, sc.Variable], sc.Variable + sciline.ScopeTwoParams[Component, RunType, sc.Variable], sc.Variable ): """Position of a component such as source, sample, monitor, or detector.""" @@ -229,7 +225,7 @@ class NeXusComponentLocationSpec(NeXusLocationSpec, Generic[Component, RunType]) @dataclass -class NeXusDataLocationSpec(NeXusLocationSpec, Generic[ComponentType, RunType]): +class NeXusDataLocationSpec(NeXusLocationSpec, Generic[Component, RunType]): """NeXus filename and parameters to identify (parts of) detector data to load.""" @@ -237,9 +233,7 @@ class NeXusDataLocationSpec(NeXusLocationSpec, Generic[ComponentType, RunType]): @dataclass -class NeXusTransformationChain( - snx.TransformationChain, Generic[ComponentType, RunType] -): +class NeXusTransformationChain(snx.TransformationChain, Generic[Component, RunType]): @classmethod def from_base(cls: type[T], base: snx.TransformationChain) -> T: return cls( @@ -253,14 +247,14 @@ def compute_position(self) -> sc.Variable | sc.DataArray: @dataclass -class NeXusTransformation(Generic[ComponentType, RunType]): +class NeXusTransformation(Generic[Component, RunType]): value: sc.Variable @staticmethod def from_chain( - chain: NeXusTransformationChain[ComponentType, RunType], + chain: NeXusTransformationChain[Component, RunType], # TODO can add filter options here - ) -> 'NeXusTransformation[ComponentType, RunType]': + ) -> 'NeXusTransformation[Component, RunType]': transform = chain.compute() if transform.ndim == 0: return NeXusTransformation(value=transform) diff --git a/src/ess/reduce/nexus/workflow.py b/src/ess/reduce/nexus/workflow.py index a3c40d08..104af322 100644 --- a/src/ess/reduce/nexus/workflow.py +++ b/src/ess/reduce/nexus/workflow.py @@ -15,8 +15,8 @@ CalibratedBeamline, CalibratedDetector, CalibratedMonitor, + Component, ComponentPosition, - ComponentType, DetectorBankSizes, DetectorData, DetectorPositionOffset, @@ -75,8 +75,8 @@ def gravity_vector_neg_y() -> GravityVector: def component_spec_by_name( - filename: NeXusFileSpec[RunType], name: NeXusComponentName[ComponentType] -) -> NeXusComponentLocationSpec[ComponentType, RunType]: + filename: NeXusFileSpec[RunType], name: NeXusComponentName[Component] +) -> NeXusComponentLocationSpec[Component, RunType]: """ Create a location spec for a component group in a NeXus file. @@ -87,7 +87,7 @@ def component_spec_by_name( name: Name of the component group. """ - return NeXusComponentLocationSpec[ComponentType, RunType]( + return NeXusComponentLocationSpec[Component, RunType]( filename=filename.value, component_name=name ) @@ -110,9 +110,9 @@ def unique_component_spec( def data_by_name( filename: NeXusFileSpec[RunType], - name: NeXusComponentName[ComponentType], + name: NeXusComponentName[Component], selection: PulseSelection[RunType], -) -> NeXusDataLocationSpec[ComponentType, RunType]: +) -> NeXusDataLocationSpec[Component, RunType]: """ Create a location spec for monitor or detector data in a NeXus file. @@ -125,7 +125,7 @@ def data_by_name( selection: Time range (start and stop as a Python slice object). """ - return NeXusDataLocationSpec[ComponentType, RunType]( + return NeXusDataLocationSpec[Component, RunType]( filename=filename.value, component_name=name, selection=selection.value ) @@ -170,9 +170,9 @@ def nx_class_for_sample() -> NeXusClassName[snx.NXsample]: def load_nexus_component( - location: NeXusComponentLocationSpec[ComponentType, RunType], - nx_class: NeXusClassName[ComponentType], -) -> NeXusComponent[ComponentType, RunType]: + location: NeXusComponentLocationSpec[Component, RunType], + nx_class: NeXusClassName[Component], +) -> NeXusComponent[Component, RunType]: """ Load a NeXus component group from a file. @@ -197,14 +197,14 @@ def load_nexus_component( nx_class: NX_class to identify the component. """ - return NeXusComponent[ComponentType, RunType]( + return NeXusComponent[Component, RunType]( nexus.load_component(location, nx_class=nx_class, definitions=definitions) ) def load_nexus_data( - location: NeXusDataLocationSpec[ComponentType, RunType], -) -> NeXusData[ComponentType, RunType]: + location: NeXusDataLocationSpec[Component, RunType], +) -> NeXusData[Component, RunType]: """ Load event or histogram data from a NeXus detector group. @@ -213,7 +213,7 @@ def load_nexus_data( location: Location spec for the detector group. """ - return NeXusData[ComponentType, RunType]( + return NeXusData[Component, RunType]( nexus.load_data( file_path=location.filename, entry_name=location.entry_name, @@ -224,8 +224,8 @@ def load_nexus_data( def get_transformation_chain( - detector: NeXusComponent[ComponentType, RunType], -) -> NeXusTransformationChain[ComponentType, RunType]: + detector: NeXusComponent[Component, RunType], +) -> NeXusTransformationChain[Component, RunType]: """ Extract the transformation chain from a NeXus detector group. @@ -235,21 +235,21 @@ def get_transformation_chain( NeXus detector group. """ chain = detector['depends_on'] - return NeXusTransformationChain[ComponentType, RunType].from_base(chain) + return NeXusTransformationChain[Component, RunType].from_base(chain) def to_transformation( - chain: NeXusTransformationChain[ComponentType, RunType], -) -> NeXusTransformation[ComponentType, RunType]: + chain: NeXusTransformationChain[Component, RunType], +) -> NeXusTransformation[Component, RunType]: """Convert transformation chain into a single transformation matrix.""" - return NeXusTransformation[ComponentType, RunType].from_chain(chain) + return NeXusTransformation[Component, RunType].from_chain(chain) def compute_position( - transformation: NeXusTransformation[ComponentType, RunType], -) -> ComponentPosition[ComponentType, RunType]: + transformation: NeXusTransformation[Component, RunType], +) -> ComponentPosition[Component, RunType]: """Compute the position of a component from a transformation matrix.""" - return ComponentPosition[ComponentType, RunType](transformation.value * origin) + return ComponentPosition[Component, RunType](transformation.value * origin) def get_calibrated_detector( From 35186161a6bef34468389840a6ef44802e44f595 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 3 Oct 2024 15:09:38 +0200 Subject: [PATCH 12/19] Rename monitor comp names --- src/ess/reduce/nexus/types.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ess/reduce/nexus/types.py b/src/ess/reduce/nexus/types.py index ace5d98e..76b7ef50 100644 --- a/src/ess/reduce/nexus/types.py +++ b/src/ess/reduce/nexus/types.py @@ -91,9 +91,9 @@ class TransmissionRun(Generic[ScatteringRunType]): """Identifier for an arbitrary monitor""" Monitor5 = NewType('Monitor5', int) """Identifier for an arbitrary monitor""" -Incident = NewType('Incident', int) +IncidentMonitor = NewType('IncidentMonitor', int) """Incident monitor""" -Transmission = NewType('Transmission', int) +TransmissionMonitor = NewType('TransmissionMonitor', int) """Transmission monitor""" MonitorType = TypeVar( 'MonitorType', @@ -102,8 +102,8 @@ class TransmissionRun(Generic[ScatteringRunType]): Monitor3, Monitor4, Monitor5, - Incident, - Transmission, + IncidentMonitor, + TransmissionMonitor, ) """TypeVar used for specifying the monitor type such as Incident or Transmission""" @@ -117,8 +117,8 @@ class TransmissionRun(Generic[ScatteringRunType]): Monitor3, Monitor4, Monitor5, - Incident, - Transmission, + IncidentMonitor, + TransmissionMonitor, ) UniqueComponentType = TypeVar('UniqueComponentType', snx.NXsample, snx.NXsource) """Components that can be identified by their type as there will only be one.""" From 54d39c4f8078f4dd3b5b11e9d378cf7c9792b88e Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 3 Oct 2024 15:11:43 +0200 Subject: [PATCH 13/19] Rename some more --- src/ess/reduce/nexus/types.py | 14 +++++----- src/ess/reduce/nexus/workflow.py | 46 +++++++++++++++----------------- 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/src/ess/reduce/nexus/types.py b/src/ess/reduce/nexus/types.py index 76b7ef50..8b4b92bc 100644 --- a/src/ess/reduce/nexus/types.py +++ b/src/ess/reduce/nexus/types.py @@ -120,19 +120,19 @@ class TransmissionRun(Generic[ScatteringRunType]): IncidentMonitor, TransmissionMonitor, ) -UniqueComponentType = TypeVar('UniqueComponentType', snx.NXsample, snx.NXsource) +UniqueComponent = TypeVar('UniqueComponent', snx.NXsample, snx.NXsource) """Components that can be identified by their type as there will only be one.""" -class NeXusComponentName(sciline.Scope[Component, str], str): - """Name of a monitor or detector component in a NeXus file.""" +class NeXusName(sciline.Scope[Component, str], str): + """Name of a component in a NeXus file.""" -class NeXusClassName(sciline.Scope[Component, str], str): +class NeXusClass(sciline.Scope[Component, str], str): """NX_class of a component in a NeXus file.""" -NeXusDetectorName = NeXusComponentName[snx.NXdetector] +NeXusDetectorName = NeXusName[snx.NXdetector] """Name of a detector (bank) in a NeXus file.""" @@ -150,9 +150,7 @@ class NeXusData(sciline.ScopeTwoParams[Component, RunType, sc.DataArray], sc.Dat """ -class ComponentPosition( - sciline.ScopeTwoParams[Component, RunType, sc.Variable], sc.Variable -): +class Position(sciline.ScopeTwoParams[Component, RunType, sc.Variable], sc.Variable): """Position of a component such as source, sample, monitor, or detector.""" diff --git a/src/ess/reduce/nexus/workflow.py b/src/ess/reduce/nexus/workflow.py index 104af322..4961c938 100644 --- a/src/ess/reduce/nexus/workflow.py +++ b/src/ess/reduce/nexus/workflow.py @@ -16,7 +16,6 @@ CalibratedDetector, CalibratedMonitor, Component, - ComponentPosition, DetectorBankSizes, DetectorData, DetectorPositionOffset, @@ -25,19 +24,20 @@ MonitorData, MonitorPositionOffset, MonitorType, - NeXusClassName, + NeXusClass, NeXusComponent, NeXusComponentLocationSpec, - NeXusComponentName, NeXusData, NeXusDataLocationSpec, NeXusFileSpec, + NeXusName, NeXusTransformation, NeXusTransformationChain, + Position, PreopenNeXusFile, PulseSelection, RunType, - UniqueComponentType, + UniqueComponent, ) origin = sc.vector([0, 0, 0], unit="m") @@ -75,7 +75,7 @@ def gravity_vector_neg_y() -> GravityVector: def component_spec_by_name( - filename: NeXusFileSpec[RunType], name: NeXusComponentName[Component] + filename: NeXusFileSpec[RunType], name: NeXusName[Component] ) -> NeXusComponentLocationSpec[Component, RunType]: """ Create a location spec for a component group in a NeXus file. @@ -94,7 +94,7 @@ def component_spec_by_name( def unique_component_spec( filename: NeXusFileSpec[RunType], -) -> NeXusComponentLocationSpec[UniqueComponentType, RunType]: +) -> NeXusComponentLocationSpec[UniqueComponent, RunType]: """ Create a location spec for a unique component group in a NeXus file. @@ -103,14 +103,12 @@ def unique_component_spec( filename: NeXus file to use for the location spec. """ - return NeXusComponentLocationSpec[UniqueComponentType, RunType]( - filename=filename.value - ) + return NeXusComponentLocationSpec[UniqueComponent, RunType](filename=filename.value) def data_by_name( filename: NeXusFileSpec[RunType], - name: NeXusComponentName[Component], + name: NeXusName[Component], selection: PulseSelection[RunType], ) -> NeXusDataLocationSpec[Component, RunType]: """ @@ -153,25 +151,25 @@ def load_nexus_sample( return NeXusComponent[snx.NXsample, RunType](dg) -def nx_class_for_monitor() -> NeXusClassName[MonitorType]: - return NeXusClassName[MonitorType](snx.NXmonitor) +def nx_class_for_monitor() -> NeXusClass[MonitorType]: + return NeXusClass[MonitorType](snx.NXmonitor) -def nx_class_for_detector() -> NeXusClassName[snx.NXdetector]: - return NeXusClassName[snx.NXdetector](snx.NXdetector) +def nx_class_for_detector() -> NeXusClass[snx.NXdetector]: + return NeXusClass[snx.NXdetector](snx.NXdetector) -def nx_class_for_source() -> NeXusClassName[snx.NXsource]: - return NeXusClassName[snx.NXsource](snx.NXsource) +def nx_class_for_source() -> NeXusClass[snx.NXsource]: + return NeXusClass[snx.NXsource](snx.NXsource) -def nx_class_for_sample() -> NeXusClassName[snx.NXsample]: - return NeXusClassName[snx.NXsample](snx.NXsample) +def nx_class_for_sample() -> NeXusClass[snx.NXsample]: + return NeXusClass[snx.NXsample](snx.NXsample) def load_nexus_component( location: NeXusComponentLocationSpec[Component, RunType], - nx_class: NeXusClassName[Component], + nx_class: NeXusClass[Component], ) -> NeXusComponent[Component, RunType]: """ Load a NeXus component group from a file. @@ -247,9 +245,9 @@ def to_transformation( def compute_position( transformation: NeXusTransformation[Component, RunType], -) -> ComponentPosition[Component, RunType]: +) -> Position[Component, RunType]: """Compute the position of a component from a transformation matrix.""" - return ComponentPosition[Component, RunType](transformation.value * origin) + return Position[Component, RunType](transformation.value * origin) def get_calibrated_detector( @@ -291,8 +289,8 @@ def get_calibrated_detector( def assemble_beamline( detector: CalibratedDetector[RunType], - source_position: ComponentPosition[snx.NXsource, RunType], - sample_position: ComponentPosition[snx.NXsample, RunType], + source_position: Position[snx.NXsource, RunType], + sample_position: Position[snx.NXsample, RunType], gravity: GravityVector, ) -> CalibratedBeamline[RunType]: """ @@ -347,7 +345,7 @@ def assemble_detector_data( def get_calibrated_monitor( monitor: NeXusComponent[MonitorType, RunType], offset: MonitorPositionOffset[RunType, MonitorType], - source_position: ComponentPosition[snx.NXsource, RunType], + source_position: Position[snx.NXsource, RunType], ) -> CalibratedMonitor[RunType, MonitorType]: """ Extract the data array corresponding to a monitor's signal field. From 1aa06c9200e38f3c044d9a2ddace71783184980e Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 3 Oct 2024 15:33:03 +0200 Subject: [PATCH 14/19] Update tests --- tests/nexus/nexus_loader_test.py | 4 +- tests/nexus/workflow_test.py | 121 +++++++++++-------------------- 2 files changed, 45 insertions(+), 80 deletions(-) diff --git a/tests/nexus/nexus_loader_test.py b/tests/nexus/nexus_loader_test.py index 29f9a71e..9fac4f0d 100644 --- a/tests/nexus/nexus_loader_test.py +++ b/tests/nexus/nexus_loader_test.py @@ -228,7 +228,7 @@ def test_load_data_loads_expected_event_data(nexus_file, expected_bank12): def test_load_data_loads_expected_histogram_data(nexus_file, expected_monitor): histogram = nexus.load_data( nexus_file, - component_name=nexus.types.NeXusMonitorName[nexus.types.Monitor1]('monitor'), + component_name=nexus.types.NeXusName[nexus.types.Monitor1]('monitor'), ) sc.testing.assert_identical(histogram, expected_monitor) @@ -401,7 +401,7 @@ def test_load_monitor(nexus_file, expected_monitor, entry_name, selection): loc = NeXusLocationSpec( filename=nexus_file, entry_name=entry_name, - component_name=nexus.types.NeXusMonitorName[nexus.types.Monitor1]('monitor'), + component_name=nexus.types.NeXusName[nexus.types.Monitor1]('monitor'), ) if selection is not None: loc.selection = selection diff --git a/tests/nexus/workflow_test.py b/tests/nexus/workflow_test.py index da30115c..04da03eb 100644 --- a/tests/nexus/workflow_test.py +++ b/tests/nexus/workflow_test.py @@ -12,8 +12,8 @@ Filename, Monitor1, MonitorData, - NeXusDetectorName, - NeXusMonitorName, + NeXusName, + NeXusTransformation, SampleRun, ) from ess.reduce.nexus.workflow import ( @@ -24,8 +24,8 @@ @pytest.fixture(params=[{}, {'aux': 1}]) -def group_with_no_position(request) -> workflow.NeXusSample[SampleRun]: - return workflow.NeXusSample[SampleRun](sc.DataGroup(request.param)) +def group_with_no_position(request) -> workflow.NeXusComponent[snx.NXsample, SampleRun]: + return workflow.NeXusComponent[snx.NXsample, SampleRun](sc.DataGroup(request.param)) @pytest.fixture() @@ -45,40 +45,34 @@ def depends_on() -> snx.TransformationChain: ) -def test_sample_position_returns_position_of_group() -> None: - position = sc.vector([1.0, 2.0, 3.0], unit='m') - sample_group = workflow.NeXusSample[SampleRun](sc.DataGroup(position=position)) - assert_identical(workflow.get_sample_position(sample_group), position) - - -def test_get_sample_position_returns_origin_if_position_not_found( - group_with_no_position, -) -> None: - assert_identical( - workflow.get_sample_position(group_with_no_position), workflow.origin - ) +@pytest.fixture() +def transform( + depends_on: snx.TransformationChain, +) -> NeXusTransformation[snx.NXdetector, SampleRun]: + return NeXusTransformation.from_chain(depends_on) -def test_get_source_position_returns_position_of_group( - depends_on: snx.TransformationChain, -) -> None: +def test_can_compute_position_of_group(depends_on: snx.TransformationChain) -> None: position = sc.vector([1.0, 2.0, 3.0], unit='m') - source_group = workflow.NeXusSource[SampleRun](sc.DataGroup(depends_on=depends_on)) - chain = workflow.get_source_transformation_chain(source_group) - assert_identical(workflow.get_source_position(chain), position) + group = workflow.NeXusComponent[snx.NXsource, SampleRun]( + sc.DataGroup(depends_on=depends_on) + ) + chain = workflow.get_transformation_chain(group) + trans = workflow.to_transformation(chain) + assert_identical(workflow.compute_position(trans), position) -def test_get_source_transformation_chain_raises_exception_if_position_not_found( +def test_get_transformation_chain_raises_exception_if_position_not_found( group_with_no_position, ) -> None: with pytest.raises(KeyError, match='depends_on'): - workflow.get_source_transformation_chain(group_with_no_position) + workflow.get_transformation_chain(group_with_no_position) @pytest.fixture() def nexus_detector( depends_on: snx.TransformationChain, -) -> workflow.NeXusDetector[SampleRun]: +) -> workflow.NeXusComponent[snx.NXdetector, SampleRun]: detector_number = sc.arange('detector_number', 6, unit=None) data = sc.DataArray( sc.empty_like(detector_number), @@ -87,7 +81,7 @@ def nexus_detector( 'x_pixel_offset': sc.linspace('detector_number', 0, 1, num=6, unit='m'), }, ) - return workflow.NeXusDetector[SampleRun]( + return workflow.NeXusComponent[snx.NXdetector, SampleRun]( sc.DataGroup(data=data, depends_on=depends_on, nexus_component_name='detector1') ) @@ -98,67 +92,51 @@ def source_position() -> sc.Variable: def test_get_calibrated_detector_extracts_data_field_from_nexus_detector( - nexus_detector, - source_position, + nexus_detector, transform ) -> None: detector = workflow.get_calibrated_detector( - nexus_detector, - offset=workflow.no_offset, - source_position=source_position, - sample_position=workflow.origin, - gravity=workflow.gravity_vector_neg_y(), - bank_sizes={}, - ) - assert_identical( - detector.drop_coords(('sample_position', 'source_position', 'gravity')), - compute_component_position(nexus_detector)['data'], + nexus_detector, offset=workflow.no_offset, bank_sizes={}, transform=transform ) + assert_identical(detector, compute_component_position(nexus_detector)['data']) def test_get_calibrated_detector_folds_detector_number_if_mapping_given( - nexus_detector, - source_position, + nexus_detector, transform ) -> None: sizes = {'xpixel': 2, 'ypixel': 3} bank_sizes = {'detector1': sizes} detector = workflow.get_calibrated_detector( nexus_detector, offset=workflow.no_offset, - source_position=source_position, - sample_position=workflow.origin, - gravity=workflow.gravity_vector_neg_y(), bank_sizes=bank_sizes, + transform=transform, ) assert detector.sizes == sizes def test_get_calibrated_detector_works_if_nexus_component_name_is_missing( - nexus_detector, source_position + nexus_detector, transform ): del nexus_detector['nexus_component_name'] detector = workflow.get_calibrated_detector( nexus_detector, offset=workflow.no_offset, - source_position=source_position, - sample_position=workflow.origin, - gravity=workflow.gravity_vector_neg_y(), bank_sizes={}, + transform=transform, ) assert detector.sizes == nexus_detector['data'].sizes def test_get_calibrated_detector_adds_offset_to_position( nexus_detector, - source_position, + transform, ) -> None: offset = sc.vector([0.1, 0.2, 0.3], unit='m') detector = workflow.get_calibrated_detector( nexus_detector, offset=offset, - source_position=source_position, - sample_position=workflow.origin, - gravity=workflow.gravity_vector_neg_y(), bank_sizes={}, + transform=transform, ) position = ( compute_component_position(nexus_detector)['data'].coords['position'] + offset @@ -167,34 +145,21 @@ def test_get_calibrated_detector_adds_offset_to_position( assert_identical(detector.coords['position'], position) -def test_get_calibrated_detector_forwards_coords( - nexus_detector, - source_position, -) -> None: +def test_get_calibrated_detector_forwards_coords(nexus_detector, transform) -> None: nexus_detector['data'].coords['abc'] = sc.scalar(1.2) detector = workflow.get_calibrated_detector( - nexus_detector, - offset=workflow.no_offset, - source_position=source_position, - sample_position=workflow.origin, - gravity=workflow.gravity_vector_neg_y(), - bank_sizes={}, + nexus_detector, offset=workflow.no_offset, bank_sizes={}, transform=transform ) assert 'abc' in detector.coords def test_get_calibrated_detector_forwards_masks( nexus_detector, - source_position, + transform, ) -> None: nexus_detector['data'].masks['mymask'] = sc.scalar(False) detector = workflow.get_calibrated_detector( - nexus_detector, - offset=workflow.no_offset, - source_position=source_position, - sample_position=workflow.origin, - gravity=workflow.gravity_vector_neg_y(), - bank_sizes={}, + nexus_detector, offset=workflow.no_offset, bank_sizes={}, transform=transform ) assert 'mymask' in detector.masks @@ -214,13 +179,13 @@ def calibrated_detector() -> workflow.CalibratedDetector[SampleRun]: @pytest.fixture() -def detector_event_data() -> workflow.NeXusDetectorData[SampleRun]: +def detector_event_data() -> workflow.NeXusData[snx.NXdetector, SampleRun]: content = sc.DataArray( sc.ones(dims=['event'], shape=[17], unit='counts'), coords={'event_id': sc.arange('event', 17, unit=None) % sc.index(6)}, ) weights = sc.bins(data=content, dim='event') - return workflow.NeXusDetectorData[SampleRun]( + return workflow.NeXusData[snx.NXdetector, SampleRun]( sc.DataArray( weights, coords={ @@ -279,9 +244,9 @@ def test_assemble_detector_preserves_masks(calibrated_detector, detector_event_d @pytest.fixture() def nexus_monitor( depends_on: snx.TransformationChain, -) -> workflow.NeXusMonitor[SampleRun, Monitor1]: +) -> workflow.NeXusComponent[Monitor1, SampleRun]: data = sc.DataArray(sc.scalar(1.2), coords={'something': sc.scalar(13)}) - return workflow.NeXusMonitor[SampleRun, Monitor1]( + return workflow.NeXusComponent[Monitor1, SampleRun]( sc.DataGroup(data=data, depends_on=depends_on) ) @@ -323,10 +288,10 @@ def calibrated_monitor() -> workflow.CalibratedMonitor[SampleRun, Monitor1]: @pytest.fixture() -def monitor_event_data() -> workflow.NeXusMonitorData[SampleRun, Monitor1]: +def monitor_event_data() -> workflow.NeXusData[Monitor1, SampleRun]: content = sc.DataArray(sc.ones(dims=['event'], shape=[17], unit='counts')) weights = sc.bins(data=content, dim='event') - return workflow.NeXusMonitorData[SampleRun, Monitor1]( + return workflow.NeXusData[Monitor1, SampleRun]( sc.DataArray( weights, coords={ @@ -380,7 +345,7 @@ def test_assemble_monitor_preserves_masks(calibrated_monitor, monitor_event_data def test_load_monitor_workflow() -> None: wf = LoadMonitorWorkflow() wf[Filename[SampleRun]] = data.loki_tutorial_sample_run_60250() - wf[NeXusMonitorName[Monitor1]] = 'monitor_1' + wf[NeXusName[Monitor1]] = 'monitor_1' da = wf.compute(MonitorData[SampleRun, Monitor1]) assert 'position' in da.coords assert 'source_position' in da.coords @@ -391,7 +356,7 @@ def test_load_monitor_workflow() -> None: def test_load_detector_workflow() -> None: wf = LoadDetectorWorkflow() wf[Filename[SampleRun]] = data.loki_tutorial_sample_run_60250() - wf[NeXusDetectorName] = 'larmor_detector' + wf[NeXusName[snx.NXdetector]] = 'larmor_detector' da = wf.compute(DetectorData[SampleRun]) assert 'position' in da.coords assert 'sample_position' in da.coords @@ -403,8 +368,8 @@ def test_load_detector_workflow() -> None: def test_generic_nexus_workflow() -> None: wf = GenericNeXusWorkflow() wf[Filename[SampleRun]] = data.loki_tutorial_sample_run_60250() - wf[NeXusMonitorName[Monitor1]] = 'monitor_1' - wf[NeXusDetectorName] = 'larmor_detector' + wf[NeXusName[Monitor1]] = 'monitor_1' + wf[NeXusName[snx.NXdetector]] = 'larmor_detector' da = wf.compute(DetectorData[SampleRun]) assert 'position' in da.coords assert 'sample_position' in da.coords From f71405a5d84eec74841f2540c2352543a4be461e Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 3 Oct 2024 15:36:33 +0200 Subject: [PATCH 15/19] docstring --- src/ess/reduce/nexus/types.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/ess/reduce/nexus/types.py b/src/ess/reduce/nexus/types.py index 8b4b92bc..e8e97403 100644 --- a/src/ess/reduce/nexus/types.py +++ b/src/ess/reduce/nexus/types.py @@ -251,8 +251,16 @@ class NeXusTransformation(Generic[Component, RunType]): @staticmethod def from_chain( chain: NeXusTransformationChain[Component, RunType], - # TODO can add filter options here ) -> 'NeXusTransformation[Component, RunType]': + """ + Convert a transformation chain to a single transformation. + + As transformation chains may be time-dependent, this method will need to select + a specific time point to convert to a single transformation. This may include + averaging as well as threshold checks. This is not implemented yet and we + therefore currently raise an error if the transformation chain does not compute + to a scalar. + """ transform = chain.compute() if transform.ndim == 0: return NeXusTransformation(value=transform) From 3b80bc2cf674f43841e26a20d005c3df6c5f48e5 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 3 Oct 2024 15:47:41 +0200 Subject: [PATCH 16/19] Cleanup unused subclass --- src/ess/reduce/nexus/types.py | 16 ++++------------ src/ess/reduce/nexus/workflow.py | 2 +- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/src/ess/reduce/nexus/types.py b/src/ess/reduce/nexus/types.py index e8e97403..835a7ad5 100644 --- a/src/ess/reduce/nexus/types.py +++ b/src/ess/reduce/nexus/types.py @@ -230,18 +230,10 @@ class NeXusDataLocationSpec(NeXusLocationSpec, Generic[Component, RunType]): T = TypeVar('T', bound='NeXusTransformationChain') -@dataclass -class NeXusTransformationChain(snx.TransformationChain, Generic[Component, RunType]): - @classmethod - def from_base(cls: type[T], base: snx.TransformationChain) -> T: - return cls( - parent=base.parent, - value=base.value, - transformations=base.transformations, - ) - - def compute_position(self) -> sc.Variable | sc.DataArray: - return self.compute() * sc.vector([0, 0, 0], unit='m') +class NeXusTransformationChain( + sciline.ScopeTwoParams[Component, RunType, snx.TransformationChain], + snx.TransformationChain, +): ... @dataclass diff --git a/src/ess/reduce/nexus/workflow.py b/src/ess/reduce/nexus/workflow.py index 4961c938..c6fce1c8 100644 --- a/src/ess/reduce/nexus/workflow.py +++ b/src/ess/reduce/nexus/workflow.py @@ -233,7 +233,7 @@ def get_transformation_chain( NeXus detector group. """ chain = detector['depends_on'] - return NeXusTransformationChain[Component, RunType].from_base(chain) + return NeXusTransformationChain[Component, RunType](chain) def to_transformation( From f58b51f68074943293f541be153fc979460ffb62 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 3 Oct 2024 15:53:22 +0200 Subject: [PATCH 17/19] Document reason for function split --- src/ess/reduce/nexus/workflow.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/ess/reduce/nexus/workflow.py b/src/ess/reduce/nexus/workflow.py index c6fce1c8..5dc87c1f 100644 --- a/src/ess/reduce/nexus/workflow.py +++ b/src/ess/reduce/nexus/workflow.py @@ -296,6 +296,12 @@ def assemble_beamline( """ Add beamline information (gravity vector, source- and sample-position) to detector. + This is performed separately and after :py:func:`get_calibrated_detector` to avoid + as false dependency of, e.g., the reshaped detector numbers on the sample position. + The latter can change during a run, e.g., for a rotating sample. The detector + numbers might be used, e.g., to mask certain detector pixels, and should not depend + on the sample position. + Parameters ---------- detector: From e161dc903e5686b25204a88ce06d719c2a7b7ab3 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 3 Oct 2024 15:58:11 +0200 Subject: [PATCH 18/19] Comment on offset --- src/ess/reduce/nexus/workflow.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/ess/reduce/nexus/workflow.py b/src/ess/reduce/nexus/workflow.py index 5dc87c1f..c0528795 100644 --- a/src/ess/reduce/nexus/workflow.py +++ b/src/ess/reduce/nexus/workflow.py @@ -254,6 +254,12 @@ def get_calibrated_detector( detector: NeXusComponent[snx.NXdetector, RunType], *, transform: NeXusTransformation[snx.NXdetector, RunType], + # Strictly speaking we could apply an offset by modifying the transformation chain, + # using a more generic implementation. However, this may in general require + # extending the chain and it is currently not clear if that is desirable. As far as + # I am aware the offset is currently mainly used for handling files from other + # facilities and it is not clear if it is needed for ESS data and should be kept at + # all. offset: DetectorPositionOffset[RunType], bank_sizes: DetectorBankSizes, ) -> CalibratedDetector[RunType]: From 1618d6c17b94d6c8dc654e267ac4472937487406 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Mon, 7 Oct 2024 10:07:58 +0200 Subject: [PATCH 19/19] Bump to new minimum scippnexus --- pyproject.toml | 2 +- requirements/base.in | 2 +- requirements/base.txt | 10 +++++----- requirements/basetest.txt | 18 +++++++++--------- requirements/ci.txt | 20 +++++++++++--------- requirements/dev.txt | 20 ++++++++++---------- requirements/docs.txt | 24 ++++++++++++------------ requirements/nightly.txt | 4 ++-- requirements/static.txt | 10 +++++----- requirements/wheels.txt | 6 +++--- 10 files changed, 59 insertions(+), 57 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 04975538..3a18e2ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ requires-python = ">=3.10" dependencies = [ "sciline>=24.06.2", "scipp>=24.02.0", - "scippnexus>=24.03.0", + "scippnexus>=24.10.0", ] dynamic = ["version"] diff --git a/requirements/base.in b/requirements/base.in index 8df10a71..4e26069d 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -4,4 +4,4 @@ # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! sciline>=24.06.2 scipp>=24.02.0 -scippnexus>=24.03.0 +scippnexus>=24.10.0 diff --git a/requirements/base.txt b/requirements/base.txt index 5ac51b6f..9793ba22 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:01b3a1b355da9250816f108088c3e93e58323bf7 +# SHA1:f3f405557d50cce037f6d681dfb1c5c13f5bef49 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -7,24 +7,24 @@ # cyclebane==24.6.0 # via sciline -h5py==3.11.0 +h5py==3.12.1 # via scippnexus networkx==3.3 # via cyclebane -numpy==2.1.0 +numpy==2.1.2 # via # h5py # scipp # scipy python-dateutil==2.9.0.post0 # via scippnexus -sciline==24.6.2 +sciline==24.6.3 # via -r base.in scipp==24.9.1 # via # -r base.in # scippnexus -scippnexus==24.8.1 +scippnexus==24.10.0 # via -r base.in scipy==1.14.1 # via scippnexus diff --git a/requirements/basetest.txt b/requirements/basetest.txt index b9eee571..58dab139 100644 --- a/requirements/basetest.txt +++ b/requirements/basetest.txt @@ -7,7 +7,7 @@ # asttokens==2.4.1 # via stack-data -certifi==2024.7.4 +certifi==2024.8.30 # via requests charset-normalizer==3.3.2 # via requests @@ -19,13 +19,13 @@ exceptiongroup==1.2.2 # via # ipython # pytest -executing==2.0.1 +executing==2.1.0 # via stack-data -idna==3.8 +idna==3.10 # via requests iniconfig==2.0.0 # via pytest -ipython==8.26.0 +ipython==8.28.0 # via ipywidgets ipywidgets==8.1.5 # via -r basetest.in @@ -43,13 +43,13 @@ parso==0.8.4 # via jedi pexpect==4.9.0 # via ipython -platformdirs==4.2.2 +platformdirs==4.3.6 # via pooch pluggy==1.5.0 # via pytest pooch==1.8.2 # via -r basetest.in -prompt-toolkit==3.0.47 +prompt-toolkit==3.0.48 # via ipython ptyprocess==0.7.0 # via pexpect @@ -57,7 +57,7 @@ pure-eval==0.2.3 # via stack-data pygments==2.18.0 # via ipython -pytest==8.3.2 +pytest==8.3.3 # via -r basetest.in requests==2.32.3 # via pooch @@ -65,7 +65,7 @@ six==1.16.0 # via asttokens stack-data==0.6.3 # via ipython -tomli==2.0.1 +tomli==2.0.2 # via pytest traitlets==5.14.3 # via @@ -75,7 +75,7 @@ traitlets==5.14.3 # matplotlib-inline typing-extensions==4.12.2 # via ipython -urllib3==2.2.2 +urllib3==2.2.3 # via requests wcwidth==0.2.13 # via prompt-toolkit diff --git a/requirements/ci.txt b/requirements/ci.txt index 1b6bca87..5898171f 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -7,7 +7,7 @@ # cachetools==5.5.0 # via tox -certifi==2024.7.4 +certifi==2024.8.30 # via requests chardet==5.2.0 # via tox @@ -17,7 +17,7 @@ colorama==0.4.6 # via tox distlib==0.3.8 # via virtualenv -filelock==3.15.4 +filelock==3.16.1 # via # tox # virtualenv @@ -25,32 +25,34 @@ gitdb==4.0.11 # via gitpython gitpython==3.1.43 # via -r ci.in -idna==3.8 +idna==3.10 # via requests packaging==24.1 # via # -r ci.in # pyproject-api # tox -platformdirs==4.2.2 +platformdirs==4.3.6 # via # tox # virtualenv pluggy==1.5.0 # via tox -pyproject-api==1.7.1 +pyproject-api==1.8.0 # via tox requests==2.32.3 # via -r ci.in smmap==5.0.1 # via gitdb -tomli==2.0.1 +tomli==2.0.2 # via # pyproject-api # tox -tox==4.18.0 +tox==4.21.2 # via -r ci.in -urllib3==2.2.2 +typing-extensions==4.12.2 + # via tox +urllib3==2.2.3 # via requests -virtualenv==20.26.3 +virtualenv==20.26.6 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index a74c561b..3f08a471 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -14,7 +14,7 @@ -r wheels.txt annotated-types==0.7.0 # via pydantic -anyio==4.4.0 +anyio==4.6.0 # via # httpx # jupyter-server @@ -26,7 +26,7 @@ arrow==1.3.0 # via isoduration async-lru==2.0.4 # via jupyterlab -cffi==1.17.0 +cffi==1.17.1 # via argon2-cffi-bindings click==8.1.7 # via @@ -42,9 +42,9 @@ funcy==2.0 # via copier h11==0.14.0 # via httpcore -httpcore==1.0.5 +httpcore==1.0.6 # via httpx -httpx==0.27.0 +httpx==0.27.2 # via jupyterlab isoduration==20.11.0 # via jsonschema @@ -71,7 +71,7 @@ jupyter-server==2.14.2 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.2.4 +jupyterlab==4.2.5 # via -r dev.in jupyterlab-server==2.27.3 # via jupyterlab @@ -85,15 +85,15 @@ pip-compile-multi==2.6.4 # via -r dev.in pip-tools==7.4.1 # via pip-compile-multi -plumbum==1.8.3 +plumbum==1.9.0 # via copier -prometheus-client==0.20.0 +prometheus-client==0.21.0 # via jupyter-server pycparser==2.22 # via cffi -pydantic==2.8.2 +pydantic==2.9.2 # via copier -pydantic-core==2.20.1 +pydantic-core==2.23.4 # via pydantic python-json-logger==2.0.7 # via jupyter-events @@ -119,7 +119,7 @@ terminado==0.18.1 # jupyter-server-terminals toposort==1.10 # via pip-compile-multi -types-python-dateutil==2.9.0.20240821 +types-python-dateutil==2.9.0.20241003 # via arrow uri-template==1.3.0 # via jsonschema diff --git a/requirements/docs.txt b/requirements/docs.txt index fd6f29c1..0dee3c45 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -26,7 +26,7 @@ beautifulsoup4==4.12.3 # pydata-sphinx-theme bleach==6.1.0 # via nbconvert -certifi==2024.7.4 +certifi==2024.8.30 # via requests charset-normalizer==3.3.2 # via requests @@ -34,7 +34,7 @@ comm==0.2.2 # via # ipykernel # ipywidgets -debugpy==1.8.5 +debugpy==1.8.6 # via ipykernel decorator==5.1.1 # via ipython @@ -48,19 +48,19 @@ docutils==0.21.2 # sphinx exceptiongroup==1.2.2 # via ipython -executing==2.0.1 +executing==2.1.0 # via stack-data fastjsonschema==2.20.0 # via nbformat graphviz==0.20.3 # via -r docs.in -idna==3.8 +idna==3.10 # via requests imagesize==1.4.1 # via sphinx ipykernel==6.29.5 # via -r docs.in -ipython==8.26.0 +ipython==8.28.0 # via # -r docs.in # ipykernel @@ -79,7 +79,7 @@ jsonschema==4.23.0 # via nbformat jsonschema-specifications==2023.12.1 # via jsonschema -jupyter-client==8.6.2 +jupyter-client==8.6.3 # via # ipykernel # nbclient @@ -106,7 +106,7 @@ matplotlib-inline==0.1.7 # via # ipykernel # ipython -mdit-py-plugins==0.4.1 +mdit-py-plugins==0.4.2 # via myst-parser mdurl==0.1.2 # via markdown-it-py @@ -139,9 +139,9 @@ parso==0.8.4 # via jedi pexpect==4.9.0 # via ipython -platformdirs==4.2.2 +platformdirs==4.3.6 # via jupyter-core -prompt-toolkit==3.0.47 +prompt-toolkit==3.0.48 # via ipython psutil==6.0.0 # via ipykernel @@ -187,7 +187,7 @@ sphinx==8.0.2 # sphinx-autodoc-typehints # sphinx-copybutton # sphinx-design -sphinx-autodoc-typehints==2.2.3 +sphinx-autodoc-typehints==2.4.4 # via -r docs.in sphinx-copybutton==0.5.2 # via -r docs.in @@ -209,7 +209,7 @@ stack-data==0.6.3 # via ipython tinycss2==1.3.0 # via nbconvert -tomli==2.0.1 +tomli==2.0.2 # via sphinx tornado==6.4.1 # via @@ -232,7 +232,7 @@ typing-extensions==4.12.2 # via # ipython # pydata-sphinx-theme -urllib3==2.2.2 +urllib3==2.2.3 # via requests wcwidth==0.2.13 # via prompt-toolkit diff --git a/requirements/nightly.txt b/requirements/nightly.txt index 79edae8c..dcc8ec31 100644 --- a/requirements/nightly.txt +++ b/requirements/nightly.txt @@ -10,11 +10,11 @@ cyclebane @ git+https://github.com/scipp/cyclebane@main # via # -r nightly.in # sciline -h5py==3.11.0 +h5py==3.12.1 # via scippnexus networkx==3.3 # via cyclebane -numpy==2.1.0 +numpy==2.1.2 # via # h5py # scipp diff --git a/requirements/static.txt b/requirements/static.txt index 85da246d..4910f304 100644 --- a/requirements/static.txt +++ b/requirements/static.txt @@ -9,17 +9,17 @@ cfgv==3.4.0 # via pre-commit distlib==0.3.8 # via virtualenv -filelock==3.15.4 +filelock==3.16.1 # via virtualenv -identify==2.6.0 +identify==2.6.1 # via pre-commit nodeenv==1.9.1 # via pre-commit -platformdirs==4.2.2 +platformdirs==4.3.6 # via virtualenv -pre-commit==3.8.0 +pre-commit==4.0.0 # via -r static.in pyyaml==6.0.2 # via pre-commit -virtualenv==20.26.3 +virtualenv==20.26.6 # via pre-commit diff --git a/requirements/wheels.txt b/requirements/wheels.txt index a1fa46e2..d1c1063b 100644 --- a/requirements/wheels.txt +++ b/requirements/wheels.txt @@ -5,11 +5,11 @@ # # pip-compile-multi # -build==1.2.1 +build==1.2.2.post1 # via -r wheels.in packaging==24.1 # via build -pyproject-hooks==1.1.0 +pyproject-hooks==1.2.0 # via build -tomli==2.0.1 +tomli==2.0.2 # via build