diff --git a/src/snapred/backend/dao/ingredients/GroceryListItem.py b/src/snapred/backend/dao/ingredients/GroceryListItem.py index c52251f59..12ca109a0 100644 --- a/src/snapred/backend/dao/ingredients/GroceryListItem.py +++ b/src/snapred/backend/dao/ingredients/GroceryListItem.py @@ -3,6 +3,7 @@ from pydantic import BaseModel, model_validator from snapred.backend.log.logger import snapredLogger +from snapred.meta.InternalConstants import ReservedRunNumber from snapred.meta.mantid.WorkspaceNameGenerator import WorkspaceNameGenerator as wng logger = snapredLogger.getLogger(__name__) @@ -32,9 +33,9 @@ class GroceryListItem(BaseModel): """ # Reserved instrument-cache run-number values: - RESERVED_NATIVE_RUNNUMBER: ClassVar[str] = "000000" # unmodified _native_ instrument: + RESERVED_NATIVE_RUNNUMBER: ClassVar[str] = ReservedRunNumber.NATIVE # unmodified _native_ instrument: # from 'SNAP_Definition.xml' - RESERVED_LITE_RUNNUMBER: ClassVar[str] = "000001" # unmodified _lite_ instrument : + RESERVED_LITE_RUNNUMBER: ClassVar[str] = ReservedRunNumber.LITE # unmodified _lite_ instrument : # from 'SNAPLite.xml' workspaceType: GroceryTypes diff --git a/src/snapred/backend/data/GroceryService.py b/src/snapred/backend/data/GroceryService.py index f627aa320..7279b7799 100644 --- a/src/snapred/backend/data/GroceryService.py +++ b/src/snapred/backend/data/GroceryService.py @@ -25,6 +25,7 @@ from snapred.backend.service.WorkspaceMetadataService import WorkspaceMetadataService from snapred.meta.Config import Config from snapred.meta.decorators.Singleton import Singleton +from snapred.meta.InternalConstants import ReservedRunNumber from snapred.meta.mantid.WorkspaceNameGenerator import ( NameBuilder, WorkspaceName, @@ -563,10 +564,7 @@ def _fetchInstrumentDonor(self, runNumber: str, useLiteMode: bool) -> WorkspaceN # Initialize the instrument parameters # (Reserved run-numbers will use the unmodified instrument.) - if ( - runNumber != GroceryListItem.RESERVED_NATIVE_RUNNUMBER - and runNumber != GroceryListItem.RESERVED_LITE_RUNNUMBER - ): + if runNumber not in ReservedRunNumber.values(): detectorState: DetectorState = self._getDetectorState(runNumber) self.updateInstrumentParameters(wsName, detectorState) self._loadedInstruments[key] = wsName @@ -865,13 +863,14 @@ def fetchGroupingDefinition(self, item: GroceryListItem) -> Dict[str, Any]: :rtype: Dict[str, Any] """ - key = self._key(item.groupingScheme, item.runNumber, item.useLiteMode) + stateId, _ = self.dataService.generateStateId(item.runNumber) + key = self._key(item.groupingScheme, stateId, item.useLiteMode) workspaceName = self._createGroupingWorkspaceName(item.groupingScheme, item.runNumber, item.useLiteMode) + workspaceName = self._loadedGroupings.get(key, workspaceName) self._updateGroupingCacheFromADS(key, workspaceName) - groupingIsLoaded = self._loadedGroupings.get(key) is not None - if groupingIsLoaded: + if key in self._loadedGroupings: data = { "result": True, "loader": "cached", diff --git a/src/snapred/backend/data/LocalDataService.py b/src/snapred/backend/data/LocalDataService.py index 9daddf1ce..0fe667c07 100644 --- a/src/snapred/backend/data/LocalDataService.py +++ b/src/snapred/backend/data/LocalDataService.py @@ -51,6 +51,7 @@ from snapred.meta.Config import Config from snapred.meta.decorators.ExceptionHandler import ExceptionHandler from snapred.meta.decorators.Singleton import Singleton +from snapred.meta.InternalConstants import ReservedRunNumber, ReservedStateId from snapred.meta.mantid.WorkspaceNameGenerator import ( ValueFormatter as wnvf, ) @@ -288,8 +289,11 @@ def _readPVFile(self, runId: str): @lru_cache @ExceptionHandler(StateValidationException) def generateStateId(self, runId: str) -> Tuple[str, str]: - detectorState = self.readDetectorState(runId) - SHA = self._stateIdFromDetectorState(detectorState) + if runId in ReservedRunNumber.values(): + SHA = ObjectSHA(hex=ReservedStateId.forRun(runId)) + else: + detectorState = self.readDetectorState(runId) + SHA = self._stateIdFromDetectorState(detectorState) return SHA.hex, SHA.decodedKey def _stateIdFromDetectorState(self, detectorState: DetectorState) -> ObjectSHA: diff --git a/src/snapred/backend/service/ReductionService.py b/src/snapred/backend/service/ReductionService.py index 67ce7784e..7f8b08069 100644 --- a/src/snapred/backend/service/ReductionService.py +++ b/src/snapred/backend/service/ReductionService.py @@ -81,7 +81,7 @@ def __init__(self): self.registerPath("checkWritePermissions", self.checkReductionWritePermissions) self.registerPath("getSavePath", self.getSavePath) self.registerPath("getStateIds", self.getStateIds) - self.registerPath("validateReduction", self.validateReduction) + self.registerPath("validate", self.validateReduction) self.registerPath("artificialNormalization", self.artificialNormalization) self.registerPath("grabWorkspaceforArtificialNorm", self.grabWorkspaceforArtificialNorm) return diff --git a/src/snapred/meta/InternalConstants.py b/src/snapred/meta/InternalConstants.py new file mode 100644 index 000000000..566104298 --- /dev/null +++ b/src/snapred/meta/InternalConstants.py @@ -0,0 +1,20 @@ +from snapred.backend.dao.ObjectSHA import ObjectSHA +from snapred.meta.Enum import StrEnum + + +class ReservedRunNumber(StrEnum): + NATIVE: str = "000000" + LITE: str = "000001" + + +class ReservedStateId(StrEnum): + NATIVE: str = ObjectSHA(hex="0000000000000000").hex + LITE: str = ObjectSHA(hex="0000000000000001").hex + + @classmethod + def forRun(cls, runNumber): + match runNumber: + case ReservedRunNumber.NATIVE.value: + return cls.NATIVE + case ReservedRunNumber.LITE.value: + return cls.LITE diff --git a/src/snapred/ui/view/reduction/ReductionRequestView.py b/src/snapred/ui/view/reduction/ReductionRequestView.py index bdbb859d4..383dc6eea 100644 --- a/src/snapred/ui/view/reduction/ReductionRequestView.py +++ b/src/snapred/ui/view/reduction/ReductionRequestView.py @@ -1,12 +1,12 @@ from typing import Callable, List, Optional -from qtpy.QtCore import Signal, Slot +from qtpy.QtCore import Slot from qtpy.QtWidgets import ( QHBoxLayout, QLineEdit, + QListWidget, QMessageBox, QPushButton, - QTextEdit, QVBoxLayout, ) from snapred.backend.dao.state.RunNumber import RunNumber @@ -20,8 +20,6 @@ @Resettable class ReductionRequestView(BackendRequestView): - signalRemoveRunNumber = Signal(int) - def __init__( self, parent=None, @@ -49,8 +47,8 @@ def __init__( self.runNumberLayout.addLayout(self.runNumberButtonLayout) # Run number display - self.runNumberDisplay = QTextEdit() - self.runNumberDisplay.setReadOnly(True) + self.runNumberDisplay = QListWidget() + self.runNumberDisplay.setSortingEnabled(False) # Lite mode toggle, pixel masks dropdown, and retain unfocused data checkbox self.liteModeToggle = self._labeledField("Lite Mode", Toggle(parent=self, state=True)) @@ -78,8 +76,6 @@ def __init__( self.enterRunNumberButton.clicked.connect(self.addRunNumber) self.clearButton.clicked.connect(self.clearRunNumbers) - self.signalRemoveRunNumber.connect(self._removeRunNumber) - @Slot() def addRunNumber(self): # TODO: FIX THIS! @@ -127,32 +123,20 @@ def parseInputRunNumbers(self) -> List[str]: return [str(num) for num in runs] - @Slot() - def removeRunNumber(self, runNumber): - self.signalRemoveRunNumber.emit(runNumber) - - @Slot() - def _removeRunNumber(self, runNumber): - if runNumber not in self.runNumbers: - logger.warning( - f"[ReductionRequestView]: attempting to remove run {runNumber} not in the list {self.runNumbers}" - ) - return - self.runNumbers.remove(runNumber) - self.updateRunNumberList() - def updateRunNumberList(self): - self.runNumberDisplay.setText("\n".join(map(str, sorted(self.runNumbers)))) + self.runNumberDisplay.clear() + self.runNumberDisplay.addItems(self.runNumbers) def clearRunNumbers(self): self.runNumbers.clear() self.runNumberDisplay.clear() def verify(self): - currentText = self.runNumberDisplay.toPlainText() - runNumbers = [num.strip() for num in currentText.split("\n") if num.strip()] + runNumbers = [self.runNumberDisplay.item(x).text() for x in range(self.runNumberDisplay.count())] if not runNumbers: raise ValueError("Please enter at least one run number.") + if runNumbers != self.runNumbers: + raise ValueError("Unexpected issue verifying run numbers. Please clear and re-enter.") for runNumber in runNumbers: if not runNumber.isdigit(): raise ValueError( diff --git a/src/snapred/ui/workflow/ReductionWorkflow.py b/src/snapred/ui/workflow/ReductionWorkflow.py index d4a615f60..e5a916f7e 100644 --- a/src/snapred/ui/workflow/ReductionWorkflow.py +++ b/src/snapred/ui/workflow/ReductionWorkflow.py @@ -58,7 +58,7 @@ def __init__(self, parent=None): ) .build() ) - + self._keeps = set() self._reductionRequestView.retainUnfocusedDataCheckbox.checkedChanged.connect(self._enableConvertToUnits) self._artificialNormalizationView.signalValueChanged.connect(self.onArtificialNormalizationValueChange) @@ -96,7 +96,7 @@ def _populatePixelMaskDropdown(self): return runNumbers = self._reductionRequestView.getRunNumbers() - useLiteMode = self._reductionRequestView.liteModeToggle.field.getState() # noqa: F841 + self.useLiteMode = self._reductionRequestView.liteModeToggle.field.getState() # noqa: F841 self._reductionRequestView.liteModeToggle.setEnabled(False) self._reductionRequestView.pixelMaskDropdown.setEnabled(False) @@ -111,7 +111,7 @@ def _populatePixelMaskDropdown(self): payload=ReductionRequest( # All runNumbers are from the same state => any one can be used here runNumber=runNumbers[0], - useLiteMode=useLiteMode, + useLiteMode=self.useLiteMode, ), ).data except Exception as e: # noqa: BLE001 @@ -126,7 +126,6 @@ def _populatePixelMaskDropdown(self): self._reductionRequestView.liteModeToggle.setEnabled(True) self._reductionRequestView.pixelMaskDropdown.setEnabled(True) self._reductionRequestView.retainUnfocusedDataCheckbox.setEnabled(True) - # self._reductionRequestView.convertUnitsDropdown.setEnabled(True) def _validateRunNumbers(self, runNumbers: List[str]): # For now, all run numbers in a reduction batch must be from the same instrument state. @@ -153,28 +152,41 @@ def _onPixelMaskSelection(self): # # ## Why would I want to set the ~obfuscated-by-pydantic~ `pixelMasks` field of the class object? + def _createReductionRequest(self, runNumber, artificialNormalizationIngredients=None): + """ + Create a standardized ReductionRequest object for passing to the ReductionService + """ + return ReductionRequest( + runNumber=str(runNumber), + useLiteMode=self.useLiteMode, + timestamp=self.timestamp, + continueFlags=self.continueAnywayFlags, + pixelMasks=self.pixelMasks, + keepUnfocused=self._reductionRequestView.retainUnfocusedDataCheckbox.isChecked(), + convertUnitsTo=self._reductionRequestView.convertUnitsDropdown.currentText(), + artificialNormalizationIngredients=artificialNormalizationIngredients, + ) + def _triggerReduction(self, workflowPresenter): view = workflowPresenter.widget.tabView # noqa: F841 - runNumbers = self._reductionRequestView.getRunNumbers() - pixelMasks = self._reconstructPixelMaskNames(self._reductionRequestView.getPixelMasks()) + self.runNumbers = self._reductionRequestView.getRunNumbers() + self.pixelMasks = self._reconstructPixelMaskNames(self._reductionRequestView.getPixelMasks()) # Use one timestamp for the entire set of runNumbers: - timestamp = self.request(path="reduction/getUniqueTimestamp").data - for runNumber in runNumbers: + self.timestamp = self.request(path="reduction/getUniqueTimestamp").data + + # all runs in same state, use the first run to load groupings + request_ = self._createReductionRequest(self.runNumbers[0]) + response = self.request(path="reduction/groupings", payload=request_) + self._keeps = set(response.data["groupingWorkspaces"]) + + for runNumber in self.runNumbers: self._artificialNormalizationView.updateRunNumber(runNumber) - request_ = ReductionRequest( - runNumber=str(runNumber), - useLiteMode=self._reductionRequestView.liteModeToggle.field.getState(), - timestamp=timestamp, - continueFlags=self.continueAnywayFlags, - pixelMasks=pixelMasks, - keepUnfocused=self._reductionRequestView.retainUnfocusedDataCheckbox.isChecked(), - convertUnitsTo=self._reductionRequestView.convertUnitsDropdown.currentText(), - ) + request_ = self._createReductionRequest(runNumber) # Validate reduction; if artificial normalization is needed, handle it - response = self.request(path="reduction/validateReduction", payload=request_) + response = self.request(path="reduction/validate", payload=request_) if ContinueWarning.Type.MISSING_NORMALIZATION in self.continueAnywayFlags: self._artificialNormalizationView.showAdjustView() response = self.request(path="reduction/grabWorkspaceforArtificialNorm", payload=request_) @@ -186,7 +198,14 @@ def _triggerReduction(self, workflowPresenter): if response.code == ResponseCode.OK: record, unfocusedData = response.data.record, response.data.unfocusedData self._finalizeReduction(record, unfocusedData) - workflowPresenter.advanceWorkflow() + # after each run, clean workspaces except groupings, calibrations, normalizations, and outputs + self._keeps.update(self.outputs) + self._clearWorkspaces(exclude=self._keeps, clearCachedWorkspaces=True) + workflowPresenter.advanceWorkflow() + # SPECIAL FOR THE REDUCTION WORKFLOW: clear everything _except_ the output workspaces + # _before_ transitioning to the "save" panel. + # TODO: make '_clearWorkspaces' a public method (i.e make this combination a special `cleanup` method). + self._clearWorkspaces(exclude=self.outputs, clearCachedWorkspaces=True) return self.responses[-1] def _artificialNormalization(self, workflowPresenter, responseData, runNumber): @@ -194,7 +213,7 @@ def _artificialNormalization(self, workflowPresenter, responseData, runNumber): view = workflowPresenter.widget.tabView # noqa: F841 request_ = CreateArtificialNormalizationRequest( runNumber=runNumber, - useLiteMode=self._reductionRequestView.liteModeToggle.field.getState(), + useLiteMode=self.useLiteMode, peakWindowClippingSize=int(self._artificialNormalizationView.peakWindowClippingSize.field.text()), smoothingParameter=self._artificialNormalizationView.getSmoothingParameter(), decreaseParameter=self._artificialNormalizationView.decreaseParameterDropdown.currentIndex() == 1, @@ -219,7 +238,7 @@ def onArtificialNormalizationValueChange(self, smoothingValue, lss, decreasePara request_ = CreateArtificialNormalizationRequest( runNumber=runNumber, - useLiteMode=self._reductionRequestView.liteModeToggle.field.getState(), + useLiteMode=self.useLiteMode, peakWindowClippingSize=peakWindowClippingSize, smoothingParameter=smoothingValue, decreaseParameter=decreaseParameter, @@ -276,11 +295,6 @@ def _finalizeReduction(self, record, unfocusedData): # Note that the run number is deliberately not deleted from the run numbers list. # Almost certainly it should be moved to a "completed run numbers" list. - # SPECIAL FOR THE REDUCTION WORKFLOW: clear everything _except_ the output workspaces - # _before_ transitioning to the "save" panel. - # TODO: make '_clearWorkspaces' a public method (i.e make this combination a special `cleanup` method). - self._clearWorkspaces(exclude=self.outputs, clearCachedWorkspaces=True) - @property def widget(self): return self.workflow.presenter.widget diff --git a/tests/integration/test_workflow_panels_happy_path.py b/tests/integration/test_workflow_panels_happy_path.py index 641240c5f..c8c83c5e1 100644 --- a/tests/integration/test_workflow_panels_happy_path.py +++ b/tests/integration/test_workflow_panels_happy_path.py @@ -607,8 +607,10 @@ def test_calibration_and_reduction_panels_happy_path( # enter a "Run Number": requestView.runNumberInput.setText(reductionRunNumber) qtbot.mouseClick(requestView.enterRunNumberButton, Qt.MouseButton.LeftButton) - _currentText = requestView.runNumberDisplay.toPlainText() - _runNumbers = [num.strip() for num in _currentText.split("\n") if num.strip()] + + _count = requestView.runNumberDisplay.count() + _runNumbers = [requestView.runNumberDisplay.item(x).text() for x in range(_count)] + assert reductionRunNumber in _runNumbers """ @@ -1257,8 +1259,8 @@ def completionMessageBoxAssert(*args, **kwargs): # noqa: ARG001 requestView.runNumberInput.setText(reductionRunNumber) qtbot.mouseClick(requestView.enterRunNumberButton, Qt.MouseButton.LeftButton) - _currentText = requestView.runNumberDisplay.toPlainText() - _runNumbers = [num.strip() for num in _currentText.split("\n") if num.strip()] + _count = requestView.runNumberDisplay.count() + _runNumbers = [requestView.runNumberDisplay.item(x).text() for x in range(_count)] assert reductionRunNumber in _runNumbers diff --git a/tests/unit/backend/data/test_GroceryService.py b/tests/unit/backend/data/test_GroceryService.py index b011fed44..38ec150b2 100644 --- a/tests/unit/backend/data/test_GroceryService.py +++ b/tests/unit/backend/data/test_GroceryService.py @@ -31,6 +31,7 @@ from snapred.backend.dao.WorkspaceMetadata import UNSET, DiffcalStateMetadata, WorkspaceMetadata from snapred.backend.data.GroceryService import GroceryService from snapred.meta.Config import Config, Resource +from snapred.meta.InternalConstants import ReservedRunNumber from snapred.meta.mantid.WorkspaceNameGenerator import ValueFormatter as wnvf from snapred.meta.mantid.WorkspaceNameGenerator import WorkspaceNameGenerator as wng from snapred.meta.mantid.WorkspaceNameGenerator import WorkspaceType @@ -133,6 +134,7 @@ def setUpClass(cls): def setUp(self): self.instance = GroceryService(dataService=self.fridge) + self.stateId, _ = self.instance.dataService.generateStateId(self.runNumber) self.groupingItem = ( GroceryListItem.builder() .fromRun(self.runNumber) @@ -996,10 +998,11 @@ def test_fetch_grouping(self): groupFilepath = Resource.getPath("inputs/testInstrument/fakeSNAPFocGroup_Natural.xml") self.instance._createGroupingFilename = mock.Mock(return_value=groupFilepath) testItem = (self.groupingItem.groupingScheme, self.runNumber, self.groupingItem.useLiteMode) + testKey = (self.groupingItem.groupingScheme, self.stateId, self.groupingItem.useLiteMode) # call once and load groupingWorkspaceName = self.instance._createGroupingWorkspaceName(*testItem) - groupKey = self.instance._key(*testItem) + groupKey = self.instance._key(*testKey) res = self.instance.fetchGroupingDefinition(self.groupingItem) assert res["result"] assert res["loader"] == "LoadGroupingDefinition" @@ -1027,21 +1030,23 @@ def test_fetch_lite_data_map(self, mockGroceryList): """ groupMapFilepath = Resource.getPath("inputs/testInstrument/fakeSNAPLiteGroupMap.xml") self.instance._fetchInstrumentDonor = mock.Mock(return_value=self.sampleWS) - self.instance._createGroupingWorkspaceName = mock.Mock(return_value="lite_map") + # self.instance._createGroupingWorkspaceName = mock.Mock(return_value="lite_map") self.instance._createGroupingFilename = mock.Mock(return_value=groupMapFilepath) # have to subvert the validation methods in grocerylistitem mockLiteMapGroceryItem = GroceryListItem( workspaceType="grouping", - runNumber=self.runNumber, + runNumber=ReservedRunNumber.NATIVE, groupingScheme="Lite", ) mockLiteMapGroceryItem.instrumentSource = self.instrumentFilePath mockGroceryList.builder.return_value.grouping.return_value.build.return_value = mockLiteMapGroceryItem + stateId, _ = self.instance.dataService.generateStateId(mockLiteMapGroceryItem.runNumber) # call once and load - testItem = ("Lite", GroceryListItem.RESERVED_NATIVE_RUNNUMBER, False) + testItem = ("Lite", ReservedRunNumber.NATIVE, False) + testKey = ("Lite", stateId, False) groupingWorkspaceName = self.instance._createGroupingWorkspaceName(*testItem) - groupKey = self.instance._key(*testItem) + groupKey = self.instance._key(*testKey) res = self.instance.fetchLiteDataMap() assert res == groupingWorkspaceName diff --git a/tests/unit/meta/test_InternalConstants.py b/tests/unit/meta/test_InternalConstants.py new file mode 100644 index 000000000..7e7a91ea0 --- /dev/null +++ b/tests/unit/meta/test_InternalConstants.py @@ -0,0 +1,11 @@ +import unittest + +from snapred.meta.InternalConstants import ReservedRunNumber, ReservedStateId + + +class TestCallback(unittest.TestCase): + def test_state_from_run(self): + reservedRunNumbers = ReservedRunNumber.values() + reservedStateIds = ReservedStateId.values() + for x, y in zip(reservedRunNumbers, reservedStateIds): + assert y == ReservedStateId.forRun(x)