Cache groupings by state (#503)

* cache grouping workspace by state

* fix integration test

* fixup! fix integration test

* fixup! fixup! fix integration test

* fixup! fixup! fixup! fix integration test

* add tests for codecov

* move location of reserved state assignment

* fixup! move location of reserved state assignment

src/snapred/backend/dao/ingredients/GroceryListItem.py

@@ -3,6 +3,7 @@
 from pydantic import BaseModel, model_validator
 
 from snapred.backend.log.logger import snapredLogger
+from snapred.meta.InternalConstants import ReservedRunNumber
 from snapred.meta.mantid.WorkspaceNameGenerator import WorkspaceNameGenerator as wng
 
 logger = snapredLogger.getLogger(__name__)
@@ -32,9 +33,9 @@ class GroceryListItem(BaseModel):
     """
 
     # Reserved instrument-cache run-number values:
-    RESERVED_NATIVE_RUNNUMBER: ClassVar[str] = "000000"  # unmodified _native_ instrument:
+    RESERVED_NATIVE_RUNNUMBER: ClassVar[str] = ReservedRunNumber.NATIVE  # unmodified _native_ instrument:
     #   from 'SNAP_Definition.xml'
-    RESERVED_LITE_RUNNUMBER: ClassVar[str] = "000001"  # unmodified _lite_ instrument  :
+    RESERVED_LITE_RUNNUMBER: ClassVar[str] = ReservedRunNumber.LITE  # unmodified _lite_ instrument  :
     #   from 'SNAPLite.xml'
 
     workspaceType: GroceryTypes

src/snapred/backend/data/GroceryService.py

@@ -25,6 +25,7 @@
 from snapred.backend.service.WorkspaceMetadataService import WorkspaceMetadataService
 from snapred.meta.Config import Config
 from snapred.meta.decorators.Singleton import Singleton
+from snapred.meta.InternalConstants import ReservedRunNumber
 from snapred.meta.mantid.WorkspaceNameGenerator import (
     NameBuilder,
     WorkspaceName,
@@ -563,10 +564,7 @@ def _fetchInstrumentDonor(self, runNumber: str, useLiteMode: bool) -> WorkspaceN
 
                 # Initialize the instrument parameters
                 # (Reserved run-numbers will use the unmodified instrument.)
-                if (
-                    runNumber != GroceryListItem.RESERVED_NATIVE_RUNNUMBER
-                    and runNumber != GroceryListItem.RESERVED_LITE_RUNNUMBER
-                ):
+                if runNumber not in ReservedRunNumber.values():
                     detectorState: DetectorState = self._getDetectorState(runNumber)
                     self.updateInstrumentParameters(wsName, detectorState)
             self._loadedInstruments[key] = wsName
@@ -865,13 +863,14 @@ def fetchGroupingDefinition(self, item: GroceryListItem) -> Dict[str, Any]:
 
         :rtype: Dict[str, Any]
         """
-        key = self._key(item.groupingScheme, item.runNumber, item.useLiteMode)
+        stateId, _ = self.dataService.generateStateId(item.runNumber)
+        key = self._key(item.groupingScheme, stateId, item.useLiteMode)
         workspaceName = self._createGroupingWorkspaceName(item.groupingScheme, item.runNumber, item.useLiteMode)
+        workspaceName = self._loadedGroupings.get(key, workspaceName)
 
         self._updateGroupingCacheFromADS(key, workspaceName)
-        groupingIsLoaded = self._loadedGroupings.get(key) is not None
 
-        if groupingIsLoaded:
+        if key in self._loadedGroupings:
             data = {
                 "result": True,
                 "loader": "cached",

src/snapred/backend/data/LocalDataService.py

@@ -51,6 +51,7 @@
 from snapred.meta.Config import Config
 from snapred.meta.decorators.ExceptionHandler import ExceptionHandler
 from snapred.meta.decorators.Singleton import Singleton
+from snapred.meta.InternalConstants import ReservedRunNumber, ReservedStateId
 from snapred.meta.mantid.WorkspaceNameGenerator import (
     ValueFormatter as wnvf,
 )
@@ -288,8 +289,11 @@ def _readPVFile(self, runId: str):
     @lru_cache
     @ExceptionHandler(StateValidationException)
     def generateStateId(self, runId: str) -> Tuple[str, str]:
-        detectorState = self.readDetectorState(runId)
-        SHA = self._stateIdFromDetectorState(detectorState)
+        if runId in ReservedRunNumber.values():
+            SHA = ObjectSHA(hex=ReservedStateId.forRun(runId))
+        else:
+            detectorState = self.readDetectorState(runId)
+            SHA = self._stateIdFromDetectorState(detectorState)
         return SHA.hex, SHA.decodedKey
 
     def _stateIdFromDetectorState(self, detectorState: DetectorState) -> ObjectSHA:

src/snapred/backend/service/ReductionService.py

@@ -81,7 +81,7 @@ def __init__(self):
         self.registerPath("checkWritePermissions", self.checkReductionWritePermissions)
         self.registerPath("getSavePath", self.getSavePath)
         self.registerPath("getStateIds", self.getStateIds)
-        self.registerPath("validateReduction", self.validateReduction)
+        self.registerPath("validate", self.validateReduction)
         self.registerPath("artificialNormalization", self.artificialNormalization)
         self.registerPath("grabWorkspaceforArtificialNorm", self.grabWorkspaceforArtificialNorm)
         return

src/snapred/meta/InternalConstants.py

@@ -0,0 +1,20 @@
+from snapred.backend.dao.ObjectSHA import ObjectSHA
+from snapred.meta.Enum import StrEnum
+
+
+class ReservedRunNumber(StrEnum):
+    NATIVE: str = "000000"
+    LITE: str = "000001"
+
+
+class ReservedStateId(StrEnum):
+    NATIVE: str = ObjectSHA(hex="0000000000000000").hex
+    LITE: str = ObjectSHA(hex="0000000000000001").hex
+
+    @classmethod
+    def forRun(cls, runNumber):
+        match runNumber:
+            case ReservedRunNumber.NATIVE.value:
+                return cls.NATIVE
+            case ReservedRunNumber.LITE.value:
+                return cls.LITE

src/snapred/ui/view/reduction/ReductionRequestView.py

@@ -1,12 +1,12 @@
 from typing import Callable, List, Optional
 
-from qtpy.QtCore import Signal, Slot
+from qtpy.QtCore import Slot
 from qtpy.QtWidgets import (
     QHBoxLayout,
     QLineEdit,
+    QListWidget,
     QMessageBox,
     QPushButton,
-    QTextEdit,
     QVBoxLayout,
 )
 from snapred.backend.dao.state.RunNumber import RunNumber
@@ -20,8 +20,6 @@
 
 @Resettable
 class ReductionRequestView(BackendRequestView):
-    signalRemoveRunNumber = Signal(int)
-
     def __init__(
         self,
         parent=None,
@@ -49,8 +47,8 @@ def __init__(
         self.runNumberLayout.addLayout(self.runNumberButtonLayout)
 
         # Run number display
-        self.runNumberDisplay = QTextEdit()
-        self.runNumberDisplay.setReadOnly(True)
+        self.runNumberDisplay = QListWidget()
+        self.runNumberDisplay.setSortingEnabled(False)
 
         # Lite mode toggle, pixel masks dropdown, and retain unfocused data checkbox
         self.liteModeToggle = self._labeledField("Lite Mode", Toggle(parent=self, state=True))
@@ -78,8 +76,6 @@ def __init__(
         self.enterRunNumberButton.clicked.connect(self.addRunNumber)
         self.clearButton.clicked.connect(self.clearRunNumbers)
 
-        self.signalRemoveRunNumber.connect(self._removeRunNumber)
-
     @Slot()
     def addRunNumber(self):
         # TODO: FIX THIS!
@@ -127,32 +123,20 @@ def parseInputRunNumbers(self) -> List[str]:
 
         return [str(num) for num in runs]
 
-    @Slot()
-    def removeRunNumber(self, runNumber):
-        self.signalRemoveRunNumber.emit(runNumber)
-
-    @Slot()
-    def _removeRunNumber(self, runNumber):
-        if runNumber not in self.runNumbers:
-            logger.warning(
-                f"[ReductionRequestView]: attempting to remove run {runNumber} not in the list {self.runNumbers}"
-            )
-            return
-        self.runNumbers.remove(runNumber)
-        self.updateRunNumberList()
-
     def updateRunNumberList(self):
-        self.runNumberDisplay.setText("\n".join(map(str, sorted(self.runNumbers))))
+        self.runNumberDisplay.clear()
+        self.runNumberDisplay.addItems(self.runNumbers)
 
     def clearRunNumbers(self):
         self.runNumbers.clear()
         self.runNumberDisplay.clear()
 
     def verify(self):
-        currentText = self.runNumberDisplay.toPlainText()
-        runNumbers = [num.strip() for num in currentText.split("\n") if num.strip()]
+        runNumbers = [self.runNumberDisplay.item(x).text() for x in range(self.runNumberDisplay.count())]
         if not runNumbers:
             raise ValueError("Please enter at least one run number.")
+        if runNumbers != self.runNumbers:
+            raise ValueError("Unexpected issue verifying run numbers.  Please clear and re-enter.")
         for runNumber in runNumbers:
             if not runNumber.isdigit():
                 raise ValueError(

src/snapred/ui/workflow/ReductionWorkflow.py

@@ -58,7 +58,7 @@ def __init__(self, parent=None):
             )
             .build()
         )
-
+        self._keeps = set()
         self._reductionRequestView.retainUnfocusedDataCheckbox.checkedChanged.connect(self._enableConvertToUnits)
         self._artificialNormalizationView.signalValueChanged.connect(self.onArtificialNormalizationValueChange)
 
@@ -96,7 +96,7 @@ def _populatePixelMaskDropdown(self):
             return
 
         runNumbers = self._reductionRequestView.getRunNumbers()
-        useLiteMode = self._reductionRequestView.liteModeToggle.field.getState()  # noqa: F841
+        self.useLiteMode = self._reductionRequestView.liteModeToggle.field.getState()  # noqa: F841
 
         self._reductionRequestView.liteModeToggle.setEnabled(False)
         self._reductionRequestView.pixelMaskDropdown.setEnabled(False)
@@ -111,7 +111,7 @@ def _populatePixelMaskDropdown(self):
                 payload=ReductionRequest(
                     # All runNumbers are from the same state => any one can be used here
                     runNumber=runNumbers[0],
-                    useLiteMode=useLiteMode,
+                    useLiteMode=self.useLiteMode,
                 ),
             ).data
         except Exception as e:  # noqa: BLE001
@@ -126,7 +126,6 @@ def _populatePixelMaskDropdown(self):
         self._reductionRequestView.liteModeToggle.setEnabled(True)
         self._reductionRequestView.pixelMaskDropdown.setEnabled(True)
         self._reductionRequestView.retainUnfocusedDataCheckbox.setEnabled(True)
-        # self._reductionRequestView.convertUnitsDropdown.setEnabled(True)
 
     def _validateRunNumbers(self, runNumbers: List[str]):
         # For now, all run numbers in a reduction batch must be from the same instrument state.
@@ -153,28 +152,41 @@ def _onPixelMaskSelection(self):
         #
         # ## Why would I want to set the ~obfuscated-by-pydantic~ `pixelMasks` field of the class object?
 
+    def _createReductionRequest(self, runNumber, artificialNormalizationIngredients=None):
+        """
+        Create a standardized ReductionRequest object for passing to the ReductionService
+        """
+        return ReductionRequest(
+            runNumber=str(runNumber),
+            useLiteMode=self.useLiteMode,
+            timestamp=self.timestamp,
+            continueFlags=self.continueAnywayFlags,
+            pixelMasks=self.pixelMasks,
+            keepUnfocused=self._reductionRequestView.retainUnfocusedDataCheckbox.isChecked(),
+            convertUnitsTo=self._reductionRequestView.convertUnitsDropdown.currentText(),
+            artificialNormalizationIngredients=artificialNormalizationIngredients,
+        )
+
     def _triggerReduction(self, workflowPresenter):
         view = workflowPresenter.widget.tabView  # noqa: F841
 
-        runNumbers = self._reductionRequestView.getRunNumbers()
-        pixelMasks = self._reconstructPixelMaskNames(self._reductionRequestView.getPixelMasks())
+        self.runNumbers = self._reductionRequestView.getRunNumbers()
+        self.pixelMasks = self._reconstructPixelMaskNames(self._reductionRequestView.getPixelMasks())
 
         # Use one timestamp for the entire set of runNumbers:
-        timestamp = self.request(path="reduction/getUniqueTimestamp").data
-        for runNumber in runNumbers:
+        self.timestamp = self.request(path="reduction/getUniqueTimestamp").data
+
+        # all runs in same state, use the first run to load groupings
+        request_ = self._createReductionRequest(self.runNumbers[0])
+        response = self.request(path="reduction/groupings", payload=request_)
+        self._keeps = set(response.data["groupingWorkspaces"])
+
+        for runNumber in self.runNumbers:
             self._artificialNormalizationView.updateRunNumber(runNumber)
-            request_ = ReductionRequest(
-                runNumber=str(runNumber),
-                useLiteMode=self._reductionRequestView.liteModeToggle.field.getState(),
-                timestamp=timestamp,
-                continueFlags=self.continueAnywayFlags,
-                pixelMasks=pixelMasks,
-                keepUnfocused=self._reductionRequestView.retainUnfocusedDataCheckbox.isChecked(),
-                convertUnitsTo=self._reductionRequestView.convertUnitsDropdown.currentText(),
-            )
+            request_ = self._createReductionRequest(runNumber)
 
             # Validate reduction; if artificial normalization is needed, handle it
-            response = self.request(path="reduction/validateReduction", payload=request_)
+            response = self.request(path="reduction/validate", payload=request_)
             if ContinueWarning.Type.MISSING_NORMALIZATION in self.continueAnywayFlags:
                 self._artificialNormalizationView.showAdjustView()
                 response = self.request(path="reduction/grabWorkspaceforArtificialNorm", payload=request_)
@@ -186,15 +198,22 @@ def _triggerReduction(self, workflowPresenter):
                 if response.code == ResponseCode.OK:
                     record, unfocusedData = response.data.record, response.data.unfocusedData
                     self._finalizeReduction(record, unfocusedData)
-                    workflowPresenter.advanceWorkflow()
+            # after each run, clean workspaces except groupings, calibrations, normalizations, and outputs
+            self._keeps.update(self.outputs)
+            self._clearWorkspaces(exclude=self._keeps, clearCachedWorkspaces=True)
+        workflowPresenter.advanceWorkflow()
+        # SPECIAL FOR THE REDUCTION WORKFLOW: clear everything _except_ the output workspaces
+        #   _before_ transitioning to the "save" panel.
+        # TODO: make '_clearWorkspaces' a public method (i.e make this combination a special `cleanup` method).
+        self._clearWorkspaces(exclude=self.outputs, clearCachedWorkspaces=True)
         return self.responses[-1]
 
     def _artificialNormalization(self, workflowPresenter, responseData, runNumber):
         """Handles artificial normalization for the workflow."""
         view = workflowPresenter.widget.tabView  # noqa: F841
         request_ = CreateArtificialNormalizationRequest(
             runNumber=runNumber,
-            useLiteMode=self._reductionRequestView.liteModeToggle.field.getState(),
+            useLiteMode=self.useLiteMode,
             peakWindowClippingSize=int(self._artificialNormalizationView.peakWindowClippingSize.field.text()),
             smoothingParameter=self._artificialNormalizationView.getSmoothingParameter(),
             decreaseParameter=self._artificialNormalizationView.decreaseParameterDropdown.currentIndex() == 1,
@@ -219,7 +238,7 @@ def onArtificialNormalizationValueChange(self, smoothingValue, lss, decreasePara
 
         request_ = CreateArtificialNormalizationRequest(
             runNumber=runNumber,
-            useLiteMode=self._reductionRequestView.liteModeToggle.field.getState(),
+            useLiteMode=self.useLiteMode,
             peakWindowClippingSize=peakWindowClippingSize,
             smoothingParameter=smoothingValue,
             decreaseParameter=decreaseParameter,
@@ -276,11 +295,6 @@ def _finalizeReduction(self, record, unfocusedData):
             # Note that the run number is deliberately not deleted from the run numbers list.
             # Almost certainly it should be moved to a "completed run numbers" list.
 
-        # SPECIAL FOR THE REDUCTION WORKFLOW: clear everything _except_ the output workspaces
-        #   _before_ transitioning to the "save" panel.
-        # TODO: make '_clearWorkspaces' a public method (i.e make this combination a special `cleanup` method).
-        self._clearWorkspaces(exclude=self.outputs, clearCachedWorkspaces=True)
-
     @property
     def widget(self):
         return self.workflow.presenter.widget

tests/integration/test_workflow_panels_happy_path.py

@@ -607,8 +607,10 @@ def test_calibration_and_reduction_panels_happy_path(
             #    enter a "Run Number":
             requestView.runNumberInput.setText(reductionRunNumber)
             qtbot.mouseClick(requestView.enterRunNumberButton, Qt.MouseButton.LeftButton)
-            _currentText = requestView.runNumberDisplay.toPlainText()
-            _runNumbers = [num.strip() for num in _currentText.split("\n") if num.strip()]
+
+            _count = requestView.runNumberDisplay.count()
+            _runNumbers = [requestView.runNumberDisplay.item(x).text() for x in range(_count)]
+
             assert reductionRunNumber in _runNumbers
 
             """
@@ -1257,8 +1259,8 @@ def completionMessageBoxAssert(*args, **kwargs):  # noqa: ARG001
             requestView.runNumberInput.setText(reductionRunNumber)
             qtbot.mouseClick(requestView.enterRunNumberButton, Qt.MouseButton.LeftButton)
 
-            _currentText = requestView.runNumberDisplay.toPlainText()
-            _runNumbers = [num.strip() for num in _currentText.split("\n") if num.strip()]
+            _count = requestView.runNumberDisplay.count()
+            _runNumbers = [requestView.runNumberDisplay.item(x).text() for x in range(_count)]
 
             assert reductionRunNumber in _runNumbers