Ewm7782 diffcal residual addtion (#493)

* Integration for more debugging on analysis.

* Final updates after debugging on analysis

* Code cov and failing integration test

* Trying something else for integration test

* Update to CalculateResidualDiffCalAlgo.py

* Update to recalculate residual correctly when fitted peaks change.

* Updated to Recipe instead of Algo and made other requested changes

src/snapred/backend/dao/ingredients/CalculateDiffCalResidualIngredients.py

@@ -0,0 +1,14 @@
+from pydantic import BaseModel, ConfigDict
+
+from snapred.meta.mantid.WorkspaceNameGenerator import WorkspaceName
+
+
+class CalculateDiffCalResidualIngredients(BaseModel):
+    inputWorkspace: WorkspaceName
+    outputWorkspace: WorkspaceName
+    fitPeaksDiagnosticWorkspace: WorkspaceName
+
+    model_config = ConfigDict(
+        # required in order to use 'WorkspaceName'
+        arbitrary_types_allowed=True,
+    )

src/snapred/backend/dao/request/CalculateResidualRequest.py

@@ -0,0 +1,14 @@
+from pydantic import BaseModel, ConfigDict
+
+from snapred.meta.mantid.WorkspaceNameGenerator import WorkspaceName
+
+
+class CalculateResidualRequest(BaseModel):
+    inputWorkspace: WorkspaceName
+    outputWorkspace: WorkspaceName
+    fitPeaksDiagnostic: WorkspaceName
+
+    model_config = ConfigDict(
+        # required in order to use 'WorkspaceName'
+        arbitrary_types_allowed=True,
+    )

src/snapred/backend/recipe/CalculateDiffCalResidualRecipe.py

@@ -0,0 +1,133 @@
+from typing import Dict
+
+import numpy as np
+from pydantic import BaseModel
+
+from snapred.backend.dao.ingredients.CalculateDiffCalResidualIngredients import (
+    CalculateDiffCalResidualIngredients as Ingredients,
+)
+from snapred.backend.log.logger import snapredLogger
+from snapred.backend.recipe.algorithm.Utensils import Utensils
+from snapred.backend.recipe.Recipe import Recipe
+from snapred.meta.mantid.WorkspaceNameGenerator import WorkspaceName
+
+logger = snapredLogger.getLogger(__name__)
+
+
+class CalculateDiffCalServing(BaseModel):
+    outputWorkspace: str
+
+
+class CalculateDiffCalResidualRecipe(Recipe[Ingredients]):
+    def __init__(self, utensils: Utensils = None):
+        if utensils is None:
+            utensils = Utensils()
+            utensils.PyInit()
+        self.mantidSnapper = utensils.mantidSnapper
+        self._counts = 0
+
+    def logger(self):
+        return logger
+
+    def validateInputs(self, ingredients: Ingredients, groceries: Dict[str, WorkspaceName]):
+        super().validateInputs(ingredients, groceries)
+
+    def chopIngredients(self, ingredients: Ingredients) -> None:
+        """Receive the ingredients from the recipe."""
+        self.inputWorkspaceName = ingredients.inputWorkspace
+        self.outputWorkspaceName = ingredients.outputWorkspace
+        inputGroupWorkspace = ingredients.fitPeaksDiagnosticWorkspace
+
+        fitPeaksGroupWorkspace = self.mantidSnapper.mtd[inputGroupWorkspace]
+        lastWorkspaceName = fitPeaksGroupWorkspace.getNames()[-1]
+        self.fitPeaksDiagnosticWorkSpaceName = lastWorkspaceName
+
+    def unbagGroceries(self):
+        pass
+
+    def prep(self, ingredients: Ingredients):
+        """
+        Convenience method to prepare the recipe for execution.
+        """
+        self.validateInputs(ingredients, groceries=None)
+        self.chopIngredients(ingredients)
+        self.unbagGroceries()
+        self.stirInputs()
+        self.queueAlgos()
+
+    def queueAlgos(self):
+        # Step 1: Clone the input workspace to initialize the output workspace
+        self.mantidSnapper.CloneWorkspace(
+            f"Creating outputWorkspace: {self.outputWorkspaceName}...",
+            InputWorkspace=self.inputWorkspaceName,
+            OutputWorkspace=self.outputWorkspaceName,
+        )
+
+        # Step 2: Check for overlapping spectra and manage them
+        fitPeaksWorkspace = self.mantidSnapper.mtd[self.fitPeaksDiagnosticWorkSpaceName]
+        numHistograms = fitPeaksWorkspace.getNumberHistograms()
+        processedSpectra = []
+        spectrumDict = {}
+
+        for i in range(numHistograms):
+            spectrumId = fitPeaksWorkspace.getSpectrum(i).getSpectrumNo()
+            singleSpectrumName = f"{self.fitPeaksDiagnosticWorkSpaceName}_spectrum_{spectrumId}"
+
+            # If this spectrum number is already processed, average with existing
+            if spectrumId in spectrumDict:
+                existingName = spectrumDict[spectrumId]
+                self.mantidSnapper.Plus(
+                    f"Averaging overlapping spectrum {spectrumId}...",
+                    LHSWorkspace=existingName,
+                    RHSWorkspace=singleSpectrumName,
+                    OutputWorkspace=singleSpectrumName,
+                )
+            else:
+                # Extract spectrum by position
+                self.mantidSnapper.ExtractSingleSpectrum(
+                    f"Extracting spectrum with SpectrumNumber {spectrumId}...",
+                    InputWorkspace=self.fitPeaksDiagnosticWorkSpaceName,
+                    OutputWorkspace=singleSpectrumName,
+                    WorkspaceIndex=i,
+                )
+
+                # Replace zero values with NaN
+                self.mantidSnapper.ReplaceSpecialValues(
+                    f"Replacing zeros with NaN in spectrum with SpectrumNumber {spectrumId}...",
+                    InputWorkspace=singleSpectrumName,
+                    OutputWorkspace=singleSpectrumName,
+                    SmallNumberThreshold=1e-10,
+                    SmallNumberValue=np.nan,
+                )
+
+                spectrumDict[spectrumId] = singleSpectrumName
+
+            # Append the processed spectrum to the list
+            processedSpectra.append(singleSpectrumName)
+
+        # Step 3: Combine all processed spectra into a single workspace
+        combinedWorkspace = processedSpectra[0]
+        for spectrum in processedSpectra[1:]:
+            self.mantidSnapper.ConjoinWorkspaces(
+                f"Combining spectrum {spectrum}...", InputWorkspace1=combinedWorkspace, InputWorkspace2=spectrum
+            )
+
+        # Step 4: Calculate the residual difference between the combined workspace and input workspace
+        self.mantidSnapper.Minus(
+            f"Subtracting {combinedWorkspace} from {self.inputWorkspaceName}...",
+            LHSWorkspace=combinedWorkspace,
+            RHSWorkspace=self.inputWorkspaceName,
+            OutputWorkspace=self.outputWorkspaceName,
+        )
+
+    def execute(self):
+        self.mantidSnapper.executeQueue()
+        # Set the output property to the final residual workspace
+        self.outputWorkspace = self.mantidSnapper.mtd[self.outputWorkspaceName]
+
+    def cook(self, ingredients: Ingredients):
+        self.prep(ingredients)
+        self.execute()
+        return CalculateDiffCalServing(
+            outputWorkspace=self.outputWorkspaceName,
+        )

src/snapred/backend/recipe/Recipe.py

@@ -98,13 +98,21 @@ def validateInputs(self, ingredients: Ingredients, groceries: Dict[str, Workspac
         """
         Validate the input properties before chopping or unbagging
         """
-        self._validateIngredients(ingredients)
+        if ingredients is not None:
+            self._validateIngredients(ingredients)
+        else:
+            logger.info("No ingredients given, skipping ingredient validation")
+            pass
         # ensure all of the given workspaces exist
         # NOTE may need to be tweaked to ignore output workspaces...
-        logger.info(f"Validating the given workspaces: {groceries.values()}")
-        for key in self.mandatoryInputWorkspaces():
-            ws = groceries.get(key)
-            self._validateGrocery(key, ws)
+        if groceries is not None:
+            logger.info(f"Validating the given workspaces: {groceries.values()}")
+            for key in self.mandatoryInputWorkspaces():
+                ws = groceries.get(key)
+                self._validateGrocery(key, ws)
+        else:
+            logger.info("No groceries given, skipping workspace validation")
+            pass
 
     def stirInputs(self):
         """

src/snapred/backend/service/CalibrationService.py

@@ -10,11 +10,13 @@
 )
 from snapred.backend.dao.indexing import IndexEntry
 from snapred.backend.dao.ingredients import (
+    CalculateDiffCalResidualIngredients,
     CalibrationMetricsWorkspaceIngredients,
     DiffractionCalibrationIngredients,
     GroceryListItem,
 )
 from snapred.backend.dao.request import (
+    CalculateResidualRequest,
     CalibrationAssessmentRequest,
     CalibrationExportRequest,
     CalibrationIndexRequest,
@@ -34,6 +36,7 @@
 from snapred.backend.data.DataFactoryService import DataFactoryService
 from snapred.backend.data.GroceryService import GroceryService
 from snapred.backend.log.logger import snapredLogger
+from snapred.backend.recipe.CalculateDiffCalResidualRecipe import CalculateDiffCalResidualRecipe
 from snapred.backend.recipe.GenerateCalibrationMetricsWorkspaceRecipe import GenerateCalibrationMetricsWorkspaceRecipe
 from snapred.backend.recipe.GenericRecipe import (
     CalibrationMetricExtractionRecipe,
@@ -94,6 +97,7 @@ def __init__(self):
         self.registerPath("diffraction", self.diffractionCalibration)
         self.registerPath("diffractionWithIngredients", self.diffractionCalibrationWithIngredients)
         self.registerPath("validateWritePermissions", self.validateWritePermissions)
+        self.registerPath("residual", self.calculateResidual)
         return
 
     @staticmethod
@@ -248,6 +252,15 @@ def validateWritePermissions(self, request: CalibrationWritePermissionsRequest):
                 + "</font>"
             )
 
+    @FromString
+    def calculateResidual(self, request: CalculateResidualRequest):
+        ingredients = CalculateDiffCalResidualIngredients(
+            inputWorkspace=request.inputWorkspace,
+            outputWorkspace=request.outputWorkspace,
+            fitPeaksDiagnosticWorkspace=request.fitPeaksDiagnostic,
+        )
+        return CalculateDiffCalResidualRecipe().cook(ingredients)
+
     @FromString
     def focusSpectra(self, request: FocusSpectraRequest):
         # prep the ingredients -- a pixel group

src/snapred/ui/view/DiffCalTweakPeakView.py

@@ -203,7 +203,7 @@ def emitPurge(self):
             return
         self.signalPurgeBadPeaks.emit(maxChiSq)
 
-    def updateGraphs(self, workspace, peaks, diagnostic):
+    def updateGraphs(self, workspace, peaks, diagnostic, residual):
         # get the updated workspaces and optimal graph grid
         self.peaks = pydantic.TypeAdapter(List[GroupPeakList]).validate_python(peaks)
         numGraphs = len(peaks)
@@ -228,24 +228,37 @@ def updateGraphs(self, workspace, peaks, diagnostic):
             ax = self.figure.add_subplot(nrows, ncols, wkspIndex + 1, projection="mantid")
             ax.tick_params(direction="in")
             ax.set_title(f"Group ID: {wkspIndex + 1}")
-            # plot the data and fitted
+            # plot the data and fitted curve
             ax.plot(mtd[workspace], wkspIndex=wkspIndex, label="data", normalize_by_bin_width=True)
             ax.plot(fitted_peaks, wkspIndex=wkspIndex, label="fit", color="black", normalize_by_bin_width=True)
+
+            # plot the residual data
+            ax.plot(
+                mtd[residual],
+                wkspIndex=wkspIndex,
+                label="residual",
+                color="limegreen",
+                linewidth=2,
+                normalize_by_bin_width=True,
+            )
+
             ax.legend(loc=1)
+
             # fill in the discovered peaks for easier viewing
             x, y, _, _ = get_spectrum(mtd[workspace], wkspIndex, normalize_by_bin_width=True)
             # for each detected peak in this group, shade in the peak region
             for chi2, peak in zip(chisq, peaks):
                 # areas inside peak bounds (to be shaded)
                 under_peaks = [(peak.minimum < xx and xx < peak.maximum) for xx in x]
                 # the color: blue = GOOD, red = BAD
-                color = "blue" if chi2 < float(self.maxChiSqField.text()) else "red"
-                alpha = 0.3 if chi2 < float(self.maxChiSqField.text()) else 0.8
+                color = "blue" if chi2 < maxChiSq else "red"
+                alpha = 0.3 if chi2 < maxChiSq else 0.8
                 # now shade
                 ax.fill_between(x, y, where=under_peaks, color=color, alpha=alpha)
             # plot the min and max value for peaks
             ax.axvline(x=max(min(x), float(self.fieldXtalDMin.field.text())), label="xtal $d_{min}$", color="red")
             ax.axvline(x=min(max(x), float(self.fieldXtalDMax.field.text())), label="xtal $d_{max}$", color="red")
+
         # resize window and redraw
         self.setMinimumHeight(
             self.initialLayoutHeight + int((self.figure.get_size_inches()[1] + self.FIGURE_MARGIN) * self.figure.dpi)

src/snapred/ui/workflow/DiffCalWorkflow.py

@@ -9,6 +9,7 @@
 from snapred.backend.dao.indexing.Versioning import VersionedObject
 from snapred.backend.dao.Limit import Pair
 from snapred.backend.dao.request import (
+    CalculateResidualRequest,
     CalibrationAssessmentRequest,
     CalibrationExportRequest,
     CalibrationWritePermissionsRequest,
@@ -27,7 +28,9 @@
 from snapred.meta.Config import Config
 from snapred.meta.decorators.ExceptionToErrLog import ExceptionToErrLog
 from snapred.meta.mantid.AllowedPeakTypes import SymmetricPeakEnum
-from snapred.meta.mantid.WorkspaceNameGenerator import WorkspaceType as wngt
+from snapred.meta.mantid.WorkspaceNameGenerator import (
+    WorkspaceType as wngt,
+)
 from snapred.ui.presenter.WorkflowPresenter import WorkflowPresenter
 from snapred.ui.view.DiffCalAssessmentView import DiffCalAssessmentView
 from snapred.ui.view.DiffCalRequestView import DiffCalRequestView
@@ -249,15 +252,17 @@ def _specifyRun(self, workflowPresenter):
         self.prevFWHM = payload.fwhmMultipliers  # NOTE set in __init__ to defaults
         self.prevGroupingIndex = view.groupingFileDropdown.currentIndex()
         self.fitPeaksDiagnostic = f"fit_peak_diag_{self.runNumber}_{self.prevGroupingIndex}_pre"
-
+        self.residualWorkspace = f"diffcal_residual_{self.runNumber}"
         # focus the workspace to view the peaks
         self._renewFocus(self.prevGroupingIndex)
-        response = self._renewFitPeaks(self.peakFunction)
+        self._renewFitPeaks(self.peakFunction)
+        response = self._calculateResidual()
 
         self._tweakPeakView.updateGraphs(
             self.focusedWorkspace,
             self.ingredients.groupedPeakLists,
             self.fitPeaksDiagnostic,
+            self.residualWorkspace,
         )
         return response
 
@@ -288,18 +293,21 @@ def onValueChange(self, groupingIndex, xtalDMin, xtalDMax, peakFunction, fwhm, m
             ):
                 self._renewIngredients(xtalDMin, xtalDMax, peakFunction, fwhm, maxChiSq)
                 self._renewFitPeaks(peakFunction)
+                self._calculateResidual()
                 self.peaksWerePurged = False
 
             # if the grouping file changes, load new grouping and refocus
             if groupingIndex != self.prevGroupingIndex:
                 self._renewIngredients(xtalDMin, xtalDMax, peakFunction, fwhm, maxChiSq)
                 self._renewFocus(groupingIndex)
                 self._renewFitPeaks(peakFunction)
+                self._calculateResidual()
 
             self._tweakPeakView.updateGraphs(
                 self.focusedWorkspace,
                 self.ingredients.groupedPeakLists,
                 self.fitPeaksDiagnostic,
+                self.residualWorkspace,
             )
 
             # update the values for next call to this method
@@ -356,7 +364,16 @@ def _renewFitPeaks(self, peakFunction):
             detectorPeaks=self.ingredients.groupedPeakLists,
             peakFunction=peakFunction,
         )
-        return self.request(path="calibration/fitpeaks", payload=payload.json())
+        response = self.request(path="calibration/fitpeaks", payload=payload.json())
+        return response
+
+    def _calculateResidual(self):
+        payload = CalculateResidualRequest(
+            inputWorkspace=self.focusedWorkspace,
+            outputWorkspace=self.residualWorkspace,
+            fitPeaksDiagnostic=self.fitPeaksDiagnostic,
+        )
+        return self.request(path="calibration/residual", payload=payload.json())
 
     @ExceptionToErrLog
     @Slot(float)

tests/integration/test_workflow_panels_happy_path.py

@@ -892,11 +892,11 @@ def test_diffraction_calibration_panel_happy_path(self, qtbot, qapp, calibration
             qtbot.wait(10000)
 
             #    continue to the next panel
-            with qtbot.waitSignal(actionCompleted, timeout=60000):
+            with qtbot.waitSignal(actionCompleted, timeout=80000):
                 qtbot.mouseClick(workflowNodeTabs.currentWidget().continueButton, Qt.MouseButton.LeftButton)
 
             qtbot.waitUntil(
-                lambda: isinstance(workflowNodeTabs.currentWidget().view, DiffCalAssessmentView), timeout=60000
+                lambda: isinstance(workflowNodeTabs.currentWidget().view, DiffCalAssessmentView), timeout=80000
             )
 
             # Placing this next `stop` correctly, causes some difficulty:
@@ -909,7 +909,7 @@ def test_diffraction_calibration_panel_happy_path(self, qtbot, qapp, calibration
             #    nothing to do here, for this test
 
             #    continue to the next panel
-            with qtbot.waitSignal(actionCompleted, timeout=60000):
+            with qtbot.waitSignal(actionCompleted, timeout=80000):
                 qtbot.mouseClick(workflowNodeTabs.currentWidget().continueButton, Qt.MouseButton.LeftButton)
 
             qtbot.waitUntil(lambda: isinstance(workflowNodeTabs.currentWidget().view, DiffCalSaveView), timeout=5000)