Merge branch 'next' into ewm6381-fix-pixel-masks-dropdown

src/snapred/backend/dao/ingredients/DiffractionCalibrationIngredients.py

@@ -28,5 +28,4 @@ class DiffractionCalibrationIngredients(BaseModel, extra="forbid"):
     peakFunction: SymmetricPeakEnum = SymmetricPeakEnum[Config["calibration.diffraction.peakFunction"]]
     maxOffset: float = Config["calibration.diffraction.maximumOffset"]
     maxChiSq: float = Config["constants.GroupDiffractionCalibration.MaxChiSq"]
-    skipPixelCalibration: bool = False
     removeBackground: bool = False

src/snapred/backend/dao/ingredients/GroceryListItem.py

@@ -3,6 +3,7 @@
 from pydantic import BaseModel, model_validator
 
 from snapred.backend.log.logger import snapredLogger
+from snapred.meta.InternalConstants import ReservedRunNumber
 from snapred.meta.mantid.WorkspaceNameGenerator import WorkspaceNameGenerator as wng
 
 logger = snapredLogger.getLogger(__name__)
@@ -32,9 +33,9 @@ class GroceryListItem(BaseModel):
     """
 
     # Reserved instrument-cache run-number values:
-    RESERVED_NATIVE_RUNNUMBER: ClassVar[str] = "000000"  # unmodified _native_ instrument:
+    RESERVED_NATIVE_RUNNUMBER: ClassVar[str] = ReservedRunNumber.NATIVE  # unmodified _native_ instrument:
     #   from 'SNAP_Definition.xml'
-    RESERVED_LITE_RUNNUMBER: ClassVar[str] = "000001"  # unmodified _lite_ instrument  :
+    RESERVED_LITE_RUNNUMBER: ClassVar[str] = ReservedRunNumber.LITE  # unmodified _lite_ instrument  :
     #   from 'SNAPLite.xml'
 
     workspaceType: GroceryTypes

src/snapred/backend/dao/request/DiffractionCalibrationRequest.py

@@ -3,6 +3,7 @@
 
 from pydantic import BaseModel, field_validator
 
+from snapred.backend.dao.indexing.Versioning import VERSION_DEFAULT
 from snapred.backend.dao.Limit import Pair
 from snapred.backend.dao.state.FocusGroup import FocusGroup
 from snapred.backend.error.ContinueWarning import ContinueWarning
@@ -35,11 +36,12 @@ class DiffractionCalibrationRequest(BaseModel, extra="forbid"):
     maximumOffset: float = Config["calibration.diffraction.maximumOffset"]
     fwhmMultipliers: Pair[float] = Pair.model_validate(Config["calibration.parameters.default.FWHMMultiplier"])
     maxChiSq: float = Config["constants.GroupDiffractionCalibration.MaxChiSq"]
-    skipPixelCalibration: bool = False
     removeBackground: bool = False
 
     continueFlags: Optional[ContinueWarning.Type] = ContinueWarning.Type.UNSET
 
+    startingTableVersion: int = VERSION_DEFAULT
+
     @field_validator("fwhmMultipliers", mode="before")
     @classmethod
     def validate_fwhmMultipliers(cls, v: Any) -> Pair[float]:

src/snapred/backend/dao/request/SimpleDiffCalRequest.py

@@ -8,4 +8,3 @@
 class SimpleDiffCalRequest(BaseModel):
     ingredients: DiffractionCalibrationIngredients
     groceries: Dict[str, str]
-    skipPixelCalibration: bool

src/snapred/backend/data/GroceryService.py

@@ -25,6 +25,7 @@
 from snapred.backend.service.WorkspaceMetadataService import WorkspaceMetadataService
 from snapred.meta.Config import Config
 from snapred.meta.decorators.Singleton import Singleton
+from snapred.meta.InternalConstants import ReservedRunNumber
 from snapred.meta.mantid.WorkspaceNameGenerator import (
     NameBuilder,
     WorkspaceName,
@@ -563,10 +564,7 @@ def _fetchInstrumentDonor(self, runNumber: str, useLiteMode: bool) -> WorkspaceN
 
                 # Initialize the instrument parameters
                 # (Reserved run-numbers will use the unmodified instrument.)
-                if (
-                    runNumber != GroceryListItem.RESERVED_NATIVE_RUNNUMBER
-                    and runNumber != GroceryListItem.RESERVED_LITE_RUNNUMBER
-                ):
+                if runNumber not in ReservedRunNumber.values():
                     detectorState: DetectorState = self._getDetectorState(runNumber)
                     self.updateInstrumentParameters(wsName, detectorState)
             self._loadedInstruments[key] = wsName
@@ -865,13 +863,14 @@ def fetchGroupingDefinition(self, item: GroceryListItem) -> Dict[str, Any]:
 
         :rtype: Dict[str, Any]
         """
-        key = self._key(item.groupingScheme, item.runNumber, item.useLiteMode)
+        stateId, _ = self.dataService.generateStateId(item.runNumber)
+        key = self._key(item.groupingScheme, stateId, item.useLiteMode)
         workspaceName = self._createGroupingWorkspaceName(item.groupingScheme, item.runNumber, item.useLiteMode)
+        workspaceName = self._loadedGroupings.get(key, workspaceName)
 
         self._updateGroupingCacheFromADS(key, workspaceName)
-        groupingIsLoaded = self._loadedGroupings.get(key) is not None
 
-        if groupingIsLoaded:
+        if key in self._loadedGroupings:
             data = {
                 "result": True,
                 "loader": "cached",

src/snapred/backend/data/LocalDataService.py

@@ -51,6 +51,7 @@
 from snapred.meta.Config import Config
 from snapred.meta.decorators.ExceptionHandler import ExceptionHandler
 from snapred.meta.decorators.Singleton import Singleton
+from snapred.meta.InternalConstants import ReservedRunNumber, ReservedStateId
 from snapred.meta.mantid.WorkspaceNameGenerator import (
     ValueFormatter as wnvf,
 )
@@ -288,8 +289,11 @@ def _readPVFile(self, runId: str):
     @lru_cache
     @ExceptionHandler(StateValidationException)
     def generateStateId(self, runId: str) -> Tuple[str, str]:
-        detectorState = self.readDetectorState(runId)
-        SHA = self._stateIdFromDetectorState(detectorState)
+        if runId in ReservedRunNumber.values():
+            SHA = ObjectSHA(hex=ReservedStateId.forRun(runId))
+        else:
+            detectorState = self.readDetectorState(runId)
+            SHA = self._stateIdFromDetectorState(detectorState)
         return SHA.hex, SHA.decodedKey
 
     def _stateIdFromDetectorState(self, detectorState: DetectorState) -> ObjectSHA:

src/snapred/backend/recipe/PixelDiffCalRecipe.py

@@ -81,6 +81,7 @@ def unbagGroceries(self, groceries: Dict[str, WorkspaceName]) -> None:  # noqa A
         self.maskWS = groceries["maskWorkspace"]
         # the name of the output calibration table
         self.DIFCpixel = groceries["calibrationTable"]
+        self.DIFCprev = groceries.get("previousCalibration", "")
         self.isEventWs = self.mantidSnapper.mtd[self.wsTOF].id() == "EventWorkspace"
         # the input data converted to d-spacing
         self.wsDSP = wng.diffCalInputDSP().runNumber(self.runNumber).build()
@@ -100,13 +101,20 @@ def unbagGroceries(self, groceries: Dict[str, WorkspaceName]) -> None:  # noqa A
                 OutputWorkspace=self.wsDSP + "_withoutBackground",
             )
 
-        self.mantidSnapper.CalculateDiffCalTable(
-            "Calculate initial table of DIFC values",
-            InputWorkspace=self.wsTOF,
-            CalibrationTable=self.DIFCpixel,
-            OffsetMode="Signed",
-            BinWidth=self.dBin,
-        )
+        if self.DIFCprev == "":
+            self.mantidSnapper.CalculateDiffCalTable(
+                "Calculate initial table of DIFC values",
+                InputWorkspace=self.wsTOF,
+                CalibrationTable=self.DIFCpixel,
+                OffsetMode="Signed",
+                BinWidth=self.dBin,
+            )
+        else:
+            self.mantidSnapper.RenameWorkspace(
+                "Begin DIFC table at previous",
+                InputWorkspace=self.DIFCprev,
+                OutputWorkspace=self.DIFCpixel,
+            )
 
     def stirInputs(self):
         self.groupWorkspaceIndices = self.mantidSnapper.GroupedDetectorIDs(

src/snapred/backend/recipe/algorithm/LoadCalibrationWorkspaces.py

@@ -136,6 +136,11 @@ def PyExec(self) -> None:
         self.mantidSnapper.executeQueue()
 
         if self.calibrationTable:
+            # NOTE ConvertDiffCal has issues ifthe "tofmin" column is present,
+            # and LoadFiffCal is written to always add this column on load.
+            # Easiest temporary solution is to delete this column, until mantid is updated.
+            calTab = self.mantidSnapper.mtd[self.calibrationTable]
+            calTab.removeColumn("tofmin")
             self.setPropertyValue("CalibrationTable", self.calibrationTable)
         if self.maskWorkspace:
             self.setPropertyValue("MaskWorkspace", self.maskWorkspace)

src/snapred/backend/service/CalibrationService.py

@@ -95,7 +95,8 @@ def __init__(self):
         self.registerPath("loadQualityAssessment", self.loadQualityAssessment)
         self.registerPath("index", self.getCalibrationIndex)
         self.registerPath("diffraction", self.diffractionCalibration)
-        self.registerPath("diffractionWithIngredients", self.diffractionCalibrationWithIngredients)
+        self.registerPath("pixel", self.pixelCalibration)
+        self.registerPath("group", self.groupCalibration)
         self.registerPath("validateWritePermissions", self.validateWritePermissions)
         self.registerPath("residual", self.calculateResidual)
         return
@@ -135,6 +136,10 @@ def fetchDiffractionCalibrationGroceries(self, request: DiffractionCalibrationRe
         self.groceryClerk.name("groupingWorkspace").fromRun(request.runNumber).grouping(
             request.focusGroup.name
         ).useLiteMode(request.useLiteMode).add()
+        self.groceryClerk.name("previousCalibration").diffcal_table(
+            request.runNumber, request.startingTableVersion
+        ).useLiteMode(request.useLiteMode).add()
+        # names
         diffcalOutputName = (
             wng.diffCalOutput().unit(wng.Units.DSP).runNumber(request.runNumber).group(request.focusGroup.name).build()
         )
@@ -155,21 +160,18 @@ def fetchDiffractionCalibrationGroceries(self, request: DiffractionCalibrationRe
     @FromString
     def diffractionCalibration(self, request: DiffractionCalibrationRequest) -> Dict[str, Any]:
         self.validateRequest(request)
+
         payload = SimpleDiffCalRequest(
             ingredients=self.prepDiffractionCalibrationIngredients(request),
             groceries=self.fetchDiffractionCalibrationGroceries(request),
-            skipPixelCalibration=request.skipPixelCalibration,
         )
-        return self.diffractionCalibrationWithIngredients(payload)
 
-    @FromString
-    def diffractionCalibrationWithIngredients(self, request: SimpleDiffCalRequest) -> Dict[str, Any]:
-        pixelRes = self.pixelCalibration(request)
+        pixelRes = self.pixelCalibration(payload)
         if not pixelRes.result:
             raise RuntimeError("Pixel Calibration failed")
 
-        request.groceries["previousCalibration"] = pixelRes.calibrationTable
-        groupRes = self.groupCalibration(request)
+        payload.groceries["previousCalibration"] = pixelRes.calibrationTable
+        groupRes = self.groupCalibration(payload)
         if not groupRes.result:
             raise RuntimeError("Group Calibration failed")
 
@@ -185,28 +187,20 @@ def diffractionCalibrationWithIngredients(self, request: SimpleDiffCalRequest) -
     @FromString
     def pixelCalibration(self, request: SimpleDiffCalRequest) -> PixelDiffCalServing:
         # cook recipe
-        if request.skipPixelCalibration:
-            res = PixelDiffCalServing(
-                result=True,
-                medianOffsets=[],
-                maskWorkspace=request.groceries.get("maskWorkspace", ""),
-                calibrationTable=request.groceries["calibrationTable"],
+        res = PixelDiffCalRecipe().cook(request.ingredients, request.groceries)
+        maskWS = self.groceryService.getWorkspaceForName(res.maskWorkspace)
+        percentMasked = maskWS.getNumberMasked() / maskWS.getNumberHistograms()
+        threshold = Config["constants.maskedPixelThreshold"]
+        if percentMasked > threshold:
+            res.result = False
+            raise Exception(
+                (
+                    f"WARNING: More than {threshold*100}% of pixels failed calibration. Please check your input "
+                    "data. If input data has poor statistics, you may get better results by disabling Cross "
+                    "Correlation. You can also improve statistics by activating Lite mode if this is not "
+                    "already activated."
+                ),
             )
-        else:
-            res = PixelDiffCalRecipe().cook(request.ingredients, request.groceries)
-            maskWS = self.groceryService.getWorkspaceForName(res.maskWorkspace)
-            percentMasked = maskWS.getNumberMasked() / maskWS.getNumberHistograms()
-            threshold = Config["constants.maskedPixelThreshold"]
-            if percentMasked > threshold:
-                res.result = False
-                raise Exception(
-                    (
-                        f"WARNING: More than {threshold*100}% of pixels failed calibration. Please check your input "
-                        "data. If input data has poor statistics, you may get better results by disabling Cross "
-                        "Correlation. You can also improve statistics by activating Lite mode if this is not "
-                        "already activated."
-                    ),
-                )
         return res
 
     @FromString

src/snapred/backend/service/ReductionService.py

@@ -81,7 +81,7 @@ def __init__(self):
         self.registerPath("checkWritePermissions", self.checkReductionWritePermissions)
         self.registerPath("getSavePath", self.getSavePath)
         self.registerPath("getStateIds", self.getStateIds)
-        self.registerPath("validateReduction", self.validateReduction)
+        self.registerPath("validate", self.validateReduction)
         self.registerPath("artificialNormalization", self.artificialNormalization)
         self.registerPath("grabWorkspaceforArtificialNorm", self.grabWorkspaceforArtificialNorm)
         return

src/snapred/meta/InternalConstants.py

@@ -0,0 +1,20 @@
+from snapred.backend.dao.ObjectSHA import ObjectSHA
+from snapred.meta.Enum import StrEnum
+
+
+class ReservedRunNumber(StrEnum):
+    NATIVE: str = "000000"
+    LITE: str = "000001"
+
+
+class ReservedStateId(StrEnum):
+    NATIVE: str = ObjectSHA(hex="0000000000000000").hex
+    LITE: str = ObjectSHA(hex="0000000000000001").hex
+
+    @classmethod
+    def forRun(cls, runNumber):
+        match runNumber:
+            case ReservedRunNumber.NATIVE.value:
+                return cls.NATIVE
+            case ReservedRunNumber.LITE.value:
+                return cls.LITE

src/snapred/ui/view/DiffCalRequestView.py

@@ -43,12 +43,16 @@ def __init__(self, samples=[], groups=[], parent=None):
         self.litemodeToggle.setEnabled(True)
         self.peakFunctionDropdown.setCurrentIndex(0)
 
+        # skip pixel calibration toggle
+        self.skipPixelCalToggle = self._labeledField("Skip Pixel Calibration", Toggle(parent=self, state=False))
+
         # add all widgets to layout
         self.layout.addWidget(self.runNumberField, 0, 0)
         self.layout.addWidget(self.litemodeToggle, 0, 1)
-        self.layout.addWidget(self.removeBackgroundToggle, 0, 2)
+        self.layout.addWidget(self.skipPixelCalToggle, 0, 2)
         self.layout.addWidget(self.fieldConvergenceThreshold, 1, 0)
         self.layout.addWidget(self.fieldNBinsAcrossPeakWidth, 1, 1)
+        self.layout.addWidget(self.removeBackgroundToggle, 1, 2)
         self.layout.addWidget(self.sampleDropdown, 2, 0)
         self.layout.addWidget(self.groupingFileDropdown, 2, 1)
         self.layout.addWidget(self.peakFunctionDropdown, 2, 2)
@@ -75,3 +79,6 @@ def getLiteMode(self):
 
     def getRemoveBackground(self):
         return self.removeBackgroundToggle.field.getState()
+
+    def getSkipPixelCalibration(self):
+        return self.skipPixelCalToggle.field.getState()

src/snapred/ui/view/DiffCalTweakPeakView.py

@@ -99,16 +99,12 @@ def __init__(self, samples=[], groups=[], parent=None):
         self.purgePeaksButton = QPushButton("Purge Bad Peaks")
         self.purgePeaksButton.clicked.connect(self.emitPurge)
 
-        # skip pixel calibration button
-        self.skipPixelCalToggle = self._labeledField("Skip Pixel Calibration", Toggle(parent=self, state=False))
-
         # add all elements to the grid layout
         self.layout.addWidget(self.runNumberField, 0, 0)
         self.layout.addWidget(self.litemodeToggle, 0, 1)
         self.layout.addWidget(self.navigationBar, 1, 0)
         self.layout.addWidget(self.canvas, 2, 0, 1, -1)
         self.layout.addLayout(peakControlLayout, 3, 0, 1, 2)
-        self.layout.addWidget(self.skipPixelCalToggle, 3, 2, 1, 2)
         self.layout.addWidget(self.sampleDropdown, 4, 0)
         self.layout.addWidget(self.groupingFileDropdown, 4, 1)
         self.layout.addWidget(self.peakFunctionDropdown, 4, 2)