WIP: PERF: OOC-optimized algorithm variants for 30+ filters

CMakeLists.txt

@@ -53,6 +53,14 @@ option(SIMPLNX_DOWNLOAD_TEST_FILES "Download the test files" ON)
 # ------------------------------------------------------------------------------
 option(SIMPLNX_WRITE_TEST_OUTPUT "Write unit test output files" OFF)
 
+# ------------------------------------------------------------------------------
+# Controls which algorithm paths are exercised by dual-dispatch unit tests.
+#   0 (Both)       - tests run with forceOoc=false AND forceOoc=true (default)
+#   1 (OocOnly)    - tests run with forceOoc=true only (use for OOC builds)
+#   2 (InCoreOnly) - tests run with forceOoc=false only (quick validation)
+# ------------------------------------------------------------------------------
+set(SIMPLNX_TEST_ALGORITHM_PATH "0" CACHE STRING "Algorithm paths to test: 0=Both, 1=OocOnly, 2=InCoreOnly")
+
 # ------------------------------------------------------------------------------
 # Is the SimplnxCore Plugin enabled [DEFAULT=ON]
 # ------------------------------------------------------------------------------
@@ -255,6 +263,7 @@ if(SIMPLNX_ENABLE_MULTICORE)
   target_link_libraries(simplnx PUBLIC TBB::tbb)
 endif()
 
+
 target_link_libraries(simplnx
   PUBLIC
     fmt::fmt
@@ -458,6 +467,7 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/DataStructure/DynamicListArray.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/EmptyDataStore.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/EmptyListStore.hpp
+  ${SIMPLNX_SOURCE_DIR}/DataStructure/EmptyStringStore.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IArray.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IDataArray.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IDataStore.hpp
@@ -539,6 +549,7 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/Utilities/DataGroupUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/DataObjectUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/DataStoreUtilities.hpp
+  ${SIMPLNX_SOURCE_DIR}/Utilities/AlgorithmDispatch.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/FilePathGenerator.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/ColorTableUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/FileUtilities.hpp
@@ -561,6 +572,7 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/Utilities/SamplingUtils.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/SegmentFeatures.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/TimeUtilities.hpp
+  ${SIMPLNX_SOURCE_DIR}/Utilities/UnionFind.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/TooltipGenerator.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/TooltipRowItem.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/OStreamUtilities.hpp

cmake/Plugin.cmake

@@ -383,6 +383,7 @@ function(create_simplnx_plugin_unit_test)
   target_compile_definitions(${UNIT_TEST_TARGET}
     PRIVATE
       SIMPLNX_BUILD_DIR="$<TARGET_FILE_DIR:simplnx_test>"
+      SIMPLNX_TEST_ALGORITHM_PATH=${SIMPLNX_TEST_ALGORITHM_PATH}
   )
 
   target_compile_options(${UNIT_TEST_TARGET}

src/Plugins/ITKImageProcessing/src/ITKImageProcessing/Common/ITKArrayHelper.hpp

@@ -855,7 +855,7 @@ Result<OutputActions> DataCheck(const DataStructure& dataStructure, const DataPa
   const auto& inputArray = dataStructure.getDataRefAs<IDataArray>(inputArrayPath);
   const auto& inputDataStore = inputArray.getIDataStoreRef();
 
-  if(!inputArray.getDataFormat().empty())
+  if(inputArray.getStoreType() == IDataStore::StoreType::OutOfCore)
   {
     return MakeErrorResult<OutputActions>(Constants::k_OutOfCoreDataNotSupported,
                                           fmt::format("Input Array '{}' utilizes out-of-core data. This is not supported within ITK filters.", inputArrayPath.toString()));
@@ -877,7 +877,7 @@ Result<detail::ITKFilterFunctorResult_t<FilterCreationFunctorT>> Execute(DataStr
 
   using ResultT = detail::ITKFilterFunctorResult_t<FilterCreationFunctorT>;
 
-  if(!inputArray.getDataFormat().empty())
+  if(inputArray.getStoreType() == IDataStore::StoreType::OutOfCore)
   {
     return MakeErrorResult(Constants::k_OutOfCoreDataNotSupported, fmt::format("Input Array '{}' utilizes out-of-core data. This is not supported within ITK filters.", inputArrayPath.toString()));
   }

src/Plugins/ITKImageProcessing/test/ITKTestBase.cpp

@@ -30,7 +30,7 @@ std::string ComputeMD5HashTyped(const IDataArray& outputDataArray)
   usize arraySize = dataArray.getSize();
 
   MD5 md5;
-  if(outputDataArray.getDataFormat().empty())
+  if(outputDataArray.getIDataStoreRef().getStoreType() != IDataStore::StoreType::OutOfCore)
   {
     const T* dataPtr = dataArray.template getIDataStoreRefAs<DataStore<T>>().data();
     md5.update(reinterpret_cast<const uint8*>(dataPtr), arraySize * sizeof(T));
@@ -135,47 +135,7 @@ namespace ITKTestBase
 bool IsArrayInMemory(DataStructure& dataStructure, const DataPath& outputDataPath)
 {
   const auto& outputDataArray = dataStructure.getDataRefAs<IDataArray>(outputDataPath);
-  DataType outputDataType = outputDataArray.getDataType();
-
-  switch(outputDataType)
-  {
-  case DataType::float32: {
-    return dynamic_cast<const DataArray<float32>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::float64: {
-    return dynamic_cast<const DataArray<float64>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::int8: {
-    return dynamic_cast<const DataArray<int8>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::uint8: {
-    return dynamic_cast<const DataArray<uint8>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::int16: {
-    return dynamic_cast<const DataArray<int16>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::uint16: {
-    return dynamic_cast<const DataArray<uint16>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::int32: {
-    return dynamic_cast<const DataArray<int32>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::uint32: {
-    return dynamic_cast<const DataArray<uint32>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::int64: {
-    return dynamic_cast<const DataArray<int64>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::uint64: {
-    return dynamic_cast<const DataArray<uint64>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::boolean: {
-    [[fallthrough]];
-  }
-  default: {
-    return {};
-  }
-  }
+  return outputDataArray.getIDataStoreRef().getStoreType() != IDataStore::StoreType::OutOfCore;
 }
 //------------------------------------------------------------------------------
 std::string ComputeMd5Hash(DataStructure& dataStructure, const DataPath& outputDataPath)

src/Plugins/OrientationAnalysis/CMakeLists.txt

@@ -172,6 +172,8 @@ set(filter_algorithms
   AlignSectionsMisorientation
   AlignSectionsMutualInformation
   BadDataNeighborOrientationCheck
+  BadDataNeighborOrientationCheckScanline
+  BadDataNeighborOrientationCheckWorklist
   CAxisSegmentFeatures
   ComputeAvgCAxes
   ComputeAvgOrientations
@@ -186,9 +188,12 @@ set(filter_algorithms
   ComputeFZQuaternions
   ComputeGBCD
   ComputeGBCDMetricBased
-  ComputeGBCDPoleFigure
+  ComputeGBCDPoleFigureDirect
+  ComputeGBCDPoleFigureScanline
   ComputeGBPDMetricBased
   ComputeIPFColors
+  ComputeIPFColorsDirect
+  ComputeIPFColorsScanline
   ComputeKernelAvgMisorientations
   ComputeMisorientations
   ComputeQuaternionConjugate

src/Plugins/OrientationAnalysis/docs/AlignSectionsMisorientationFilter.md

@@ -45,6 +45,20 @@ In this new structure, what follows is what the created structures represent:
 In previous versions a file would have been produced instead. If you wish to recreate this, you can write the Attribute Matrix as a CSV/Text file.
 
 
+## Algorithm
+
+### In-Core Path
+
+For each pair of adjacent Z-sections, the algorithm computes the misorientation between voxels across the section boundary. It tests candidate X-Y shifts to find the shift that minimizes the total misorientation between the two sections. All voxel comparisons use direct array indexing with `operator[]`.
+
+### Out-of-Core Path
+
+Reads pairs of adjacent Z-slices into local memory buffers using `copyIntoBuffer()`. All misorientation comparisons for a given slice pair operate entirely on the in-memory buffers. This converts what would otherwise be random cross-slice element access into sequential bulk reads, avoiding chunk thrashing when data is stored on disk in compressed chunks.
+
+### Performance
+
+The OOC optimization matters most for large datasets that exceed available RAM. By reading entire Z-slices in bulk rather than accessing individual voxels across slice boundaries, the algorithm avoids repeatedly decompressing the same disk chunks. For in-memory datasets, the two paths produce identical results with negligible overhead difference.
+
 % Auto generated parameter table will be inserted here
 
 ## Example Pipelines

src/Plugins/OrientationAnalysis/docs/AlignSectionsMutualInformationFilter.md

@@ -49,6 +49,20 @@ In this new structure, what follows is what the created structures represent:
 
 In previous versions a file would have been produced instead. If you wish to recreate this, you can write the Attribute Matrix as a CSV/Text file.
 
+## Algorithm
+
+### In-Core Path
+
+Aligns Z-sections by maximizing the mutual information of orientation data between adjacent slices. The algorithm segments each slice into temporary features, bins the orientations, and computes joint and marginal histograms to evaluate mutual information at each candidate shift position. All orientation and feature ID data is accessed through direct array indexing with `operator[]`.
+
+### Out-of-Core Path
+
+Reads the orientation and phase data for each pair of adjacent Z-slices into local memory buffers using `copyIntoBuffer()` before computing histograms. This eliminates per-voxel out-of-core reads during the histogram binning and mutual information calculation, replacing them with two sequential bulk reads per slice pair.
+
+### Performance
+
+The OOC optimization matters most for large datasets that exceed available RAM. Histogram computation requires visiting every voxel in both slices multiple times (once per candidate shift), so eliminating per-element OOC access prevents repeated decompression of the same disk chunks. For in-memory datasets, the two paths produce identical results with negligible overhead difference.
+
 % Auto generated parameter table will be inserted here
 
 ## References

src/Plugins/OrientationAnalysis/docs/BadDataNeighborOrientationCheckFilter.md

@@ -41,6 +41,32 @@ since there are no neighbors is the +-Z directions.
 
 Only the *Mask* value defining the cell as *good* or *bad* is changed. No other cell level array is modified.
 
+## Algorithm
+
+The algorithm operates in a multi-level iterative scheme, starting at level 6 (all 6 face-neighbors must agree) and decrementing to the user-specified *Required Number of Neighbors*. At each level, bad voxels are flipped to good if they have at least that many good face-neighbors with matching crystallographic orientation (misorientation below the tolerance). Starting strict and relaxing ensures high-confidence flips happen first, which can cascade to enable additional flips.
+
+### In-Core Path (BadDataNeighborOrientationCheckWorklist)
+
+When all arrays reside in contiguous in-memory storage, the algorithm uses a two-phase worklist approach:
+
+1. **Phase 1 (Initial count)**: A single linear scan counts matching good face-neighbors for every bad voxel, storing the count in a per-voxel array.
+2. **Phase 2 (Worklist propagation)**: For each level, a deque is seeded with all bad voxels meeting the threshold. As each voxel is flipped, its still-bad neighbors' counts are incremented. If a neighbor's count now meets the threshold, it is enqueued. This breadth-first flood-fill processes each voxel at most once per level, achieving O(flipped) amortized cost.
+
+### Out-of-Core Path (BadDataNeighborOrientationCheckScanline)
+
+When any of the quaternion, mask, or phase arrays are backed by chunked (OOC) disk storage, the algorithm uses a 3-slice rolling window over the Z axis:
+
+1. Three Z-slices of quaternions, phases, and mask data are maintained in memory (previous, current, next).
+2. For each bad voxel in the current slice, the count of matching good face-neighbors is recomputed on-the-fly using the rolling window buffers.
+3. If a voxel is flipped, the mask change is written back to the OOC store per-slice via `copyFromBuffer()`.
+4. The window shifts forward one Z-slice at a time, with only one new slice loaded per step.
+
+This approach trades recomputation (no persistent neighbor-count array) for strictly sequential I/O that avoids the random-access chunk thrashing that would occur with the worklist variant's BFS pattern.
+
+### Performance
+
+The in-core worklist variant is significantly faster for datasets that fit in RAM because each voxel is processed at most once per level (O(flipped) cost vs. O(N * passes) for the scanline variant). The OOC scanline variant is slower in absolute terms but avoids catastrophic performance degradation on disk-backed datasets where the worklist's random access pattern would trigger continuous chunk load/evict cycles. Memory usage is O(N) for the worklist variant vs. O(3 * sliceSize) for the scanline variant.
+
 ## Example Data
 
 | Example Input Image                                                       | Example Output Image                                                                                                                          |

src/Plugins/OrientationAnalysis/docs/CAxisSegmentFeaturesFilter.md

@@ -47,6 +47,22 @@ is to still use the 6 face neighbors ("Face Only") in order to stay consistent w
 |:--:|:--:|
 | ![Shared Edges & Points With Disconnected Region - "Face Only"](Images/SegmentFeatures/combination_face_only.png) | ![Shared Edges & Points With Disconnected Region - "All Connected"](Images/SegmentFeatures/combination_all_connected.png) |
 
+## Algorithm
+
+This filter segments voxels into features based on c-axis alignment, targeting hexagonal crystal systems. Voxels whose c-axes (the <001> crystallographic direction) are aligned within a user-defined tolerance are grouped into the same feature.
+
+### In-Core Path
+
+Uses a BFS-style flood fill where seed voxels are compared to their neighbors by computing the c-axis misalignment via direct array access with `operator[]`. Neighbors within tolerance are added to the current feature and queued for further expansion.
+
+### Out-of-Core Path
+
+Adapted to use sequential data access through the `SegmentFeatures` base class OOC support. The base class manages bulk I/O so that the flood-fill algorithm can proceed without triggering per-voxel OOC reads across chunk boundaries.
+
+### Performance
+
+The OOC optimization matters most for large datasets that exceed available RAM. Flood-fill naturally exhibits random access patterns as it grows regions across Z-slices, which can cause severe chunk thrashing with compressed on-disk storage. The OOC path mitigates this by leveraging the base class sequential access strategy. For in-memory datasets, the two paths produce identical results with negligible overhead difference.
+
 % Auto generated parameter table will be inserted here
 
 ## Example Pipelines

src/Plugins/OrientationAnalysis/docs/ComputeAvgCAxesFilter.md

@@ -24,6 +24,22 @@ This filter will error out if **ALL** phases are non-hexagonal. Any non-hexagona
 
 The output is a direction vector for each feature.
 
+## Algorithm
+
+For each element, the quaternion orientation is converted to an active rotation matrix (by transposing the passive orientation matrix), which is then applied to the crystallographic c-axis direction <001> to find the c-axis location in the sample reference frame. These rotated c-axis vectors are accumulated per feature using a running average that checks sign consistency (flipping vectors whose dot product with the running average is negative). After all elements are processed, each feature's accumulated c-axis is normalized to produce the final average direction.
+
+### In-Core Path
+
+All cell-level arrays (feature IDs, phases, quaternions) and the feature-level output array are accessed through the AbstractDataStore API. Crystal structures are validated to ensure at least one hexagonal phase is present.
+
+### Out-of-Core Path
+
+Cell-level data is read in sequential 4096-tuple chunks via `copyIntoBuffer`, with separate buffers for feature IDs, cell phases, and quaternions. Feature-level accumulation is performed in a local buffer (`avgCAxesCache`) that is bulk-read at the start and bulk-written back via `copyFromBuffer` after the normalization pass. The ensemble-level crystal structures array is cached locally at startup to avoid per-element OOC overhead.
+
+### Performance
+
+The chunked sequential reads convert what would be millions of individual virtual dispatch calls into a small number of bulk I/O operations. Since the feature-level buffer is proportional to the number of grains (thousands) rather than voxels (millions), it fits comfortably in memory even when cell data is on disk.
+
 % Auto generated parameter table will be inserted here
 
 ## Example Pipelines

src/Plugins/OrientationAnalysis/docs/ComputeAvgOrientationsFilter.md

@@ -68,6 +68,26 @@ These values may be exposed as user-configurable parameters in a future release.
 - **Features with zero elements:** Features with no elements (phase <= 0 for all voxels) will have their output arrays initialized to NaN (for vMF/Watson) or identity quaternion / zero Euler angles (for Rodrigues).
 - **Phase indexing:** The filter requires that phase values be > 0 for elements to be included in the averaging. Phase index 0 is reserved for "Unknown" in the Crystal Structures array and is always skipped.
 
+## Algorithm
+
+This filter supports three independent averaging methods that can be enabled in any combination. Each method accumulates per-element quaternion data grouped by feature ID, then produces feature-level outputs.
+
+### In-Core Path
+
+**Rodrigues Average:** Iterates over all elements once, accumulating quaternion sums into a feature-level buffer. For each element, the voxel quaternion is rotated to the nearest symmetry-equivalent orientation of the running average to handle the periodicity of orientation space. After accumulation, each feature's summed quaternion is normalized, forced into the positive hemisphere, and converted to Euler angles.
+
+**Von Mises-Fisher / Watson Average:** A preliminary pass counts elements per feature and maps feature IDs to phases. Then a `ParallelDataAlgorithm` processes features in parallel. For each feature, all element quaternions are collected, reduced to the fundamental zone, and passed to the EbsdLib `DirectionalStats` EM algorithm to estimate the mean orientation (mu) and concentration parameter (kappa).
+
+### Out-of-Core Path
+
+The Rodrigues average reads cell-level arrays (feature IDs, phases, quaternions) in sequential 64K-tuple chunks via `copyIntoBuffer`, accumulating into local `std::vector` buffers that hold only the feature-level data. Crystal structures are cached locally from the tiny ensemble-level array. Final results are bulk-written to the DataStore via `copyFromBuffer`, eliminating random-access overhead on potentially disk-backed stores.
+
+The vMF/Watson path similarly reads cell-level data through the AbstractDataStore API and caches crystal structures locally.
+
+### Performance
+
+The chunked Rodrigues path avoids per-element virtual dispatch on the DataStore, which is critical when cell-level data exceeds available RAM and falls back to HDF5-chunked storage. The feature-level buffers remain in-memory because feature counts are orders of magnitude smaller than cell counts. The vMF/Watson path benefits from parallel feature processing since each feature's EM computation is independent.
+
 % Auto generated parameter table will be inserted here
 
 ## Example Pipelines