Added small value zeroing, added --use-compression and --cmv-zero-threshold flags to TPCDistributeCMVSpec, updated drawCMV.C macro to auto detect branch format (compressed or not)

Author: tubagundem

Detectors/TPC/calibration/CMakeLists.txt

@@ -155,6 +155,10 @@ o2_add_test_root_macro(macro/prepareITFiles.C
                        COMPILE_ONLY
                        PUBLIC_LINK_LIBRARIES O2::TPCCalibration
                        LABELS tpc)
+o2_add_test_root_macro(macro/drawCMV.C
+                       COMPILE_ONLY
+                       PUBLIC_LINK_LIBRARIES O2::TPCCalibration O2::TPCBase
+                       LABELS tpc)
 
 o2_add_test(IDCFourierTransform
             COMPONENT_NAME calibration

Detectors/TPC/calibration/include/TPCCalibration/CMVContainer.h

@@ -56,7 +56,6 @@ struct CMVPerTFCompressed {
 /// CMV data for one TF across all CRUs
 /// Raw 16-bit CMV values are stored in a flat C array indexed as [cru * NTimeBinsPerTF + timeBin]
 /// CRU::MaxCRU and cmv::NTimeBinsPerTF are compile-time constants, so no dynamic allocation is needed
-/// Each TTree entry corresponds to one CMVPerTF object (one TF)
 struct CMVPerTF {
   uint32_t firstOrbit{0}; ///< First orbit of this TF, from heartbeatOrbit of the first CMV packet
   uint16_t firstBC{0};    ///< First bunch crossing of this TF, from heartbeatBC of the first CMV packet
@@ -70,6 +69,10 @@ struct CMVPerTF {
   /// Return the float CMV value for a given CRU and timebin within this TF
   float getCMVFloat(const int cru, const int timeBin) const;
 
+  /// Zero out raw CMV values whose float magnitude is below threshold (default 1.0 ADC)
+  /// This converts the sign-magnitude raw value to 0x0000 for all entries with |float value| < threshold
+  void zeroSmallValues(float threshold = 1.0f);
+
   /// Compress this object into a CMVPerTFCompressed using delta+zigzag+varint encoding
   CMVPerTFCompressed compress() const;
 

Detectors/TPC/calibration/macro/drawCMV.C

@@ -76,22 +76,44 @@ TObjArray* drawCMV(std::string_view filename, std::string_view outDir)
   constexpr int nTimeBins = cmv::NTimeBinsPerTF;
 
   TH2F* h2d = new TH2F("hCMVvsTimeBin", ";Timebin (200 ns);Common Mode Values (ADC)",
-                       nTimeBins / 16, 0, nTimeBins,
-                       110, -100, 10);
+                       100, 0, nTimeBins,
+                       110, -100.5, 9.5);
   h2d->SetStats(0);
 
-  // branch setup
-  o2::tpc::CMVPerTFCompressed* tfEntry = nullptr;
-  tree->SetBranchAddress("CMVPerTFCompressed", &tfEntry);
+  // auto-detect branch format: compressed (delta+zigzag+varint) or raw CMVPerTF
+  const bool isCompressed = (tree->GetBranch("CMVPerTFCompressed") != nullptr);
+  const bool isRaw = (tree->GetBranch("CMVPerTF") != nullptr);
+  if (!isCompressed && !isRaw) {
+    fmt::print("ERROR: neither 'CMVPerTFCompressed' nor 'CMVPerTF' branch found\n");
+    return arrCanvases;
+  }
+  fmt::print("Branch format: {}\n", isCompressed ? "CMVPerTFCompressed (delta+zigzag+varint)" : "CMVPerTF (raw)");
 
-  // allocate once outside the loop to avoid repeated zero-initialisation of the large array
-  auto* tf = new CMVPerTF();
+  // branch setup
+  o2::tpc::CMVPerTFCompressed* tfCompressed = nullptr;
+  o2::tpc::CMVPerTF* tfRaw = nullptr;
+  // staging object for decompression (only used in compressed path)
+  CMVPerTF* tfDecoded = isCompressed ? new CMVPerTF() : nullptr;
+
+  if (isCompressed) {
+    tree->SetBranchAddress("CMVPerTFCompressed", &tfCompressed);
+  } else {
+    tree->SetBranchAddress("CMVPerTF", &tfRaw);
+  }
 
   long firstOrbit = -1;
 
   for (int i = 0; i < nEntries; ++i) {
     tree->GetEntry(i);
-    tfEntry->decompress(tf);
+
+    // resolve to a unified pointer regardless of storage format
+    const CMVPerTF* tf = nullptr;
+    if (isCompressed) {
+      tfCompressed->decompress(tfDecoded);
+      tf = tfDecoded;
+    } else {
+      tf = tfRaw;
+    }
 
     if (i == 0) {
       firstOrbit = tf->firstOrbit;
@@ -104,9 +126,9 @@ TObjArray* drawCMV(std::string_view filename, std::string_view outDir)
     }
   }
 
-  delete tf;
+  delete tfDecoded;
   tree->ResetBranchAddresses();
-  delete tfEntry;
+  delete tfCompressed;
 
   fmt::print("firstOrbit: {}\n", firstOrbit);
 

Detectors/TPC/calibration/src/CMVContainer.cxx

@@ -93,6 +93,19 @@ float CMVPerTF::getCMVFloat(const int cru, const int timeBin) const
   return positive ? magnitude : -magnitude;
 }
 
+void CMVPerTF::zeroSmallValues(float threshold)
+{
+  if (threshold <= 0.f) {
+    return;
+  }
+  for (uint32_t i = 0; i < static_cast<uint32_t>(CRU::MaxCRU) * cmv::NTimeBinsPerTF; ++i) {
+    const float mag = (mDataPerTF[i] & 0x7FFF) / 128.f;
+    if (mag < threshold) {
+      mDataPerTF[i] = 0;
+    }
+  }
+}
+
 CMVPerTFCompressed CMVPerTF::compress() const
 {
   CMVPerTFCompressed out;

Detectors/TPC/workflow/include/TPCWorkflow/TPCDistributeCMVSpec.h

@@ -98,6 +98,10 @@ class TPCDistributeCMVSpec : public o2::framework::Task
       mNTFsDataDrop = mCheckEveryNData;
     }
     mDumpCMVs = ic.options().get<bool>("dump-cmvs");
+    mUseCompression = ic.options().get<bool>("use-compression");
+    mZeroThreshold = ic.options().get<float>("cmv-zero-threshold");
+    // re-initialise the interval tree now that mUseCompression is known (constructor used the default)
+    initIntervalTree();
   }
 
   void finaliseCCDB(ConcreteDataMatcher& matcher, void* obj) final
@@ -242,8 +246,18 @@ class TPCDistributeCMVSpec : public o2::framework::Task
 
     if (mProcessedCRU[currentBuffer][relTF] == mCRUs.size()) {
       ++mProcessedTFs[currentBuffer];
-      // compress the completed TF and fill one TTree entry, then reset the staging object
-      mCurrentCompressedTF = mCurrentTF.compress();
+      // zero out small values before storing (helps ROOT zlib for raw path; benign for compressed path)
+      if (mIntervalTFCount == 0) {
+        if (mZeroThreshold > 0.f) {
+          LOGP(info, "Zeroing CMV values with |value| < {:.3f} ADC counts before storing", mZeroThreshold);
+        } else {
+          LOGP(info, "cmv-zero-threshold <= 0, skipping zeroing of small CMV values");
+        }
+      }
+      mCurrentTF.zeroSmallValues(mZeroThreshold);
+      if (mUseCompression) {
+        mCurrentCompressedTF = mCurrentTF.compress();
+      }
       mIntervalTree->Fill();
       ++mIntervalTFCount;
       mCurrentTF = CMVPerTF{};
@@ -300,9 +314,11 @@ class TPCDistributeCMVSpec : public o2::framework::Task
   std::vector<bool> mSendCCDBOutputOrbitReset{};                                       ///< flag for received orbit reset time from CCDB
   std::vector<bool> mSendCCDBOutputGRPECS{};                                           ///< flag for received orbit GRPECS from CCDB
   bool mBuffer{false};                                                                 ///< buffer index
-  const bool mSendCCDB{false};                                                         ///< send output to CCDB populator
-  const bool mUsePreciseTimestamp{false};                                              ///< use precise timestamp from orbit-reset info
+  bool mSendCCDB{false};                                                               ///< send output to CCDB populator
+  bool mUsePreciseTimestamp{false};                                                    ///< use precise timestamp from orbit-reset info
   bool mDumpCMVs{false};                                                               ///< write a local ROOT debug file
+  bool mUseCompression{false};                                                         ///< use delta+zigzag+varint compression when storing in TTree; if false, store raw CMVPerTF
+  float mZeroThreshold{1.0f};                                                          ///< zero out CMV values with |float value| < threshold before storing
   long mTimestampStart{0};                                                             ///< CCDB validity start timestamp
   dataformats::Pair<long, int> mTFInfo{};                                              ///< orbit-reset time and NHBFPerTF for precise timestamp
   std::unique_ptr<TTree> mIntervalTree{};                                              ///< TTree accumulating one CMVPerTFCompressed entry per completed TF in the current interval
@@ -323,13 +339,18 @@ class TPCDistributeCMVSpec : public o2::framework::Task
   /// Returns real number of TFs taking buffer size into account
   unsigned int getNRealTFs() const { return mNTFsBuffer * mTimeFrames; }
 
-  /// Create a fresh in-memory TTree for the next aggregation interval, with mCurrentCompressedTF as the fill source
+  /// Create a fresh in-memory TTree for the next aggregation interval.
+  /// Branches on CMVPerTFCompressed (delta+zigzag+varint) or CMVPerTF (raw) depending on mUseCompression.
   void initIntervalTree()
   {
     mIntervalTree = std::make_unique<TTree>("ccdb_object", "ccdb_object");
     mIntervalTree->SetAutoSave(0);
     mIntervalTree->SetDirectory(nullptr);
-    mIntervalTree->Branch("CMVPerTFCompressed", &mCurrentCompressedTF);
+    if (mUseCompression) {
+      mIntervalTree->Branch("CMVPerTFCompressed", &mCurrentCompressedTF);
+    } else {
+      mIntervalTree->Branch("CMVPerTF", &mCurrentTF);
+    }
   }
 
   void clearBuffer(const bool currentBuffer)
@@ -543,7 +564,9 @@ DataProcessorSpec getTPCDistributeCMVSpec(const int ilane, const std::vector<uin
     Options{{"drop-data-after-nTFs", VariantType::Int, 0, {"Number of TFs after which to drop the data."}},
             {"check-data-every-n", VariantType::Int, 0, {"Number of run function called after which to check for missing data (-1 for no checking, 0 for default checking)."}},
             {"nFactorTFs", VariantType::Int, 1000, {"Number of TFs to skip for sending oldest TF."}},
-            {"dump-cmvs", VariantType::Bool, false, {"Dump CMVs to a local ROOT file for debugging"}}}}; // end DataProcessorSpec
+            {"dump-cmvs", VariantType::Bool, false, {"Dump CMVs to a local ROOT file for debugging"}},
+            {"use-compression", VariantType::Bool, false, {"Use delta+zigzag+varint compression when storing CMVs in TTree (false = store raw CMVPerTF, relies on ROOT built-in zlib)"}},
+            {"cmv-zero-threshold", VariantType::Float, 1.0f, {"Zero out CMV values with |value| < threshold (ADC counts) before storing; 0 disables zeroing"}}}}; // end DataProcessorSpec
   spec.rank = ilane;
   return spec;
 }

Detectors/TPC/workflow/src/CMVToVectorSpec.cxx

@@ -427,7 +427,7 @@ o2::framework::DataProcessorSpec getCMVToVectorSpec(const std::string inputSpec,
       {"write-raw-data-on-error", VariantType::Bool, false, {"dump raw data in case errors occurred"}},
       {"raw-file-name", VariantType::String, "/tmp/cmv_debug.{run}.{raw_type}", {"name of the raw output file"}},
       {"raw-data-type", VariantType::Int, 0, {"Which raw data to dump: 0-full TPC with DH, 1-full TPC with DH skip empty, 2-full TPC no DH, 3-full TPC no DH skip empty, 4-IDC raw only 5-CMV raw only"}},
-      {"check-incomplete-hbf", VariantType::Bool, false, {"false: don't chck; true: check and report"}},
+      {"check-incomplete-hbf", VariantType::Bool, false, {"false: don't check; true: check and report"}},
     } // end Options
   }; // end DataProcessorSpec
 }