Merge pull request #150 from mach3-software/feature/kskwarczsnki/sigmaCI

feat: Sigma Var CI

Author: KSkwarczynski

CIValidations/CMakeLists.txt

@@ -10,6 +10,7 @@ foreach(app
     NuOscillatorInterfaceValidations
     pValueValidations
     SigmaVarValidations
+    SigmaVarFDValidation
     LLHValidation
     PredictiveValidations
     )

CIValidations/FitterValidations.cpp

@@ -49,6 +49,7 @@ void FitVal(const std::string& Algo, bool MoreTests)
     MaCh3Fitter->DragRace();
     MaCh3Fitter->RunLLHScan();
     MaCh3Fitter->Run2DLLHScan();
+    MaCh3Fitter->RunSigmaVarFD();
     MaCh3Fitter->GetStepScaleBasedOnLLHScan();
   }
   MaCh3Fitter->RunMCMC();

CIValidations/MaCh3CLI

@@ -56,6 +56,7 @@ run_covariance_validations() {
   ./CIValidations/MaCh3ModeValidations
   ./CIValidations/pValueValidations
   ./CIValidations/SigmaVarValidations
+  ./CIValidations/SigmaVarFDValidation
   if [ "$HAVE_UNITTEST_SUPPORT" -eq 0 ]; then
     ./CIValidations/UnitTests/manager_tests
     ./CIValidations/UnitTests/histogram_tests

CIValidations/SigmaVarFDValidation.cpp

@@ -0,0 +1,72 @@
+// MaCh3 spline includes
+#include "Utils/Comparison.h"
+#include "TKey.h"
+
+/// Current tests include
+int main(int argc, char *argv[])
+{
+  SetMaCh3LoggerFormat();
+
+  if (argc != 1) {
+    MACH3LOG_CRITICAL("You specified arguments, but none are needed. (Program name: ", argv[0]);
+    throw MaCh3Exception(__FILE__ , __LINE__ );
+  }
+  MACH3LOG_INFO("Testing Sigma Var");
+  std::string TutorialPath = std::getenv("MaCh3Tutorial_ROOT");
+
+  std::string command = TutorialPath + "/bin/SigmaVarTutorial " +
+                        TutorialPath + "/TutorialConfigs/FitterConfig.yaml";
+  int ret = system(command.c_str());
+  if (ret != 0) {
+    MACH3LOG_WARN("Error: system call failed with code {}", ret);
+  }
+
+  auto file = std::unique_ptr<TFile>(TFile::Open("SigmaVar_Test.root", "UPDATE"));
+  std::vector<std::string> Names = {"Norm_Param_0", "Norm_Param_1", "Norm_Param_2", "BinnedSplineParam1", "BinnedSplineParam2", "BinnedSplineParam3", "BinnedSplineParam4", "BinnedSplineParam5", "EResLep", "EResTot"};
+  std::ofstream outFile("SigmaVar.txt");
+  for (size_t i = 0; i < Names.size(); ++i) {
+    std::string dirPath = "SigmaVar/" + Names[i] + "/Tutorial/";
+    TDirectory* dir = file->GetDirectory(dirPath.c_str());
+
+    if (!dir) {
+      std::cerr << "Missing directory: " << dirPath << std::endl;
+      return 1;
+    }
+
+    TList* keys = dir->GetListOfKeys();
+    if (!keys) {
+      std::cerr << "No keys found in: " << dirPath << std::endl;
+      continue;
+    }
+
+    for (int i = 0; i < keys->GetSize(); ++i) {
+      TKey* key = dynamic_cast<TKey*>(keys->At(i));
+      if (!key) continue;
+
+      TObject* obj = key->ReadObj();
+      TH1* hist = dynamic_cast<TH1*>(obj);
+      if (!hist) continue;
+
+      for (int j = 1; j <= hist->GetNbinsX(); ++j) {
+        double binContent = hist->GetBinContent(j);
+        outFile << "Dial " << Names[i]
+        << " hist = " << hist->GetTitle()
+        << " bin = " << j
+        << " content = " << std::fixed << std::setprecision(6)
+        << std::fabs(binContent) << std::endl;
+      }
+    }
+  }
+  outFile.close();
+
+  bool TheSame = CompareTwoFiles(TutorialPath + "/CIValidations/TestOutputs/SigmaVarOut.txt", "SigmaVar.txt");
+  if(!TheSame) {
+    MACH3LOG_CRITICAL("Something is wrong with SigmaVar");
+    throw MaCh3Exception(__FILE__ , __LINE__ );
+  } else {
+    MACH3LOG_INFO("Everything is correct");
+  }
+
+  file->Close();
+  return 0;
+}

CIValidations/TestOutputs/SigmaVarOut.txt

@@ -26,7 +26,8 @@ MaCh3 is predominantly C++ software although some functionality are available th
 6. [Useful Settings](#useful-settings)
 7. [How to Plot?](#how-to-plot)
     1. [How to run LLH scan](#how-to-run-llh-scan)
-    2. [Plotting with Python](#plotting-with-python)
+    2. [How to run Sigma Variation](#how-to-run-sigma-variation)
+    3. [Plotting with Python](#plotting-with-python)
 
 ## How to Start?
 To compile simply
@@ -337,14 +338,17 @@ Read more [here](https://mach3-software.github.io/MaCh3/Structs_8h.html#a960da89
 
 ## How to Plot?
 
-There are a number of apps included to make plots from the results of your fits, llh scans etc. You can find more details on them and how they work in the main MaCh3 wiki [here](https://github.com/mach3-software/MaCh3/wiki). There you will also find some instructions on how you can write yor own plotting scripts.
+There are a number of apps included to make plots from the results of your fits, llh scans etc. You can find more details on them and how they work in the main MaCh3 wiki [here](https://github.com/mach3-software/MaCh3/wiki). There you will also find some instructions on how you can write your own plotting scripts.
 
 The plotting library is configured using yaml files. You can see some examples of such config files in the plotting directory, and a detailed explanation of them is given in [the wiki](https://github.com/mach3-software/MaCh3/wiki).
 
 Some examples on how to make some "standard" plots are given below.
 
 ### How to run LLH scan
-You can run MCMC in very similar way as MCMC
+LLH scan is a procedure where one changes value of single parameter, reweight MC and calculates likelihood.
+If running with Asimov setting at prior value LLH should be 0. Rate at which LLH is increasing with increase of value indiceate how sensitive your MC is to give parameter.
+
+You can run LLH scan in very similar way as MCMC
 ```bash
 ./bin/LLHScanTutorial TutorialConfigs/FitterConfig.yaml
 ```
@@ -361,6 +365,13 @@ It is possible to compare several files simply by:
 PlotLLH LLH_Test.root LLH_Test_2.root
 ```
 
+### How to run Sigma Variation
+Sigma Var is conceptually similar to LLH scan however here instead of looking how LLH changes one investigate actual sample spectra.
+
+```bash
+./bin/SigmaVarTutorial TutorialConfigs/FitterConfig.yaml
+```
+
 ### Plotting with Python
 
 If you have installed the python interface for MaCh3 as described

README.md

@@ -4,6 +4,7 @@ foreach(app
     MCMCTutorial
     LLHScanTutorial
     PredictiveTutorial
+    SigmaVarTutorial
     KinemDistributionTutorial
     )
     add_executable(${app} ${app}.cpp)

Tutorial/CMakeLists.txt

@@ -26,7 +26,7 @@ int main(int argc, char **argv) {
   auto xsec = MaCh3CovarianceFactory<ParameterHandlerGeneric>(FitManager.get(), "Xsec");
 
   // Initialise samplePDF
-  auto SampleConfig = FitManager->raw()["General"]["TutorialSamples"].as<std::vector<std::string>>();
+  auto SampleConfig = Get<std::vector<std::string>>(FitManager->raw()["General"]["TutorialSamples"], __FILE__ , __LINE__);
   auto mySamples = MaCh3SampleHandlerFactory<SampleHandlerTutorial>(SampleConfig, xsec.get());
 
   // Output file is named after output filename in config plus plot variable(s)

Tutorial/KinemDistributionTutorial.cpp

@@ -11,7 +11,7 @@ int main(int argc, char *argv[]){
   auto xsec = MaCh3CovarianceFactory<ParameterHandlerGeneric>(FitManager.get(), "Xsec");
 
   // Initialise samplePDF
-  auto SampleConfig = FitManager->raw()["General"]["TutorialSamples"].as<std::vector<std::string>>();
+  auto SampleConfig = Get<std::vector<std::string>>(FitManager->raw()["General"]["TutorialSamples"], __FILE__ , __LINE__);
   auto mySamples = MaCh3SampleHandlerFactory<SampleHandlerTutorial>(SampleConfig, xsec.get());
 
   // Create MCMC Class

Tutorial/LLHScanTutorial.cpp

@@ -10,7 +10,7 @@ int main(int argc, char *argv[]) {
   auto xsec = MaCh3CovarianceFactory<ParameterHandlerGeneric>(FitManager.get(), "Xsec");
 
   // Initialise samplePDF
-  auto SampleConfig = FitManager->raw()["General"]["TutorialSamples"].as<std::vector<std::string>>();
+  auto SampleConfig = Get<std::vector<std::string>>(FitManager->raw()["General"]["TutorialSamples"], __FILE__ , __LINE__);
   auto mySamples = MaCh3SampleHandlerFactory<SampleHandlerTutorial>(SampleConfig, xsec.get());
 
   // Create MCMC Class