[stats] Validate Sampled Distributions

engine/action/stats.cpp

@@ -314,8 +314,8 @@ void stats_t::analyze()
     r.analyze( num_tick_results.mean() );
   } );
 
-  portion_aps.analyze();
-  portion_apse.analyze();
+  portion_aps.analyze( sim );
+  portion_apse.analyze( sim );
 
   resource_gain.analyze( iterations );
 
@@ -329,8 +329,8 @@ void stats_t::analyze()
     resource_portion[ i ] = ( resource_total > 0 ) ? ( resource_gain.actual[ i ] / resource_total ) : 0;
   }
 
-  total_amount.analyze();
-  actual_amount.analyze();
+  total_amount.analyze( sim );
+  actual_amount.analyze( sim );
 
   compound_amount = actual_amount.count() ? actual_amount.mean() : 0.0;
 

engine/class_modules/priest/sc_priest.cpp

@@ -2577,7 +2577,7 @@ void priest_t::analyze( sim_t& sim )
 
   if ( talents.shadow.voidform.enabled() )
   {
-    sample_data.voidform_duration->analyze();
+    sample_data.voidform_duration->analyze( sim );
   }
 }
 

engine/class_modules/sc_death_knight.cpp

@@ -12886,21 +12886,21 @@ void death_knight_t::analyze( sim_t& s )
 {
   player_t::analyze( s );
 
-  _runes.rune_waste.analyze();
-  _runes.cumulative_waste.analyze();
+  _runes.rune_waste.analyze( s );
+  _runes.cumulative_waste.analyze( s );
   if ( options.extra_unholy_reporting )
   {
     if ( talent.unholy.commander_of_the_dead.ok() )
-      sample_data.lesser_ghoul_duration->analyze();
+      sample_data.lesser_ghoul_duration->analyze( s );
 
-    sample_data.lesser_ghouls_summoned->analyze();
-    sample_data.lesser_ghouls_active->analyze();
-    sample_data.magus_active->analyze();
+    sample_data.lesser_ghouls_summoned->analyze( s );
+    sample_data.lesser_ghouls_active->analyze( s );
+    sample_data.magus_active->analyze( s );
 
     if ( talent.unholy.pestilence.ok() )
     {
-      sample_data.pest_dp_dur->analyze();
-      sample_data.pest_vp_dur->analyze();
+      sample_data.pest_dp_dur->analyze( s );
+      sample_data.pest_vp_dur->analyze( s );
     }
   }
 }

engine/class_modules/sc_shaman.cpp

@@ -10765,13 +10765,13 @@ void shaman_t::analyze( sim_t& sim )
 
   if ( talent.deeply_rooted_elements.ok() )
   {
-    dre_samples.analyze();
+    dre_samples.analyze( sim );
     dre_samples.create_histogram( static_cast<unsigned>( dre_samples.max() - dre_samples.min() + 1 ) );
-    dre_uptime_samples.analyze();
+    dre_uptime_samples.analyze( sim );
     dre_uptime_samples.create_histogram( static_cast<unsigned>( std::ceil( dre_uptime_samples.max() ) - std::floor( dre_uptime_samples.min() ) + 1 ) );
   }
 
-  lvs_samples.analyze();
+  lvs_samples.analyze( sim );
   lvs_samples.create_histogram( static_cast<unsigned>( lvs_samples.max() - lvs_samples.min() + 1 ) );
 }
 

engine/player/player.cpp

@@ -13826,8 +13826,8 @@ void player_t::analyze( sim_t& s )
   range::for_each( buff_list, []( buff_t* b ) { b->analyze(); } );
 
   range::for_each( proc_list, []( proc_t* pr ) { pr->analyze(); } );
-  range::for_each( uptime_list, []( uptime_t* up ) { up->analyze(); } );
-  range::for_each( benefit_list, []( benefit_t* ben ) { ben->analyze(); } );
+  range::for_each( uptime_list, [ this ]( uptime_t* up ) { up->analyze( *sim ); } );
+  range::for_each( benefit_list, [ this ]( benefit_t* ben ) { ben->analyze( *sim ); } );
   range::for_each( cooldown_waste_data_list, std::mem_fn( &cooldown_waste_data_t::analyze ) );
 
   range::sort( stats_list, []( const stats_t* l, const stats_t* r ) { return l->name_str < r->name_str; } );
@@ -13849,7 +13849,7 @@ void player_t::analyze( sim_t& s )
       return;
   }
 
-  range::for_each( sample_data_list, []( sample_data_helper_t* sd ) { sd->analyze(); } );
+  range::for_each( sample_data_list, [ this ]( sample_data_helper_t* sd ) { sd->analyze( *sim ); } );
 
   // Pet Chart Adjustment ===================================================
   size_t max_buckets = static_cast<size_t>( collected_data.fight_length.max() );

engine/player/player_collected_data.cpp

@@ -231,30 +231,30 @@ void player_collected_data_t::merge( const player_t& other_player )
 
 void player_collected_data_t::analyze( const player_t& p )
 {
-  fight_length.analyze();
+  fight_length.analyze( *p.sim );
   // DMG
-  dmg.analyze();
-  compound_dmg.analyze();
-  dps.analyze();
-  prioritydps.analyze();
-  dpse.analyze();
-  dmg_taken.analyze();
-  dtps.analyze();
+  dmg.analyze( *p.sim );
+  compound_dmg.analyze( *p.sim );
+  dps.analyze( *p.sim );
+  prioritydps.analyze( *p.sim );
+  dpse.analyze( *p.sim );
+  dmg_taken.analyze( *p.sim );
+  dtps.analyze( *p.sim );
   // Heal
-  heal.analyze();
-  compound_heal.analyze();
-  hps.analyze();
-  hpse.analyze();
-  heal_taken.analyze();
-  htps.analyze();
+  heal.analyze( *p.sim );
+  compound_heal.analyze( *p.sim );
+  hps.analyze( *p.sim );
+  hpse.analyze( *p.sim );
+  heal_taken.analyze( *p.sim );
+  htps.analyze( *p.sim );
   // Absorb
-  absorb.analyze();
-  compound_absorb.analyze();
-  aps.analyze();
-  absorb_taken.analyze();
-  atps.analyze();
+  absorb.analyze( *p.sim );
+  compound_absorb.analyze( *p.sim );
+  aps.analyze( *p.sim );
+  absorb_taken.analyze( *p.sim );
+  atps.analyze( *p.sim );
   // Tank
-  deaths.analyze();
+  deaths.analyze( *p.sim );
 
   if ( !p.sim->single_actor_batch )
   {

engine/sim/benefit.hpp

@@ -36,9 +36,9 @@ struct benefit_t : private noncopyable
       down++;
   }
 
-  void analyze()
+  void analyze( sim_t& sim )
   {
-    ratio.analyze();
+    ratio.analyze( sim );
   }
 
   void datacollection_begin()

engine/sim/cooldown_waste_data.cpp

@@ -51,8 +51,8 @@ void cooldown_waste_data_t::merge( const cooldown_waste_data_t& other )
 
 void cooldown_waste_data_t::analyze()
 {
-  normal.analyze();
-  cumulative.analyze();
+  normal.analyze( cd->sim );
+  cumulative.analyze( cd->sim );
 }
 
 void cooldown_waste_data_t::datacollection_begin()

engine/sim/proc.cpp

@@ -46,8 +46,8 @@ void proc_t::merge( const proc_t& other )
 
 void proc_t::analyze()
 {
-  count.analyze();
-  interval_sum.analyze();
+  count.analyze( sim );
+  interval_sum.analyze( sim );
 }
 
 void proc_t::datacollection_begin()

engine/sim/raid_event.cpp

@@ -2568,7 +2568,7 @@ void raid_event_t::analyze( sim_t* sim )
     if ( sim->raid_events[ i ]->type == "pull" )
     {
       auto pull_event = dynamic_cast<pull_event_t*>( sim->raid_events[ i ].get() );
-      pull_event->real_duration.analyze();
+      pull_event->real_duration.analyze( *sim );
     }
   }
 }

engine/sim/sim.cpp

@@ -2986,10 +2986,10 @@ void sim_t::analyze()
 {
   const auto start_time = chrono::wall_clock::now();
 
-  simulation_length.analyze();
+  simulation_length.analyze( *this );
   if ( simulation_length.mean() == 0 ) return;
 
-  raid_dps.analyze();
+  raid_dps.analyze( *this );
 
   for ( size_t i = 0; i < buff_list.size(); ++i )
     buff_list[ i ]->analyze();

engine/sim/uptime.hpp

@@ -6,6 +6,7 @@
 #pragma once
 
 #include "config.hpp"
+#include "sim.hpp"
 #include "util/sample_data.hpp"
 #include "util/timespan.hpp"
 #include <string>
@@ -78,10 +79,10 @@ struct uptime_t : public uptime_base_t
 
   void update(bool is_up, timespan_t current_time);
 
-  void analyze()
+  void analyze( sim_t& sim )
   {
-    uptime_sum.analyze();
-    uptime_instance.analyze();
+    uptime_sum.analyze( sim );
+    uptime_instance.analyze( sim );
   }
 
   void merge(const uptime_t& other)

engine/util/sample_data.hpp

@@ -11,14 +11,45 @@
 #include <string>
 #include <vector>
 
+#include "sim/sim.hpp"
 #include "util/generic.hpp"
 #include "util/string_view.hpp"
+#include "util/rng.hpp"
 
 /* Collection of statistical formulas for sequences
  * Note: Returns 0 for empty sequences
  */
 namespace statistics
 {
+enum significance_e : size_t
+{
+  SIGNIFICANCE_1 = 0,
+  SIGNIFICANCE_2_5,
+  SIGNIFICANCE_5,
+  SIGNIFICANCE_10,
+  SIGNIFICANCE_15,
+  SIGNIFICANCE_MAX
+};
+
+const char* significance_string( significance_e significance )
+{
+  switch ( significance )
+  {
+  case SIGNIFICANCE_1:
+    return "1%";
+  case SIGNIFICANCE_2_5:
+    return "2.5%";
+  case SIGNIFICANCE_5:
+    return "5%";
+  case SIGNIFICANCE_10:
+    return "10%";
+  case SIGNIFICANCE_15:
+    return "15%";
+  case SIGNIFICANCE_MAX:
+    return "max";
+  }
+}
+
 /* Arithmetic Sum
  */
 template <typename Range>
@@ -54,7 +85,7 @@ range::value_type_t<Range> calculate_variance( const Range& r,
   }
   auto length = std::size( r );
   if ( length > 1 )
-    tmp /= length;
+    tmp /= length - 1;
   return tmp;
 }
 
@@ -65,8 +96,7 @@ range::value_type_t<Range> calculate_variance( const Range& r )
 {
   return calculate_variance( r, calculate_mean( r ) );
 }
-
-/* Standard Deviation from a given mean
+  /* Standard Deviation from a given mean
  */
 template <typename Range>
 range::value_type_t<Range> calculate_stddev( const Range& r,
@@ -188,7 +218,46 @@ inline std::vector<double> normalize_histogram( const std::vector<size_t>& in )
   return normalize_histogram( in, count );
 }
 
-}  // end sd namespace
+/*
+ * Tests for normality
+ */
+template <typename Range>
+significance_e anderson_darling( const Range& r, range::value_type_t<Range> mean, range::value_type_t<Range> stddev )
+{
+  range::value_type_t<Range> sum{};
+  range::value_type_t<Range> y_cdf{};
+  range::value_type_t<Range> statistic{};
+
+  constexpr std::array<std::pair<significance_e, range::value_type_t<Range>>, significance_e::SIGNIFICANCE_MAX>
+      statistic_critical_values = { { { significance_e::SIGNIFICANCE_1, 1.092 },
+                                      { significance_e::SIGNIFICANCE_2_5, 0.918 },
+                                      { significance_e::SIGNIFICANCE_5, 0.787 },
+                                      { significance_e::SIGNIFICANCE_10, 0.656 },
+                                      { significance_e::SIGNIFICANCE_15, 0.576 } } };
+
+  size_t n = std::size( r );
+
+  for ( size_t i = 0; i < n; ++i )
+  {
+    y_cdf = rng::stdnormal_cdf( ( r[ i ] - mean ) / stddev );
+    sum += ( 2 * i - 1 ) * std::log( y_cdf );
+    sum += ( 2 * ( n - i ) + 1 ) * std::log( 1 - y_cdf );
+  }
+
+  statistic = ( -n - sum / n ) * ( 1 + 4 / ( 3 * n ) + 9 / ( 4 * n * n ) );
+
+  auto cmp = []( const std::pair<significance_e, range::value_type_t<Range>>& pair, range::value_type_t<Range> value ) {
+    return pair.second < value;
+  };
+
+  auto lb = std::lower_bound( statistic_critical_values.begin(), statistic_critical_values.end(), statistic, cmp );
+
+  if ( lb == statistic_critical_values.end() )
+    return significance_e::SIGNIFICANCE_MAX;
+
+  return lb->first;
+}
+}  // namespace statistics
 
 /* Simplest Samplest Data container. Only tracks sum and count
  *
@@ -387,12 +456,17 @@ class extended_sample_data_t : public simple_sample_data_with_min_max_t
   }
 
   // Analyze collected data
-  void analyze()
+
+  // void analyze( sim_t& sim ) <- allows sample data objects to emit warnings
+  void analyze( sim_t& sim )
   {
     sort();
     analyze_basics();
     analyze_variance();
     create_histogram();
+
+    // cannot warn, just asserting for now.
+    analyze_distribution( sim );
   }
 
   /*
@@ -459,6 +533,19 @@ class extended_sample_data_t : public simple_sample_data_with_min_max_t
     }
   }
 
+  /*
+   * Test Normality
+   * Requires: Analyzed Mean, stddev, sorted
+   */
+  void analyze_distribution( sim_t& sim )
+  {
+    if ( simple || data().empty() )
+      return;
+
+    sim.print_debug( "statistics_test: {} anderson-darling test for normality (significance: {})", name(),
+                     statistics::significance_string( statistics::anderson_darling( sorted_data(), _mean, std_dev ) ) );
+  }
+
 public:
   // sort data
   void sort()