diff --git a/engine/action/stats.cpp b/engine/action/stats.cpp index ea1499c417b..ad0baaec04a 100644 --- a/engine/action/stats.cpp +++ b/engine/action/stats.cpp @@ -314,8 +314,8 @@ void stats_t::analyze() r.analyze( num_tick_results.mean() ); } ); - portion_aps.analyze(); - portion_apse.analyze(); + portion_aps.analyze( sim ); + portion_apse.analyze( sim ); resource_gain.analyze( iterations ); @@ -329,8 +329,8 @@ void stats_t::analyze() resource_portion[ i ] = ( resource_total > 0 ) ? ( resource_gain.actual[ i ] / resource_total ) : 0; } - total_amount.analyze(); - actual_amount.analyze(); + total_amount.analyze( sim ); + actual_amount.analyze( sim ); compound_amount = actual_amount.count() ? actual_amount.mean() : 0.0; diff --git a/engine/class_modules/priest/sc_priest.cpp b/engine/class_modules/priest/sc_priest.cpp index c78b13938f4..6d07b0e1614 100644 --- a/engine/class_modules/priest/sc_priest.cpp +++ b/engine/class_modules/priest/sc_priest.cpp @@ -2577,7 +2577,7 @@ void priest_t::analyze( sim_t& sim ) if ( talents.shadow.voidform.enabled() ) { - sample_data.voidform_duration->analyze(); + sample_data.voidform_duration->analyze( sim ); } } diff --git a/engine/class_modules/sc_death_knight.cpp b/engine/class_modules/sc_death_knight.cpp index b16c28365b2..6013c6122e9 100644 --- a/engine/class_modules/sc_death_knight.cpp +++ b/engine/class_modules/sc_death_knight.cpp @@ -12886,21 +12886,21 @@ void death_knight_t::analyze( sim_t& s ) { player_t::analyze( s ); - _runes.rune_waste.analyze(); - _runes.cumulative_waste.analyze(); + _runes.rune_waste.analyze( s ); + _runes.cumulative_waste.analyze( s ); if ( options.extra_unholy_reporting ) { if ( talent.unholy.commander_of_the_dead.ok() ) - sample_data.lesser_ghoul_duration->analyze(); + sample_data.lesser_ghoul_duration->analyze( s ); - sample_data.lesser_ghouls_summoned->analyze(); - sample_data.lesser_ghouls_active->analyze(); - sample_data.magus_active->analyze(); + sample_data.lesser_ghouls_summoned->analyze( s ); + sample_data.lesser_ghouls_active->analyze( s ); + sample_data.magus_active->analyze( s ); if ( talent.unholy.pestilence.ok() ) { - sample_data.pest_dp_dur->analyze(); - sample_data.pest_vp_dur->analyze(); + sample_data.pest_dp_dur->analyze( s ); + sample_data.pest_vp_dur->analyze( s ); } } } diff --git a/engine/class_modules/sc_shaman.cpp b/engine/class_modules/sc_shaman.cpp index 764099462d7..c3b02f2a8a5 100644 --- a/engine/class_modules/sc_shaman.cpp +++ b/engine/class_modules/sc_shaman.cpp @@ -10765,13 +10765,13 @@ void shaman_t::analyze( sim_t& sim ) if ( talent.deeply_rooted_elements.ok() ) { - dre_samples.analyze(); + dre_samples.analyze( sim ); dre_samples.create_histogram( static_cast( dre_samples.max() - dre_samples.min() + 1 ) ); - dre_uptime_samples.analyze(); + dre_uptime_samples.analyze( sim ); dre_uptime_samples.create_histogram( static_cast( std::ceil( dre_uptime_samples.max() ) - std::floor( dre_uptime_samples.min() ) + 1 ) ); } - lvs_samples.analyze(); + lvs_samples.analyze( sim ); lvs_samples.create_histogram( static_cast( lvs_samples.max() - lvs_samples.min() + 1 ) ); } diff --git a/engine/player/player.cpp b/engine/player/player.cpp index d64072b3fcd..d2644b94e2e 100644 --- a/engine/player/player.cpp +++ b/engine/player/player.cpp @@ -13826,8 +13826,8 @@ void player_t::analyze( sim_t& s ) range::for_each( buff_list, []( buff_t* b ) { b->analyze(); } ); range::for_each( proc_list, []( proc_t* pr ) { pr->analyze(); } ); - range::for_each( uptime_list, []( uptime_t* up ) { up->analyze(); } ); - range::for_each( benefit_list, []( benefit_t* ben ) { ben->analyze(); } ); + range::for_each( uptime_list, [ this ]( uptime_t* up ) { up->analyze( *sim ); } ); + range::for_each( benefit_list, [ this ]( benefit_t* ben ) { ben->analyze( *sim ); } ); range::for_each( cooldown_waste_data_list, std::mem_fn( &cooldown_waste_data_t::analyze ) ); range::sort( stats_list, []( const stats_t* l, const stats_t* r ) { return l->name_str < r->name_str; } ); @@ -13849,7 +13849,7 @@ void player_t::analyze( sim_t& s ) return; } - range::for_each( sample_data_list, []( sample_data_helper_t* sd ) { sd->analyze(); } ); + range::for_each( sample_data_list, [ this ]( sample_data_helper_t* sd ) { sd->analyze( *sim ); } ); // Pet Chart Adjustment =================================================== size_t max_buckets = static_cast( collected_data.fight_length.max() ); diff --git a/engine/player/player_collected_data.cpp b/engine/player/player_collected_data.cpp index 0c5bd0b3dac..02cf8b7bdcb 100644 --- a/engine/player/player_collected_data.cpp +++ b/engine/player/player_collected_data.cpp @@ -231,30 +231,30 @@ void player_collected_data_t::merge( const player_t& other_player ) void player_collected_data_t::analyze( const player_t& p ) { - fight_length.analyze(); + fight_length.analyze( *p.sim ); // DMG - dmg.analyze(); - compound_dmg.analyze(); - dps.analyze(); - prioritydps.analyze(); - dpse.analyze(); - dmg_taken.analyze(); - dtps.analyze(); + dmg.analyze( *p.sim ); + compound_dmg.analyze( *p.sim ); + dps.analyze( *p.sim ); + prioritydps.analyze( *p.sim ); + dpse.analyze( *p.sim ); + dmg_taken.analyze( *p.sim ); + dtps.analyze( *p.sim ); // Heal - heal.analyze(); - compound_heal.analyze(); - hps.analyze(); - hpse.analyze(); - heal_taken.analyze(); - htps.analyze(); + heal.analyze( *p.sim ); + compound_heal.analyze( *p.sim ); + hps.analyze( *p.sim ); + hpse.analyze( *p.sim ); + heal_taken.analyze( *p.sim ); + htps.analyze( *p.sim ); // Absorb - absorb.analyze(); - compound_absorb.analyze(); - aps.analyze(); - absorb_taken.analyze(); - atps.analyze(); + absorb.analyze( *p.sim ); + compound_absorb.analyze( *p.sim ); + aps.analyze( *p.sim ); + absorb_taken.analyze( *p.sim ); + atps.analyze( *p.sim ); // Tank - deaths.analyze(); + deaths.analyze( *p.sim ); if ( !p.sim->single_actor_batch ) { diff --git a/engine/sim/benefit.hpp b/engine/sim/benefit.hpp index e9f45bf2930..60314b57514 100644 --- a/engine/sim/benefit.hpp +++ b/engine/sim/benefit.hpp @@ -36,9 +36,9 @@ struct benefit_t : private noncopyable down++; } - void analyze() + void analyze( sim_t& sim ) { - ratio.analyze(); + ratio.analyze( sim ); } void datacollection_begin() diff --git a/engine/sim/cooldown_waste_data.cpp b/engine/sim/cooldown_waste_data.cpp index c6bb1d60bc5..61b7724fd0d 100644 --- a/engine/sim/cooldown_waste_data.cpp +++ b/engine/sim/cooldown_waste_data.cpp @@ -51,8 +51,8 @@ void cooldown_waste_data_t::merge( const cooldown_waste_data_t& other ) void cooldown_waste_data_t::analyze() { - normal.analyze(); - cumulative.analyze(); + normal.analyze( cd->sim ); + cumulative.analyze( cd->sim ); } void cooldown_waste_data_t::datacollection_begin() diff --git a/engine/sim/proc.cpp b/engine/sim/proc.cpp index b56d6944536..a6cbff1293d 100644 --- a/engine/sim/proc.cpp +++ b/engine/sim/proc.cpp @@ -46,8 +46,8 @@ void proc_t::merge( const proc_t& other ) void proc_t::analyze() { - count.analyze(); - interval_sum.analyze(); + count.analyze( sim ); + interval_sum.analyze( sim ); } void proc_t::datacollection_begin() diff --git a/engine/sim/raid_event.cpp b/engine/sim/raid_event.cpp index ce0bdbaf762..7b5bb05d8de 100644 --- a/engine/sim/raid_event.cpp +++ b/engine/sim/raid_event.cpp @@ -2568,7 +2568,7 @@ void raid_event_t::analyze( sim_t* sim ) if ( sim->raid_events[ i ]->type == "pull" ) { auto pull_event = dynamic_cast( sim->raid_events[ i ].get() ); - pull_event->real_duration.analyze(); + pull_event->real_duration.analyze( *sim ); } } } diff --git a/engine/sim/sim.cpp b/engine/sim/sim.cpp index 8cf95dd350b..ff2b8702cab 100644 --- a/engine/sim/sim.cpp +++ b/engine/sim/sim.cpp @@ -2986,10 +2986,10 @@ void sim_t::analyze() { const auto start_time = chrono::wall_clock::now(); - simulation_length.analyze(); + simulation_length.analyze( *this ); if ( simulation_length.mean() == 0 ) return; - raid_dps.analyze(); + raid_dps.analyze( *this ); for ( size_t i = 0; i < buff_list.size(); ++i ) buff_list[ i ]->analyze(); diff --git a/engine/sim/uptime.hpp b/engine/sim/uptime.hpp index ca0e2cb3361..7eb6de98a2f 100644 --- a/engine/sim/uptime.hpp +++ b/engine/sim/uptime.hpp @@ -6,6 +6,7 @@ #pragma once #include "config.hpp" +#include "sim.hpp" #include "util/sample_data.hpp" #include "util/timespan.hpp" #include @@ -78,10 +79,10 @@ struct uptime_t : public uptime_base_t void update(bool is_up, timespan_t current_time); - void analyze() + void analyze( sim_t& sim ) { - uptime_sum.analyze(); - uptime_instance.analyze(); + uptime_sum.analyze( sim ); + uptime_instance.analyze( sim ); } void merge(const uptime_t& other) diff --git a/engine/util/sample_data.hpp b/engine/util/sample_data.hpp index 86b9b786220..3c0cb2aafbe 100644 --- a/engine/util/sample_data.hpp +++ b/engine/util/sample_data.hpp @@ -11,14 +11,45 @@ #include #include +#include "sim/sim.hpp" #include "util/generic.hpp" #include "util/string_view.hpp" +#include "util/rng.hpp" /* Collection of statistical formulas for sequences * Note: Returns 0 for empty sequences */ namespace statistics { +enum significance_e : size_t +{ + SIGNIFICANCE_1 = 0, + SIGNIFICANCE_2_5, + SIGNIFICANCE_5, + SIGNIFICANCE_10, + SIGNIFICANCE_15, + SIGNIFICANCE_MAX +}; + +const char* significance_string( significance_e significance ) +{ + switch ( significance ) + { + case SIGNIFICANCE_1: + return "1%"; + case SIGNIFICANCE_2_5: + return "2.5%"; + case SIGNIFICANCE_5: + return "5%"; + case SIGNIFICANCE_10: + return "10%"; + case SIGNIFICANCE_15: + return "15%"; + case SIGNIFICANCE_MAX: + return "max"; + } +} + /* Arithmetic Sum */ template @@ -54,7 +85,7 @@ range::value_type_t calculate_variance( const Range& r, } auto length = std::size( r ); if ( length > 1 ) - tmp /= length; + tmp /= length - 1; return tmp; } @@ -65,8 +96,7 @@ range::value_type_t calculate_variance( const Range& r ) { return calculate_variance( r, calculate_mean( r ) ); } - -/* Standard Deviation from a given mean + /* Standard Deviation from a given mean */ template range::value_type_t calculate_stddev( const Range& r, @@ -188,7 +218,46 @@ inline std::vector normalize_histogram( const std::vector& in ) return normalize_histogram( in, count ); } -} // end sd namespace +/* + * Tests for normality + */ +template +significance_e anderson_darling( const Range& r, range::value_type_t mean, range::value_type_t stddev ) +{ + range::value_type_t sum{}; + range::value_type_t y_cdf{}; + range::value_type_t statistic{}; + + constexpr std::array>, significance_e::SIGNIFICANCE_MAX> + statistic_critical_values = { { { significance_e::SIGNIFICANCE_1, 1.092 }, + { significance_e::SIGNIFICANCE_2_5, 0.918 }, + { significance_e::SIGNIFICANCE_5, 0.787 }, + { significance_e::SIGNIFICANCE_10, 0.656 }, + { significance_e::SIGNIFICANCE_15, 0.576 } } }; + + size_t n = std::size( r ); + + for ( size_t i = 0; i < n; ++i ) + { + y_cdf = rng::stdnormal_cdf( ( r[ i ] - mean ) / stddev ); + sum += ( 2 * i - 1 ) * std::log( y_cdf ); + sum += ( 2 * ( n - i ) + 1 ) * std::log( 1 - y_cdf ); + } + + statistic = ( -n - sum / n ) * ( 1 + 4 / ( 3 * n ) + 9 / ( 4 * n * n ) ); + + auto cmp = []( const std::pair>& pair, range::value_type_t value ) { + return pair.second < value; + }; + + auto lb = std::lower_bound( statistic_critical_values.begin(), statistic_critical_values.end(), statistic, cmp ); + + if ( lb == statistic_critical_values.end() ) + return significance_e::SIGNIFICANCE_MAX; + + return lb->first; +} +} // namespace statistics /* Simplest Samplest Data container. Only tracks sum and count * @@ -387,12 +456,17 @@ class extended_sample_data_t : public simple_sample_data_with_min_max_t } // Analyze collected data - void analyze() + + // void analyze( sim_t& sim ) <- allows sample data objects to emit warnings + void analyze( sim_t& sim ) { sort(); analyze_basics(); analyze_variance(); create_histogram(); + + // cannot warn, just asserting for now. + analyze_distribution( sim ); } /* @@ -459,6 +533,19 @@ class extended_sample_data_t : public simple_sample_data_with_min_max_t } } + /* + * Test Normality + * Requires: Analyzed Mean, stddev, sorted + */ + void analyze_distribution( sim_t& sim ) + { + if ( simple || data().empty() ) + return; + + sim.print_debug( "statistics_test: {} anderson-darling test for normality (significance: {})", name(), + statistics::significance_string( statistics::anderson_darling( sorted_data(), _mean, std_dev ) ) ); + } + public: // sort data void sort()