Merge 40 runs and compute stdev

Author: OliverRietmann

.github/scripts/merge_runs.py

@@ -1,13 +1,15 @@
-import sys
+import argparse
 import csv
 import statistics
 
-in_csv = sys.argv[1]
-out_csv = sys.argv[2]
+parser = argparse.ArgumentParser()
+parser.add_argument('-d', '--discard', type=int, default=0, help='Number of initial measurements to discard')
+parser.add_argument('-i', '--input', required=True, help='Input CSV file')
+parser.add_argument('-o', '--output', required=True, help='Output CSV file')
+args = parser.parse_args()
 
 time_dict = dict({})
-
-with open(in_csv) as csv_file:
+with open(args.input) as csv_file:
   csv_reader = csv.reader(csv_file)
   next(csv_reader)
   for name, time, _, _ in csv_reader:
@@ -18,10 +20,10 @@
 
 data = [["name", "time", "stdev"]]
 for name, time_list in time_dict.items():
-  mean = statistics.mean(time_list)
-  stdev = statistics.stdev(time_list)
+  mean = statistics.mean(time_list[args.discard:])
+  stdev = statistics.stdev(time_list[args.discard:])
   data.append([name, mean, stdev])
 
-with open(out_csv, 'w') as csv_file:
+with open(args.output, 'w') as csv_file:
   csv_writer = csv.writer(csv_file)
   csv_writer.writerows(data)

.github/workflows/standalone-benchmark.yml

@@ -87,16 +87,16 @@ jobs:
           source /etc/profile.d/modules.sh
           module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
           cd ${STANDALONE_DIR}
-          ${STANDALONE_DIR}/ca -e 50kHz -g --memSize 15000000000 --sync --runs 20 --debug 1 --PROCtimingCSV ${BENCHMARK_CSV}
+          ${STANDALONE_DIR}/ca -e 50kHz -g --memSize 15000000000 --sync --runs 42 --debug 1 --PROCtimingCSV ${BENCHMARK_CSV}
           rm -rf ${STANDALONE_DIR}/events/50kHz ${STANDALONE_DIR}/build
 
-      # - name: Display table on GitHub web
-      #   run: |
-      #     source /etc/profile.d/modules.sh
-      #     module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
-      #     python3 ${GITHUB_WORKSPACE}/.github/scripts/merge_runs.py ${BENCHMARK_CSV} ${BENCHMARK_CSV}
-      #     python3 ${GITHUB_WORKSPACE}/.github/scripts/csv_to_md.py ${BENCHMARK_CSV} ${STANDALONE_DIR}/baseline/${{ matrix.name }}.csv >> ${GITHUB_STEP_SUMMARY}
-      #     rm -rf ${STANDALONE_DIR}/baseline
+      - name: Display table on GitHub web
+        run: |
+          source /etc/profile.d/modules.sh
+          module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
+          python3 ${GITHUB_WORKSPACE}/.github/scripts/merge_runs.py --discard 2 --input ${BENCHMARK_CSV} --output ${BENCHMARK_CSV}
+          #python3 ${GITHUB_WORKSPACE}/.github/scripts/csv_to_md.py ${BENCHMARK_CSV} ${STANDALONE_DIR}/baseline/${{ matrix.name }}.csv >> ${GITHUB_STEP_SUMMARY}
+          rm -rf ${STANDALONE_DIR}/baseline
 
       - name: Upload Artifact
         uses: actions/upload-artifact@v4