Skip to content

Commit 7a092e2

Browse files
OliverRietmannOliverRietmann
committed
Store benchmark results on NGT GPUs as artifacts
1 parent fa950ad commit 7a092e2

5 files changed

Lines changed: 217 additions & 31 deletions

File tree

‎.github/scripts/csv_to_md.py‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
import argparse
2+
import csv
3+
import tabulate as tab
4+
5+
parser = argparse.ArgumentParser()
6+
parser.add_argument('-b', '--baseline', required=True, help='Baseline CSV file')
7+
parser.add_argument('-c', '--current', required=True, help='Current CSV file')
8+
args = parser.parse_args()
9+
10+
def get_2d_list(csv_filename):
11+
with open(csv_filename) as csv_file:
12+
csv_reader = csv.reader(csv_file)
13+
next(csv_reader)
14+
return [[str(name), float(mean), float(stdev)] for name, mean, stdev in csv_reader]
15+
16+
table_baseline = get_2d_list(args.baseline)
17+
table_current = get_2d_list(args.current)
18+
19+
def get_emoji(d, stdev):
20+
z = 1.96 # 95% confidence interval
21+
if d < -z * stdev:
22+
return ':green_circle:'
23+
elif d > z * stdev:
24+
return ':red_circle:'
25+
else:
26+
return ':white_circle:'
27+
28+
table = []
29+
for baseline, current in zip(table_baseline, table_current):
30+
baseline_name, baseline_mean, _ = baseline
31+
name, mean, stdev = current
32+
assert(baseline_name == name)
33+
diff = baseline_mean - mean
34+
impact = 0.0 if stdev == 0.0 else diff / stdev
35+
emoji = get_emoji(diff, stdev)
36+
table.append([name, int(mean), f'{stdev:.2f}', int(diff), f'{impact:.2f}', emoji])
37+
38+
header = ['name', 'mean (\u03BCs)', 'stdev \u03C3', 'diff \u0394', '\u0394 / \u03C3', '']
39+
print(tab.tabulate(table, header, tablefmt="github"))

‎.github/scripts/merge_runs.py‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import argparse
2+
import csv
3+
import statistics
4+
5+
parser = argparse.ArgumentParser()
6+
parser.add_argument('-d', '--discard', type=int, default=0, help='Number of initial measurements to discard')
7+
parser.add_argument('-i', '--input', required=True, help='Input CSV file')
8+
parser.add_argument('-o', '--output', required=True, help='Output CSV file')
9+
args = parser.parse_args()
10+
11+
time_dict = dict({})
12+
with open(args.input) as csv_file:
13+
csv_reader = csv.reader(csv_file)
14+
next(csv_reader)
15+
for name, time, _, _ in csv_reader:
16+
if name in time_dict.keys():
17+
time_dict[name].append(float(time))
18+
else:
19+
time_dict[name] = [float(time)]
20+
21+
data = [["name", "time", "stdev"]]
22+
for name, time_list in time_dict.items():
23+
mean = statistics.mean(time_list[args.discard:])
24+
stdev = statistics.stdev(time_list[args.discard:])
25+
data.append([name, mean, stdev])
26+
27+
with open(args.output, 'w') as csv_file:
28+
csv_writer = csv.writer(csv_file)
29+
csv_writer.writerows(data)

‎.github/scripts/profiler_amd.py‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
import argparse
2+
import csv
3+
import statistics
4+
5+
parser = argparse.ArgumentParser()
6+
parser.add_argument('-r', '--runs', type=int, required=True, help='Number of runs')
7+
parser.add_argument('-i', '--input', required=True, help='Input CSV file')
8+
parser.add_argument('-o', '--output', required=True, help='Output CSV file')
9+
args = parser.parse_args()
10+
11+
time_dict = dict({})
12+
with open(args.input) as csv_file:
13+
csv_reader = csv.reader(csv_file)
14+
next(csv_reader)
15+
for row in csv_reader:
16+
full_name = row[13]
17+
time = (int(row[15]) - int(row[14])) / 1000.0
18+
if len(full_name) > 5 and full_name[:5] == "krnl_":
19+
name = full_name[5:-3]
20+
if name in time_dict.keys():
21+
time_dict[name].append(time)
22+
else:
23+
time_dict[name] = [time]
24+
25+
data = [["name", "time", "stdev"]]
26+
for name, time_list in time_dict.items():
27+
count = len(time_list) / args.runs
28+
mean = statistics.mean(time_list) * count
29+
stdev = 0 if args.runs == 1 else statistics.stdev(time_list) * count
30+
data.append([name, mean, stdev])
31+
32+
with open(args.output, 'w') as csv_file:
33+
csv_writer = csv.writer(csv_file)
34+
csv_writer.writerows(data)

‎.github/scripts/profiler_nvidia.py‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
import argparse
2+
import csv
3+
import statistics
4+
5+
parser = argparse.ArgumentParser()
6+
parser.add_argument('-r', '--runs', type=int, required=True, help='Number of runs')
7+
parser.add_argument('-i', '--input', required=True, help='Input CSV file')
8+
parser.add_argument('-o', '--output', required=True, help='Output CSV file')
9+
args = parser.parse_args()
10+
11+
ntsi_list = []
12+
with open(args.input) as csv_file:
13+
csv_reader = csv.reader(csv_file)
14+
next(csv_reader)
15+
next(csv_reader)
16+
next(csv_reader)
17+
for row in csv_reader:
18+
if row:
19+
full_name = row[8]
20+
instances = int(row[2])
21+
time = int(row[1]) / 1000.0
22+
sigma = float(row[7]) / 1000.0
23+
if len(full_name) > 5 and full_name[:5] == "krnl_":
24+
name = full_name[5:]
25+
ntsi_list.append([name, time, sigma, instances])
26+
27+
ntsi_list.sort(key = lambda row: row[0])
28+
29+
data = [["name", "time", "stdev"]]
30+
for name, time, sigma, instances in ntsi_list:
31+
count = instances / args.runs
32+
mean = time * count
33+
stdev = sigma * count
34+
data.append([name, mean, stdev])
35+
36+
with open(args.output, 'w') as csv_file:
37+
csv_writer = csv.writer(csv_file)
38+
csv_writer.writerows(data)

‎.github/workflows/standalone-benchmark.yml‎

Lines changed: 77 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -17,67 +17,113 @@ jobs:
1717
name: [nvidia-h100, nvidia-l40s, amd-mi300x, amd-w7900]
1818
include:
1919
- name: nvidia-h100
20+
vendor: nvidia
2021
runner: cern-nextgen-h100
2122
cmake_args: -DENABLE_CUDA=1 -DENABLE_HIP=0 -DCUDA_COMPUTETARGET=90
22-
ca_args: --gpuType CUDA #--RTCTECHloadLaunchBoundsFromFile genGPUArch/nvidia-h100.par
23+
profiler: nsys profile -o nvidia-h100
24+
profiler_post: nsys stats --report cuda_gpu_kern_sum --force-export=true --format csv nvidia-h100.nsys-rep >
2325
- name: nvidia-l40s
26+
vendor: nvidia
2427
runner: cern-nextgen-l40s
2528
cmake_args: -DENABLE_CUDA=1 -DENABLE_HIP=0 -DCUDA_COMPUTETARGET=89
26-
ca_args: --gpuType CUDA #--RTCTECHloadLaunchBoundsFromFile genGPUArch/nvidia-l40s.par
29+
profiler: nsys profile -o nvidia-l40s
30+
profiler_post: nsys stats --report cuda_gpu_kern_sum --force-export=true --format csv nvidia-l40s.nsys-rep >
2731
- name: amd-mi300x
32+
vendor: amd
2833
runner: cern-nextgen-mi300x
2934
cmake_args: -DENABLE_CUDA=0 -DENABLE_HIP=1 -DHIP_AMDGPUTARGET=gfx942
30-
ca_args: --gpuType HIP
35+
profiler: rocprofv2 --basenames --output-directory /root --output-file-name amd-mi300x
36+
profiler_post: touch
3137
- name: amd-w7900
38+
vendor: amd
3239
runner: cern-nextgen-w7900
3340
cmake_args: -DENABLE_CUDA=0 -DENABLE_HIP=1 -DHIP_AMDGPUTARGET=gfx1100
34-
ca_args: --gpuType HIP --RTCTECHloadLaunchBoundsFromFile genGPUArch/amd-w7900.par
41+
profiler: rocprofv2 --basenames --output-directory /root --output-file-name amd-w7900
42+
profiler_post: touch
43+
env:
44+
WORK_DIR: /cvmfs/alice.cern.ch
45+
ALIBUILD_ARCH_PREFIX: el9-x86_64/Packages
46+
MODULEPATH: /cvmfs/alice.cern.ch/etc/toolchain/modulefiles/el9-x86_64:/cvmfs/alice.cern.ch/el9-x86_64/Modules/modulefiles
47+
STANDALONE_DIR: /root/standalone
48+
BENCHMARK_CSV: ${{ matrix.name }}.csv
49+
PROFILER_CSV: results_${{ matrix.name }}.csv
50+
LD_LIBRARY_PATH: /usr/local/cuda-13.0/compat
3551

3652
name: ${{ matrix.name }}
3753
steps:
3854
- name: Checkout Repository
39-
uses: actions/checkout@v4
55+
uses: actions/checkout@v6
4056

41-
- name: Build and Run
57+
- name: Download Files
4258
run: |
4359
mkdir -p ${STANDALONE_DIR}
44-
. ${WORK_DIR}/${ALIBUILD_ARCH_PREFIX}/ninja-fortran/fortran-v1.11.1.g9-3/etc/profile.d/init.sh
45-
. ${WORK_DIR}/${ALIBUILD_ARCH_PREFIX}/O2/${O2_REVISION}/etc/profile.d/init.sh
60+
61+
if [[ "${{ matrix.vendor }}" == "nvidia" ]]; then
62+
curl -fL --retry 3 -o ${STANDALONE_DIR}/nsys.rpm https://developer.nvidia.com/downloads/assets/tools/secure/nsight-systems/2026_2/NsightSystems-linux-cli-public-2026.2.1.210-3763964.rpm
63+
dnf install -y ${STANDALONE_DIR}/nsys.rpm
64+
rm -f ${STANDALONE_DIR}/nsys.rpm
65+
fi
4666
47-
cmake -B ${BUILD_DIR} ${{ matrix.cmake_args }} -DENABLE_OPENCL=0 -DGPUCA_BUILD_EVENT_DISPLAY=0 -DGPUCA_DETERMINISTIC_MODE=GPU -DCMAKE_INSTALL_PREFIX=${STANDALONE_DIR} ${GITHUB_WORKSPACE}/GPU/GPUTracking/Standalone/
48-
cd ${BUILD_DIR}
49-
make install -j8
67+
curl -fL --retry 3 -o ${STANDALONE_DIR}/o2-simple-GPU.out https://cernbox.cern.ch/remote.php/dav/public-files/SfYXgQOHFga2w75/o2-simple-GPU.out
5068
51-
cd ${STANDALONE_DIR}
52-
mkdir -p ${STANDALONE_DIR}/genGPUArch
53-
curl -v -o ${STANDALONE_DIR}/genGPUArch/${{ matrix.name }}.par https://cernbox.cern.ch/remote.php/dav/public-files/SfYXgQOHFga2w75/genGPUArch/${{ matrix.name }}.par
69+
mkdir -p ${STANDALONE_DIR}/baseline
70+
curl -fL --retry 3 -o ${STANDALONE_DIR}/baseline/${PROFILER_CSV} https://cernbox.cern.ch/remote.php/dav/public-files/SfYXgQOHFga2w75/baseline/${PROFILER_CSV}
71+
curl -fL --retry 3 -o ${STANDALONE_DIR}/baseline/${BENCHMARK_CSV} https://cernbox.cern.ch/remote.php/dav/public-files/SfYXgQOHFga2w75/baseline/${BENCHMARK_CSV}
5472
5573
mkdir -p ${STANDALONE_DIR}/events
74+
curl -fL --retry 3 -o ${STANDALONE_DIR}/events/o2-simple.tar.xz https://cernbox.cern.ch/remote.php/dav/public-files/SfYXgQOHFga2w75/events/o2-simple.tar.xz
75+
tar -xf ${STANDALONE_DIR}/events/o2-simple.tar.xz -C ${STANDALONE_DIR}/events
5676
57-
curl -v -o ${STANDALONE_DIR}/events/50kHz.tar.xz https://cernbox.cern.ch/remote.php/dav/public-files/SfYXgQOHFga2w75/events/50kHz.tar.xz
77+
curl -fL --retry 3 -o ${STANDALONE_DIR}/events/50kHz.tar.xz https://cernbox.cern.ch/remote.php/dav/public-files/SfYXgQOHFga2w75/events/50kHz.tar.xz
5878
tar -xf ${STANDALONE_DIR}/events/50kHz.tar.xz -C ${STANDALONE_DIR}/events
59-
${STANDALONE_DIR}/ca -e 50kHz -g --seed 0 --memSize 15000000000 --sync --runs 1 --RTCenable --PROCdeterministicGPUReconstruction 1 --RTCoptSpecialCode 1 --debug 1 ${{ matrix.ca_args }} > ${ARTIFACT_FILE}
6079
61-
curl -v -o ${STANDALONE_DIR}/events/o2-simple.tar.xz https://cernbox.cern.ch/remote.php/dav/public-files/SfYXgQOHFga2w75/events/o2-simple.tar.xz
62-
tar -xf ${STANDALONE_DIR}/events/o2-simple.tar.xz -C ${STANDALONE_DIR}/events
63-
${STANDALONE_DIR}/ca -e o2-simple -g --seed 0 --memSize 20000000000 --sync --runs 1 --RTCenable --PROCdeterministicGPUReconstruction 1 --RTCoptSpecialCode 1 --debug 6 ${{ matrix.ca_args }}
80+
- name: Build Deterministic
81+
run: &build |
82+
source /etc/profile.d/modules.sh
83+
module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
6484

65-
curl -v -o ${STANDALONE_DIR}/o2-simple-GPU.out https://cernbox.cern.ch/remote.php/dav/public-files/SfYXgQOHFga2w75/o2-simple-GPU.out
85+
mkdir -p ${STANDALONE_DIR}
86+
cmake -B ${STANDALONE_DIR}/build ${{ matrix.cmake_args }} -DENABLE_OPENCL=0 -DGPUCA_BUILD_EVENT_DISPLAY=0 -DGPUCA_DETERMINISTIC_MODE=${DETERMINISTIC_MODE} -DCMAKE_INSTALL_PREFIX=${STANDALONE_DIR} ${GITHUB_WORKSPACE}/GPU/GPUTracking/Standalone/
87+
cmake --build ${STANDALONE_DIR}/build --target install -j 8
88+
env:
89+
DETERMINISTIC_MODE: GPU
90+
91+
- name: Test GPU Track Reconstruction
92+
run: |
93+
source /etc/profile.d/modules.sh
94+
module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
95+
cd ${STANDALONE_DIR}
96+
${STANDALONE_DIR}/ca -e o2-simple -g --seed 0 --memSize 20000000000 --sync --runs 1 --RTCenable --PROCdeterministicGPUReconstruction 1 --RTCoptConstexpr 1 --RTCoptSpecialCode 1 --debug 6
6697
cmp ${STANDALONE_DIR}/GPU.out ${STANDALONE_DIR}/o2-simple-GPU.out
67-
rm -rf ${STANDALONE_DIR}/GPU.out ${STANDALONE_DIR}/o2-simple-GPU.out
98+
rm -rf ${STANDALONE_DIR}/GPU.out ${STANDALONE_DIR}/o2-simple-GPU.out ${STANDALONE_DIR}/events/o2-simple ${STANDALONE_DIR}/build
6899
69-
rm -rf ${STANDALONE_DIR}/events
100+
- name: Build Non-Deterministic
101+
run: *build
70102
env:
71-
WORK_DIR: /cvmfs/alice.cern.ch
72-
ALIBUILD_ARCH_PREFIX: el9-x86_64/Packages
73-
O2_REVISION: daily-20260217-0000-1
74-
STANDALONE_DIR: /root/standalone
75-
BUILD_DIR: /root/standalone/build
76-
ARTIFACT_FILE: /root/artifact.txt
77-
LD_LIBRARY_PATH: /usr/local/cuda-13.0/compat
103+
DETERMINISTIC_MODE: OFF
104+
105+
- name: Benchmark GPU Track Reconstruction
106+
run: |
107+
source /etc/profile.d/modules.sh
108+
module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
109+
cd ${STANDALONE_DIR}
110+
${{ matrix.profiler }} ${STANDALONE_DIR}/ca -e 50kHz -g --memSize 15000000000 --sync --debug 1 --runs 12 --runsInit 2 --debugMarkdown 1 --PROCresetTimers 1 --PROCdebugCSV /root/${BENCHMARK_CSV}
111+
${{ matrix.profiler_post }} /root/${PROFILER_CSV}
112+
rm -rf ${STANDALONE_DIR}/events/50kHz ${STANDALONE_DIR}/build
113+
114+
- name: Display table on GitHub web
115+
run: |
116+
source /etc/profile.d/modules.sh
117+
module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
118+
python3 ${GITHUB_WORKSPACE}/.github/scripts/profiler_${{ matrix.vendor }}.py --runs 12 --input /root/${PROFILER_CSV} --output /root/${PROFILER_CSV}
119+
python3 ${GITHUB_WORKSPACE}/.github/scripts/merge_runs.py --discard 2 --input /root/${BENCHMARK_CSV} --output /root/${BENCHMARK_CSV}
120+
python3 ${GITHUB_WORKSPACE}/.github/scripts/csv_to_md.py --baseline ${STANDALONE_DIR}/baseline/${PROFILER_CSV} --current /root/${PROFILER_CSV} >> ${GITHUB_STEP_SUMMARY}
121+
echo -e "\n\n" >> ${GITHUB_STEP_SUMMARY}
122+
python3 ${GITHUB_WORKSPACE}/.github/scripts/csv_to_md.py --baseline ${STANDALONE_DIR}/baseline/${BENCHMARK_CSV} --current /root/${BENCHMARK_CSV} >> ${GITHUB_STEP_SUMMARY}
123+
rm -rf ${STANDALONE_DIR}/baseline
78124
79125
- name: Upload Artifact
80-
uses: actions/upload-artifact@v4
126+
uses: actions/upload-artifact@v6
81127
with:
82128
name: ${{ matrix.name }}-artifact
83-
path: /root/artifact.txt
129+
path: "/root/*.csv"

0 commit comments

Comments
 (0)