Skip to content

Commit 998bf60

Browse files
amomchilovamomchilov
committed
Add benchmarks
1 parent 232748a commit 998bf60

6 files changed

Lines changed: 1403 additions & 0 deletions

File tree

‎Gemfile‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,8 @@ gem "minitest", "~> 5.16"
1414

1515
gem "rubocop", "~> 1.21"
1616
gem "rubocop-shopify", require: false
17+
18+
group :benchmark do
19+
gem "benchmark-ips"
20+
gem "sorbet-runtime"
21+
end

‎Gemfile.lock‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ GEM
88
specs:
99
ast (2.4.3)
1010
benchmark (0.5.0)
11+
benchmark-ips (2.14.0)
1112
date (3.5.1)
1213
erb (6.0.1)
1314
erubi (1.13.1)
@@ -114,18 +115,21 @@ PLATFORMS
114115
x86_64-linux
115116

116117
DEPENDENCIES
118+
benchmark-ips
117119
irb
118120
minitest (~> 5.16)
119121
rake (~> 13.0)
120122
rubocop (~> 1.21)
121123
rubocop-shopify
122124
sorbet
125+
sorbet-runtime
123126
tapioca (~> 0.17)
124127
type_toolkit!
125128

126129
CHECKSUMS
127130
ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383
128131
benchmark (0.5.0) sha256=465df122341aedcb81a2a24b4d3bd19b6c67c1530713fd533f3ff034e419236c
132+
benchmark-ips (2.14.0) sha256=b72bc8a65d525d5906f8cd94270dccf73452ee3257a32b89fbd6684d3e8a9b1d
129133
date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0
130134
erb (6.0.1) sha256=28ecdd99c5472aebd5674d6061e3c6b0a45c049578b071e5a52c2a7f13c197e5
131135
erubi (1.13.1) sha256=a082103b0885dbc5ecf1172fede897f9ebdb745a4b97a5e8dc63953db1ee4ad9

‎benchmark/abstract_methods_benchmark.rb‎

Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
# typed: ignore
2+
# frozen_string_literal: true
3+
4+
# Benchmark the performance overhead of calling:
5+
# - A concrete implementation of an abstract method
6+
# - An inherited concrete implementation of an abstract method
7+
# - The error case of calling an unimplemented abstract method
8+
9+
############################################# Results #############################################
10+
#
11+
# ruby 3.4.3 (2025-04-14 revision d0b7e5b6a0) +PRISM [arm64-darwin23]
12+
#
13+
# ## Interpretter
14+
#
15+
# | Call to... | Regular impl | Inherited impl | Missing impl |
16+
# |-------------------|--------------------:|------------------------:|--------------------------:|
17+
# | sorbet-runtime | (same-ish) 23.02 ns | (2.70x slower) 57.30 ns | (1.13x slower) 472.86 ns |
18+
# | manual delegation | (same-ish) 22.18 ns | (2.07x slower) 44.90 ns | *415.36 ns* |
19+
# | type_toolkit | (same-ish) 22.56 ns | *22.03 ns* | (2.11x slower) 890.38 ns |
20+
#
21+
# ## YJIT#
22+
# | Call to... | Regular impl | Inherited impl | Missing impl |
23+
# |-------------------|--------------------:|-------------------------:|--------------------------:|
24+
# | sorbet-runtime | (same-ish) 1.63 ns | (21.41x slower) 34.91 ns | (1.10x slower) 447.59 ns |
25+
# | manual delegation | (same-ish) 1.63 ns | (7.15x slower) 11.66 ns | *405.84 ns* |
26+
# | type_toolkit | (same-ish) 1.67 ns | *1.63 ns* | (1.91x slower) 774.91 ns |
27+
#
28+
####################################################################################################
29+
30+
require "bundler"
31+
Bundler.require(:default, :benchmark)
32+
33+
require "type_toolkit"
34+
require "type_toolkit/ext"
35+
36+
module TypeKitDemo
37+
# Provides the concrete implementation of `m`
38+
class Parent
39+
def m1 = "Parent#m1"
40+
end
41+
42+
module I
43+
interface!
44+
45+
abstract def m1; end
46+
abstract def m2; end
47+
abstract def not_implemented; end
48+
end
49+
50+
# Inherits the concrete implementation of `m` from DemoParentClass.
51+
class Child < Parent
52+
include I
53+
54+
def m2 = "Child#m2"
55+
end
56+
end
57+
58+
module SorbetRuntimeDemo
59+
# Provides the concrete implementation of `m`
60+
class Parent
61+
def m1 = "Parent#m1"
62+
end
63+
64+
module I
65+
extend T::Sig
66+
extend T::Helpers
67+
68+
interface!
69+
70+
sig { abstract.returns(String) }
71+
def m1; end
72+
73+
sig { abstract.returns(String) }
74+
def m2; end
75+
76+
sig { abstract.returns(String) }
77+
def not_implemented; end
78+
end
79+
80+
# Inherits the concrete implementation of `m` from DemoParentClass.
81+
class Child < Parent
82+
include I
83+
84+
def m2 = "Child#m2"
85+
end
86+
end
87+
88+
module ManualDelegationDemo
89+
class Parent
90+
def m1 = "Parent#m1"
91+
end
92+
93+
module I
94+
def m1 = defined?(super) ? super : raise
95+
def m2 = defined?(super) ? super : raise
96+
def not_implemented = defined?(super) ? super : raise
97+
end
98+
99+
# Inherits the concrete implementation of `m` from DemoParentClass.
100+
class Child < Parent
101+
include I
102+
103+
def m2 = "Child#m2"
104+
end
105+
end
106+
107+
type_toolkit_object = TypeKitDemo::Child.new
108+
manual_delegation_object = ManualDelegationDemo::Child.new
109+
sorbet_runtime_object = SorbetRuntimeDemo::Child.new
110+
111+
[:interpretter, :yjit].each do |mode|
112+
if mode == :yjit
113+
puts <<~MSG
114+
115+
116+
================================================================================
117+
Enabling YJIT...
118+
================================================================================
119+
120+
121+
MSG
122+
RubyVM::YJIT.enable
123+
end
124+
125+
warmup = 5
126+
time = 10
127+
128+
width = ["type_toolkit", "sorbet-runtime", "manual delegation"].max_by(&:length).length
129+
130+
puts "Benchmark the performance of calling the concrete implementation directly..."
131+
Benchmark.ips do |x|
132+
x.config(warmup:, time:)
133+
134+
x.report("type_toolkit".rjust(width)) do |times|
135+
i = 0
136+
while (i += 1) < times
137+
type_toolkit_object.m2
138+
end
139+
end
140+
141+
x.report("sorbet-runtime".rjust(width)) do |times|
142+
i = 0
143+
while (i += 1) < times
144+
sorbet_runtime_object.m2
145+
end
146+
end
147+
148+
x.report("manual delegation".rjust(width)) do |times|
149+
i = 0
150+
while (i += 1) < times
151+
manual_delegation_object.m2
152+
end
153+
end
154+
155+
x.compare!
156+
end
157+
158+
puts "\n\nBenchmark the performance of calling the inherited concrete implementation..."
159+
Benchmark.ips do |x|
160+
x.config(warmup:, time:)
161+
162+
x.report("type_toolkit".rjust(width)) do |times|
163+
i = 0
164+
while (i += 1) < times
165+
type_toolkit_object.m1
166+
end
167+
end
168+
169+
x.report("sorbet-runtime".rjust(width)) do |times|
170+
i = 0
171+
while (i += 1) < times
172+
sorbet_runtime_object.m1
173+
end
174+
end
175+
176+
x.report("manual delegation".rjust(width)) do |times|
177+
i = 0
178+
while (i += 1) < times
179+
manual_delegation_object.m1
180+
end
181+
end
182+
183+
x.compare!
184+
end
185+
186+
puts "\n\nTest the performance of calling an unimplemented abstract method..."
187+
Benchmark.ips do |x|
188+
x.config(warmup:, time:)
189+
190+
x.report("type_toolkit".rjust(width)) do |times|
191+
i = 0
192+
while (i += 1) < times
193+
begin
194+
type_toolkit_object.not_implemented
195+
rescue AbstractMethodNotImplementedError # rubocop:disable Lint/SuppressedException
196+
end
197+
end
198+
end
199+
200+
x.report("sorbet-runtime".rjust(width)) do |times|
201+
i = 0
202+
while (i += 1) < times
203+
begin
204+
sorbet_runtime_object.not_implemented
205+
rescue NotImplementedError # rubocop:disable Lint/SuppressedException
206+
end
207+
end
208+
end
209+
210+
x.report("manual delegation".rjust(width)) do |times|
211+
i = 0
212+
while (i += 1) < times
213+
begin
214+
manual_delegation_object.not_implemented
215+
rescue StandardError # rubocop:disable Lint/SuppressedException
216+
end
217+
end
218+
end
219+
220+
x.compare!
221+
end
222+
end

‎benchmark/interface_startup_performance.rb‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
# typed: ignore
2+
# frozen_string_literal: true
3+
4+
# Benchmark the startup performance of declaring modules/interfaces in 3 different styles:
5+
# - TypeToolkit (abstract gem)
6+
# - Sorbet runtime
7+
# - Manual delegation (defined?(super) pattern)
8+
9+
############################################# Results #############################################
10+
#
11+
# ruby 3.4.3 (2025-04-14 revision d0b7e5b6a0) +PRISM [arm64-darwin23]
12+
#
13+
# | | Interpretter | YJIT |
14+
# |-------------------|--------------------------:|---------------------------:|
15+
# | sorbet-runtime | (21.34x slower) 50.79 μs | (152.34x slower) 377.27 μs |
16+
# | type_toolkit | (4.18x slover) 9.95 μs | (4.18x slover) 10.35 μs |
17+
# | manual delegation | 2.38 μs | 2.48 μs |
18+
#
19+
####################################################################################################
20+
21+
require "bundler"
22+
Bundler.require(:default, :benchmark)
23+
24+
require "type_toolkit"
25+
require "type_toolkit/ext"
26+
27+
warmup = 1
28+
time = 1
29+
30+
width = ["type_toolkit", "sorbet-runtime", "manual delegation"].max_by(&:length).length
31+
32+
puts "Benchmark the time to declare an interface module with abstract methods..."
33+
34+
[:interpretter, :yjit].each do |mode|
35+
if mode == :yjit
36+
puts <<~MSG
37+
38+
39+
================================================================================
40+
Enabling YJIT...
41+
================================================================================
42+
43+
44+
MSG
45+
RubyVM::YJIT.enable
46+
end
47+
48+
Benchmark.ips do |x|
49+
x.config(warmup:, time:)
50+
51+
x.report("type_toolkit".rjust(width)) do |times|
52+
i = 0
53+
while (i += 1) < times
54+
interface = Module.new do
55+
interface!
56+
57+
abstract def m1; end
58+
abstract def m2; end
59+
abstract def m3; end
60+
end
61+
62+
Class.new do
63+
include interface
64+
65+
def m1 = "m1"
66+
def m2 = "m2"
67+
def m3 = "m3"
68+
end
69+
end
70+
end
71+
72+
x.report("sorbet-runtime".rjust(width)) do |times|
73+
i = 0
74+
while (i += 1) < times
75+
interface = Module.new do
76+
extend T::Sig
77+
extend T::Helpers
78+
79+
interface!
80+
81+
sig { abstract.returns(String) }
82+
def m1; end
83+
84+
sig { abstract.returns(String) }
85+
def m2; end
86+
87+
sig { abstract.returns(String) }
88+
def m3; end
89+
end
90+
91+
Class.new do
92+
extend T::Sig
93+
94+
include interface
95+
96+
sig { override.returns(String) }
97+
def m1 = "m1"
98+
99+
sig { override.returns(String) }
100+
def m2 = "m2"
101+
102+
sig { override.returns(String) }
103+
def m3 = "m3"
104+
end
105+
end
106+
end
107+
108+
x.report("manual delegation".rjust(width)) do |times|
109+
i = 0
110+
while (i += 1) < times
111+
interface = Module.new do
112+
def m1 = defined?(super) ? super : raise
113+
def m2 = defined?(super) ? super : raise
114+
def m3 = defined?(super) ? super : raise
115+
end
116+
117+
Class.new do
118+
include interface
119+
120+
def m1 = "m1"
121+
def m2 = "m2"
122+
def m3 = "m3"
123+
end
124+
end
125+
end
126+
127+
x.compare!
128+
end
129+
end

0 commit comments

Comments
 (0)