Skip to content

Commit 853d9c4

Browse files
amomchilovamomchilov
committed
Add benchmarks
1 parent 356cdea commit 853d9c4

7 files changed

Lines changed: 1408 additions & 0 deletions

File tree

‎Gemfile‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,8 @@ gem "minitest", "~> 5.16"
1515
gem "rubocop-shopify", require: false
1616
gem "rubocop-minitest", require: false
1717
gem "rubocop-rake", require: false
18+
19+
group :benchmark do
20+
gem "benchmark-ips"
21+
gem "sorbet-runtime"
22+
end

‎Gemfile.lock‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ GEM
1010
specs:
1111
ast (2.4.3)
1212
benchmark (0.5.0)
13+
benchmark-ips (2.14.0)
1314
date (3.5.1)
1415
erb (6.0.1)
1516
erubi (1.13.1)
@@ -123,19 +124,22 @@ PLATFORMS
123124
x86_64-linux
124125

125126
DEPENDENCIES
127+
benchmark-ips
126128
irb
127129
minitest (~> 5.16)
128130
rake (~> 13.0)
129131
rubocop-minitest
130132
rubocop-rake
131133
rubocop-shopify
132134
sorbet
135+
sorbet-runtime
133136
tapioca (>= 0.17)
134137
type_toolkit!
135138

136139
CHECKSUMS
137140
ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383
138141
benchmark (0.5.0) sha256=465df122341aedcb81a2a24b4d3bd19b6c67c1530713fd533f3ff034e419236c
142+
benchmark-ips (2.14.0) sha256=b72bc8a65d525d5906f8cd94270dccf73452ee3257a32b89fbd6684d3e8a9b1d
139143
date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0
140144
erb (6.0.1) sha256=28ecdd99c5472aebd5674d6061e3c6b0a45c049578b071e5a52c2a7f13c197e5
141145
erubi (1.13.1) sha256=a082103b0885dbc5ecf1172fede897f9ebdb745a4b97a5e8dc63953db1ee4ad9

‎benchmark/.rubocop.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
inherit_from: ../.rubocop.yml
2+
3+
Naming/ClassAndModuleCamelCase:
4+
Enabled: false # Sometimes underscores are useful, m'kay?
5+
6+
Style/ClassMethodsDefinitions:
7+
Enabled: false # We need to be able to compare `class << self` and `def self.`

‎benchmark/abstract_methods_benchmark.rb‎

Lines changed: 221 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,221 @@
1+
# typed: ignore
2+
# frozen_string_literal: true
3+
4+
# Benchmark the performance overhead of calling:
5+
# - A concrete implementation of an abstract method
6+
# - An inherited concrete implementation of an abstract method
7+
# - The error case of calling an unimplemented abstract method
8+
9+
############################################# Results #############################################
10+
#
11+
# ruby 3.4.3 (2025-04-14 revision d0b7e5b6a0) +PRISM [arm64-darwin23]
12+
#
13+
# ## Interpreter
14+
#
15+
# | Call to... | Regular impl | Inherited impl | Missing impl |
16+
# |-------------------|--------------------:|------------------------:|--------------------------:|
17+
# | sorbet-runtime | (same-ish) 23.02 ns | (2.70x slower) 57.30 ns | (1.13x slower) 472.86 ns |
18+
# | manual delegation | (same-ish) 22.18 ns | (2.07x slower) 44.90 ns | *415.36 ns* |
19+
# | type_toolkit | (same-ish) 22.56 ns | *22.03 ns* | (2.11x slower) 890.38 ns |
20+
#
21+
# ## YJIT#
22+
# | Call to... | Regular impl | Inherited impl | Missing impl |
23+
# |-------------------|--------------------:|-------------------------:|--------------------------:|
24+
# | sorbet-runtime | (same-ish) 1.63 ns | (21.41x slower) 34.91 ns | (1.10x slower) 447.59 ns |
25+
# | manual delegation | (same-ish) 1.63 ns | (7.15x slower) 11.66 ns | *405.84 ns* |
26+
# | type_toolkit | (same-ish) 1.67 ns | *1.63 ns* | (1.91x slower) 774.91 ns |
27+
#
28+
####################################################################################################
29+
30+
require "bundler"
31+
Bundler.require(:default, :benchmark)
32+
33+
require "type_toolkit"
34+
35+
module TypeKitDemo
36+
# Provides the concrete implementation of `m`
37+
class Parent
38+
def m1 = "Parent#m1"
39+
end
40+
41+
module I
42+
interface!
43+
44+
abstract def m1; end
45+
abstract def m2; end
46+
abstract def not_implemented; end
47+
end
48+
49+
# Inherits the concrete implementation of `m` from DemoParentClass.
50+
class Child < Parent
51+
include I
52+
53+
def m2 = "Child#m2"
54+
end
55+
end
56+
57+
module SorbetRuntimeDemo
58+
# Provides the concrete implementation of `m`
59+
class Parent
60+
def m1 = "Parent#m1"
61+
end
62+
63+
module I
64+
extend T::Sig
65+
extend T::Helpers
66+
67+
interface!
68+
69+
sig { abstract.returns(String) }
70+
def m1; end
71+
72+
sig { abstract.returns(String) }
73+
def m2; end
74+
75+
sig { abstract.returns(String) }
76+
def not_implemented; end
77+
end
78+
79+
# Inherits the concrete implementation of `m` from DemoParentClass.
80+
class Child < Parent
81+
include I
82+
83+
def m2 = "Child#m2"
84+
end
85+
end
86+
87+
module ManualDelegationDemo
88+
class Parent
89+
def m1 = "Parent#m1"
90+
end
91+
92+
module I
93+
def m1 = defined?(super) ? super : raise
94+
def m2 = defined?(super) ? super : raise
95+
def not_implemented = defined?(super) ? super : raise
96+
end
97+
98+
# Inherits the concrete implementation of `m` from DemoParentClass.
99+
class Child < Parent
100+
include I
101+
102+
def m2 = "Child#m2"
103+
end
104+
end
105+
106+
type_toolkit_object = TypeKitDemo::Child.new
107+
manual_delegation_object = ManualDelegationDemo::Child.new
108+
sorbet_runtime_object = SorbetRuntimeDemo::Child.new
109+
110+
[:interpreter, :yjit].each do |mode|
111+
if mode == :yjit
112+
puts <<~MSG
113+
114+
115+
================================================================================
116+
Enabling YJIT...
117+
================================================================================
118+
119+
120+
MSG
121+
RubyVM::YJIT.enable
122+
end
123+
124+
warmup = 5
125+
time = 10
126+
127+
width = ["type_toolkit", "sorbet-runtime", "manual delegation"].max_by(&:length).length
128+
129+
puts "Benchmark the performance of calling the concrete implementation directly..."
130+
Benchmark.ips do |x|
131+
x.config(warmup:, time:)
132+
133+
x.report("type_toolkit".rjust(width)) do |times|
134+
i = 0
135+
while (i += 1) < times
136+
type_toolkit_object.m2
137+
end
138+
end
139+
140+
x.report("sorbet-runtime".rjust(width)) do |times|
141+
i = 0
142+
while (i += 1) < times
143+
sorbet_runtime_object.m2
144+
end
145+
end
146+
147+
x.report("manual delegation".rjust(width)) do |times|
148+
i = 0
149+
while (i += 1) < times
150+
manual_delegation_object.m2
151+
end
152+
end
153+
154+
x.compare!
155+
end
156+
157+
puts "\n\nBenchmark the performance of calling the inherited concrete implementation..."
158+
Benchmark.ips do |x|
159+
x.config(warmup:, time:)
160+
161+
x.report("type_toolkit".rjust(width)) do |times|
162+
i = 0
163+
while (i += 1) < times
164+
type_toolkit_object.m1
165+
end
166+
end
167+
168+
x.report("sorbet-runtime".rjust(width)) do |times|
169+
i = 0
170+
while (i += 1) < times
171+
sorbet_runtime_object.m1
172+
end
173+
end
174+
175+
x.report("manual delegation".rjust(width)) do |times|
176+
i = 0
177+
while (i += 1) < times
178+
manual_delegation_object.m1
179+
end
180+
end
181+
182+
x.compare!
183+
end
184+
185+
puts "\n\nTest the performance of calling an unimplemented abstract method..."
186+
Benchmark.ips do |x|
187+
x.config(warmup:, time:)
188+
189+
x.report("type_toolkit".rjust(width)) do |times|
190+
i = 0
191+
while (i += 1) < times
192+
begin
193+
type_toolkit_object.not_implemented
194+
rescue AbstractMethodNotImplementedError # rubocop:disable Lint/SuppressedException
195+
end
196+
end
197+
end
198+
199+
x.report("sorbet-runtime".rjust(width)) do |times|
200+
i = 0
201+
while (i += 1) < times
202+
begin
203+
sorbet_runtime_object.not_implemented
204+
rescue NotImplementedError # rubocop:disable Lint/SuppressedException
205+
end
206+
end
207+
end
208+
209+
x.report("manual delegation".rjust(width)) do |times|
210+
i = 0
211+
while (i += 1) < times
212+
begin
213+
manual_delegation_object.not_implemented
214+
rescue StandardError # rubocop:disable Lint/SuppressedException
215+
end
216+
end
217+
end
218+
219+
x.compare!
220+
end
221+
end

‎benchmark/interface_startup_performance.rb‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
# typed: ignore
2+
# frozen_string_literal: true
3+
4+
# Benchmark the startup performance of declaring modules/interfaces in 3 different styles:
5+
# - TypeToolkit (abstract gem)
6+
# - Sorbet runtime
7+
# - Manual delegation (defined?(super) pattern)
8+
9+
############################################# Results #############################################
10+
#
11+
# ruby 3.4.3 (2025-04-14 revision d0b7e5b6a0) +PRISM [arm64-darwin23]
12+
#
13+
# | | Interpreter | YJIT |
14+
# |-------------------|-------------------------:|---------------------------:|
15+
# | sorbet-runtime | (21.34x slower) 50.79 μs | (152.34x slower) 377.27 μs |
16+
# | type_toolkit | (4.18x slower) 9.95 μs | (4.18x slower) 10.35 μs |
17+
# | manual delegation | 2.38 μs | 2.48 μs |
18+
#
19+
####################################################################################################
20+
21+
require "bundler"
22+
Bundler.require(:default, :benchmark)
23+
24+
require "type_toolkit"
25+
26+
warmup = 5
27+
time = 10
28+
29+
width = ["type_toolkit", "sorbet-runtime", "manual delegation"].max_by(&:length).length
30+
31+
puts "Benchmark the time to declare an interface module with abstract methods..."
32+
33+
[:interpreter, :yjit].each do |mode|
34+
if mode == :yjit
35+
puts <<~MSG
36+
37+
38+
================================================================================
39+
Enabling YJIT...
40+
================================================================================
41+
42+
43+
MSG
44+
RubyVM::YJIT.enable
45+
end
46+
47+
Benchmark.ips do |x|
48+
x.config(warmup:, time:)
49+
50+
x.report("type_toolkit".rjust(width)) do |times|
51+
i = 0
52+
while (i += 1) < times
53+
interface = Module.new do
54+
interface!
55+
56+
abstract def m1; end
57+
abstract def m2; end
58+
abstract def m3; end
59+
end
60+
61+
Class.new do
62+
include interface
63+
64+
def m1 = "m1"
65+
def m2 = "m2"
66+
def m3 = "m3"
67+
end
68+
end
69+
end
70+
71+
x.report("sorbet-runtime".rjust(width)) do |times|
72+
i = 0
73+
while (i += 1) < times
74+
interface = Module.new do
75+
extend T::Sig
76+
extend T::Helpers
77+
78+
interface!
79+
80+
sig { abstract.returns(String) }
81+
def m1; end
82+
83+
sig { abstract.returns(String) }
84+
def m2; end
85+
86+
sig { abstract.returns(String) }
87+
def m3; end
88+
end
89+
90+
Class.new do
91+
extend T::Sig
92+
93+
include interface
94+
95+
sig { override.returns(String) }
96+
def m1 = "m1"
97+
98+
sig { override.returns(String) }
99+
def m2 = "m2"
100+
101+
sig { override.returns(String) }
102+
def m3 = "m3"
103+
end
104+
end
105+
end
106+
107+
x.report("manual delegation".rjust(width)) do |times|
108+
i = 0
109+
while (i += 1) < times
110+
interface = Module.new do
111+
def m1 = defined?(super) ? super : raise
112+
def m2 = defined?(super) ? super : raise
113+
def m3 = defined?(super) ? super : raise
114+
end
115+
116+
Class.new do
117+
include interface
118+
119+
def m1 = "m1"
120+
def m2 = "m2"
121+
def m3 = "m3"
122+
end
123+
end
124+
end
125+
126+
x.compare!
127+
end
128+
end

0 commit comments

Comments
 (0)