From 6a55ef9f4f2e575376fefe6d7f6a92793ca1c840 Mon Sep 17 00:00:00 2001 From: Alexis Le Dantec Date: Mon, 8 Dec 2025 15:37:54 +0100 Subject: [PATCH 1/5] [WIP] Add jmh benchmark --- build.gradle | 6 + conjure-jmh/build.gradle | 21 ++ .../palantir/conjure/ConjureBenchmarks.java | 324 ++++++++++++++++++ settings.gradle | 1 + versions.lock | 14 +- versions.props | 1 + 6 files changed, 366 insertions(+), 1 deletion(-) create mode 100644 conjure-jmh/build.gradle create mode 100644 conjure-jmh/src/jmh/java/com/palantir/conjure/ConjureBenchmarks.java diff --git a/build.gradle b/build.gradle index ff3a88204..56be80b7c 100644 --- a/build.gradle +++ b/build.gradle @@ -34,6 +34,7 @@ buildscript { classpath 'com.palantir.javaformat:gradle-palantir-java-format:2.83.0' classpath 'com.palantir.suppressible-error-prone:gradle-suppressible-error-prone:2.26.0' classpath 'org.revapi:gradle-revapi:1.8.0' + classpath 'me.champeau.jmh:jmh-gradle-plugin:0.7.3' } } @@ -99,6 +100,11 @@ afterEvaluate { } } +javaVersions { + libraryTarget = 17 + runtime = 21 +} + jdks { daemonTarget = 21 } diff --git a/conjure-jmh/build.gradle b/conjure-jmh/build.gradle new file mode 100644 index 000000000..a956c8a11 --- /dev/null +++ b/conjure-jmh/build.gradle @@ -0,0 +1,21 @@ +apply plugin: 'me.champeau.jmh' +apply plugin: 'org.revapi.revapi-gradle-plugin' + +jmh { + // Use profilers to collect additional data. Supported profilers: + // [cl, comp, gc, stack, perf, perfnorm, perfasm, xperf, xperfasm, hs_cl, hs_comp, hs_gc, hs_rt, hs_thr] + profilers = ['gc'] +} + +dependencies { + jmh 'org.openjdk.jmh:jmh-core' + + jmh 'com.fasterxml.jackson.core:jackson-annotations' + jmh 'com.fasterxml.jackson.core:jackson-databind' + jmh 'com.google.code.findbugs:jsr305' + jmh 'com.google.guava:guava' + jmh 'com.palantir.conjure.java.runtime:conjure-java-jackson-serialization' + jmh 'com.palantir.safe-logging:preconditions' + + jmhAnnotationProcessor 'org.openjdk.jmh:jmh-generator-annprocess' +} diff --git a/conjure-jmh/src/jmh/java/com/palantir/conjure/ConjureBenchmarks.java b/conjure-jmh/src/jmh/java/com/palantir/conjure/ConjureBenchmarks.java new file mode 100644 index 000000000..aa15e854f --- /dev/null +++ b/conjure-jmh/src/jmh/java/com/palantir/conjure/ConjureBenchmarks.java @@ -0,0 +1,324 @@ +/* + * (c) Copyright 2025 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.conjure; + +import com.fasterxml.jackson.annotation.JsonSetter; +import com.fasterxml.jackson.annotation.Nulls; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.google.common.collect.ImmutableMap; +import com.google.errorprone.annotations.CheckReturnValue; +import com.palantir.conjure.java.serialization.ObjectMappers; +import com.palantir.logsafe.Preconditions; +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryUsage; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; +import org.jspecify.annotations.NullMarked; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.profile.GCProfiler; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +@State(Scope.Benchmark) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(1) +@SuppressWarnings({"NullAway", "designforextension", "checkstyle:RegexpSinglelineJava", "checkstyle:VisibilityModifier" +}) +public class ConjureBenchmarks { + + private static final MemoryMXBean memoryBean = ManagementFactory.getMemoryMXBean(); + + private static final ObjectMapper mapper = ObjectMappers.newClientJsonMapper(); + + // Field to retain objects and measure retained memory + private List retainedObjects; + private long baselineMemory; + + @Setup + public void before() { + retainedObjects = new ArrayList<>(); + + // Force GC and get baseline memory before benchmark starts + forceGc(); + + baselineMemory = getUsedMemory(); + System.out.println("Baseline memory: " + baselineMemory + " bytes"); + } + + @TearDown + public void after() { + // Force GC to see what's actually retained + forceGc(); + + long finalMemory = getUsedMemory(); + long retainedMemory = finalMemory - baselineMemory; + + System.out.println("\n=== Retained Memory Analysis ==="); + System.out.println("Final memory: " + finalMemory + " bytes"); + System.out.println( + "Retained memory: " + retainedMemory + " bytes (" + (retainedMemory / 1024.0 / 1024.0) + " MB)"); + System.out.println("Objects retained: " + retainedObjects.size()); + System.out.println("================================\n"); + } + + @SuppressWarnings("ImmutableEnumChecker") + public enum RawJson { + EMPTY("{}"), + SINGLETON_MAP("{\"map\":{\"key1\":\"value1\"}}"), + NORMAL_MAP("{\"map\":{\"key1\":\"value1\",\"key2\":\"value2\",\"key3\":\"value3\"}}"); + + private final byte[] json; + + RawJson(String jsonString) { + this.json = jsonString.getBytes(StandardCharsets.UTF_8); + } + } + + public enum MapImplementation { + NORMAL(NormalMap.class), + SINGLETON(SingletonMap.class), + GUAVA_IMMUTABLE(GuavaImMap.class); + + private final Class clazz; + + MapImplementation(Class clazz) { + this.clazz = clazz; + } + } + + @Param + public RawJson json; + + @Param + public MapImplementation mapImpl; + + @Benchmark + public void testAllocatingBenchmark() throws IOException { + retainedObjects.add(mapper.readValue(json.json, mapImpl.clazz)); + } + + @JsonDeserialize(builder = NormalMap.Builder.class) + public static final class NormalMap { + private final Map map; + + private NormalMap(Map map) { + this.map = Collections.unmodifiableMap(map); + } + + public Map getMap() { + return map; + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + private boolean buildInvoked; + + private Map map = new LinkedHashMap<>(); + + private Builder() {} + + @JsonSetter(value = "map", nulls = Nulls.SKIP, contentNulls = Nulls.FAIL) + public Builder map(Map newMap) { + checkNotBuilt(); + this.map = new LinkedHashMap<>(Preconditions.checkNotNull(newMap, "map cannot be null")); + return this; + } + + @CheckReturnValue + public NormalMap build() { + checkNotBuilt(); + this.buildInvoked = true; + return new NormalMap(map); + } + + private void checkNotBuilt() { + Preconditions.checkState(!buildInvoked, "Build has already been called"); + } + } + } + + private static Map specialUnmodifiableMap(Map in) { + if (in.isEmpty()) { + return Map.of(); + } + if (in.size() == 1) { + Iterator> itr = in.entrySet().iterator(); + if (itr.hasNext()) { + Entry entry = itr.next(); + if (!itr.hasNext()) { + return Map.of(entry.getKey(), entry.getValue()); + } + } + } + return Collections.unmodifiableMap(in); + } + + @JsonDeserialize(builder = SingletonMap.Builder.class) + public static final class SingletonMap { + private final Map map; + + private SingletonMap(Map map) { + this.map = specialUnmodifiableMap(map); + } + + public Map getMap() { + return map; + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + private boolean buildInvoked; + + private Map map = new LinkedHashMap<>(); + + private Builder() {} + + @JsonSetter(value = "map", nulls = Nulls.SKIP, contentNulls = Nulls.FAIL) + public Builder map(Map newMap) { + checkNotBuilt(); + this.map = new LinkedHashMap<>(Preconditions.checkNotNull(newMap, "map cannot be null")); + return this; + } + + @CheckReturnValue + public SingletonMap build() { + checkNotBuilt(); + this.buildInvoked = true; + return new SingletonMap(map); + } + + private void checkNotBuilt() { + Preconditions.checkState(!buildInvoked, "Build has already been called"); + } + } + } + + @JsonDeserialize(builder = GuavaImMap.Builder.class) + @NullMarked + public static final class GuavaImMap { + private final Map map; + + private GuavaImMap(ImmutableMap map) { + this.map = map; + } + + public Map getMap() { + return map; + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + private boolean buildInvoked; + + @Nullable + private Object map; + + private Builder() {} + + @JsonSetter(value = "map", nulls = Nulls.SKIP, contentNulls = Nulls.FAIL) + @JsonDeserialize(as = ImmutableMap.class) + public Builder map(Map newMap) { + checkNotBuilt(); + this.map = newMap instanceof ImmutableMap im + ? im + : ImmutableMap.builder().putAll(newMap); + return this; + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + @CheckReturnValue + public GuavaImMap build() { + checkNotBuilt(); + this.buildInvoked = true; + ImmutableMap finalMap; + if (map == null) { + finalMap = ImmutableMap.of(); + } else if (map instanceof ImmutableMap im) { + finalMap = im; + } else if (map instanceof ImmutableMap.Builder builder) { + finalMap = builder.buildOrThrow(); + } else { + throw new IllegalStateException("Unexpected map type: " + map.getClass()); + } + return new GuavaImMap(finalMap); + } + + private void checkNotBuilt() { + Preconditions.checkState(!buildInvoked, "Build has already been called"); + } + } + } + + private void forceGc() { + System.gc(); + System.gc(); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + private long getUsedMemory() { + MemoryUsage heapUsage = memoryBean.getHeapMemoryUsage(); + return heapUsage.getUsed(); + } + + public static void main(String[] _args) throws Exception { + Options opt = new OptionsBuilder() + .include(ConjureBenchmarks.class.getSimpleName()) + .addProfiler(GCProfiler.class) // Shows allocation rates + .build(); + + new Runner(opt).run(); + } +} diff --git a/settings.gradle b/settings.gradle index 298ea859b..677e67fb1 100644 --- a/settings.gradle +++ b/settings.gradle @@ -17,6 +17,7 @@ include 'conjure-java-client-verifier:verification-server-api' include 'conjure-java-server-verifier' include 'conjure-java-server-verifier:verification-client-api' include 'conjure-java-undertow-runtime' +include 'conjure-jmh' include 'conjure-lib' include 'conjure-undertow-annotations' include 'conjure-undertow-lib' diff --git a/versions.lock b/versions.lock index 7cd4a27d2..68191f8ce 100644 --- a/versions.lock +++ b/versions.lock @@ -314,8 +314,12 @@ net.bytebuddy:byte-buddy:1.17.7 (2 constraints: 9f161b3f) net.bytebuddy:byte-buddy-agent:1.17.7 (1 constraints: 4a0b4ede) +net.sf.jopt-simple:jopt-simple:5.0.4 (1 constraints: be0ad6cc) + net.sourceforge.argparse4j:argparse4j:0.9.0 (2 constraints: da1b0890) +org.apache.commons:commons-math3:3.6.1 (1 constraints: bf0adbcc) + org.apache.commons:commons-text:1.13.0 (1 constraints: 3e1153ce) org.apiguardian:apiguardian-api:1.1.2 (6 constraints: 5366ce6e) @@ -408,9 +412,17 @@ org.mockito:mockito-junit-jupiter:5.20.0 (1 constraints: 3905453b) org.objenesis:objenesis:3.3 (1 constraints: b20a14bd) +org.openjdk.jmh:jmh-core:1.37 (4 constraints: 2d349791) + +org.openjdk.jmh:jmh-generator-asm:1.37 (1 constraints: 2c107598) + +org.openjdk.jmh:jmh-generator-bytecode:1.37 (1 constraints: de04fb30) + +org.openjdk.jmh:jmh-generator-reflection:1.37 (2 constraints: 491e3064) + org.opentest4j:opentest4j:1.3.0 (2 constraints: cf209249) -org.ow2.asm:asm:9.8 (1 constraints: 0b0aaea4) +org.ow2.asm:asm:9.8 (2 constraints: f617ab6a) org.slf4j:jcl-over-slf4j:2.0.17 (1 constraints: b20e835e) diff --git a/versions.props b/versions.props index 14faf4ea2..8edbc55c1 100644 --- a/versions.props +++ b/versions.props @@ -40,6 +40,7 @@ org.slf4j:* = 2.0.17 org.wildfly.common:wildfly-common = 2.0.1 jakarta.ws.rs:jakarta.ws.rs-api = 4.0.0 jakarta.validation:jakarta.validation-api = 3.0.0 +org.openjdk.jmh:* = 1.37 # dependency-upgrader:OFF # Generator should be compatible with java 8 From 2a944dfb90e7d1194f3ebee55b2f20bb2c1e588f Mon Sep 17 00:00:00 2001 From: Alexis Le Dantec Date: Mon, 8 Dec 2025 15:47:20 +0100 Subject: [PATCH 2/5] Replace by GC churn --- .../palantir/conjure/ConjureBenchmarks.java | 62 ++----------------- 1 file changed, 4 insertions(+), 58 deletions(-) diff --git a/conjure-jmh/src/jmh/java/com/palantir/conjure/ConjureBenchmarks.java b/conjure-jmh/src/jmh/java/com/palantir/conjure/ConjureBenchmarks.java index aa15e854f..b1fec3151 100644 --- a/conjure-jmh/src/jmh/java/com/palantir/conjure/ConjureBenchmarks.java +++ b/conjure-jmh/src/jmh/java/com/palantir/conjure/ConjureBenchmarks.java @@ -25,15 +25,10 @@ import com.palantir.conjure.java.serialization.ObjectMappers; import com.palantir.logsafe.Preconditions; import java.io.IOException; -import java.lang.management.ManagementFactory; -import java.lang.management.MemoryMXBean; -import java.lang.management.MemoryUsage; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.TimeUnit; @@ -47,10 +42,9 @@ import org.openjdk.jmh.annotations.OutputTimeUnit; import org.openjdk.jmh.annotations.Param; import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; import org.openjdk.jmh.profile.GCProfiler; import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.options.Options; @@ -66,41 +60,8 @@ }) public class ConjureBenchmarks { - private static final MemoryMXBean memoryBean = ManagementFactory.getMemoryMXBean(); - private static final ObjectMapper mapper = ObjectMappers.newClientJsonMapper(); - // Field to retain objects and measure retained memory - private List retainedObjects; - private long baselineMemory; - - @Setup - public void before() { - retainedObjects = new ArrayList<>(); - - // Force GC and get baseline memory before benchmark starts - forceGc(); - - baselineMemory = getUsedMemory(); - System.out.println("Baseline memory: " + baselineMemory + " bytes"); - } - - @TearDown - public void after() { - // Force GC to see what's actually retained - forceGc(); - - long finalMemory = getUsedMemory(); - long retainedMemory = finalMemory - baselineMemory; - - System.out.println("\n=== Retained Memory Analysis ==="); - System.out.println("Final memory: " + finalMemory + " bytes"); - System.out.println( - "Retained memory: " + retainedMemory + " bytes (" + (retainedMemory / 1024.0 / 1024.0) + " MB)"); - System.out.println("Objects retained: " + retainedObjects.size()); - System.out.println("================================\n"); - } - @SuppressWarnings("ImmutableEnumChecker") public enum RawJson { EMPTY("{}"), @@ -133,8 +94,8 @@ public enum MapImplementation { public MapImplementation mapImpl; @Benchmark - public void testAllocatingBenchmark() throws IOException { - retainedObjects.add(mapper.readValue(json.json, mapImpl.clazz)); + public void testAllocatingBenchmark(Blackhole bh) throws IOException { + bh.consume(mapper.readValue(json.json, mapImpl.clazz)); } @JsonDeserialize(builder = NormalMap.Builder.class) @@ -298,25 +259,10 @@ private void checkNotBuilt() { } } - private void forceGc() { - System.gc(); - System.gc(); - try { - Thread.sleep(100); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - private long getUsedMemory() { - MemoryUsage heapUsage = memoryBean.getHeapMemoryUsage(); - return heapUsage.getUsed(); - } - public static void main(String[] _args) throws Exception { Options opt = new OptionsBuilder() .include(ConjureBenchmarks.class.getSimpleName()) - .addProfiler(GCProfiler.class) // Shows allocation rates + .addProfiler(GCProfiler.class, "churn=true") .build(); new Runner(opt).run(); From 6be7baa6eb4de554a565ecc1487eba01aab3e514 Mon Sep 17 00:00:00 2001 From: Alexis Le Dantec Date: Mon, 8 Dec 2025 16:51:20 +0100 Subject: [PATCH 3/5] Try to use custom profiler --- .../palantir/conjure/ConjureBenchmarks.java | 161 +++++++++++++++++- 1 file changed, 158 insertions(+), 3 deletions(-) diff --git a/conjure-jmh/src/jmh/java/com/palantir/conjure/ConjureBenchmarks.java b/conjure-jmh/src/jmh/java/com/palantir/conjure/ConjureBenchmarks.java index b1fec3151..7a4f82d4a 100644 --- a/conjure-jmh/src/jmh/java/com/palantir/conjure/ConjureBenchmarks.java +++ b/conjure-jmh/src/jmh/java/com/palantir/conjure/ConjureBenchmarks.java @@ -25,10 +25,17 @@ import com.palantir.conjure.java.serialization.ObjectMappers; import com.palantir.logsafe.Preconditions; import java.io.IOException; +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryUsage; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.TimeUnit; @@ -44,8 +51,14 @@ import org.openjdk.jmh.annotations.Scope; import org.openjdk.jmh.annotations.State; import org.openjdk.jmh.annotations.Warmup; -import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.infra.IterationParams; import org.openjdk.jmh.profile.GCProfiler; +import org.openjdk.jmh.profile.InternalProfiler; +import org.openjdk.jmh.results.AggregationPolicy; +import org.openjdk.jmh.results.IterationResult; +import org.openjdk.jmh.results.Result; +import org.openjdk.jmh.results.ScalarResult; import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.options.Options; import org.openjdk.jmh.runner.options.OptionsBuilder; @@ -94,8 +107,8 @@ public enum MapImplementation { public MapImplementation mapImpl; @Benchmark - public void testAllocatingBenchmark(Blackhole bh) throws IOException { - bh.consume(mapper.readValue(json.json, mapImpl.clazz)); + public void testAllocatingBenchmark() throws IOException { + MemoryProfiler.addRetained(mapper.readValue(json.json, mapImpl.clazz)); } @JsonDeserialize(builder = NormalMap.Builder.class) @@ -263,8 +276,150 @@ public static void main(String[] _args) throws Exception { Options opt = new OptionsBuilder() .include(ConjureBenchmarks.class.getSimpleName()) .addProfiler(GCProfiler.class, "churn=true") + .addProfiler(MemoryProfiler.class) + .shouldDoGC(true) .build(); new Runner(opt).run(); } + + public static final class MemoryProfiler implements InternalProfiler { + private static final List retained = new ArrayList<>(10_000_000); + private static final MemoryMXBean memoryBean = ManagementFactory.getMemoryMXBean(); + + private long beforeUsedMemory = 0L; + + private static void addRetained(Object obj) { + retained.add(obj); + } + + private static void clearRetained() { + retained.clear(); + } + + @Override + public void beforeIteration(BenchmarkParams _benchmarkParams, IterationParams _iterationParams) { + clearRetained(); + runSystemGC(); + beforeUsedMemory = getUsedMemory(); + } + + @Override + public Collection afterIteration( + BenchmarkParams _benchmarkParams, IterationParams _iterationParams, IterationResult result) { + runSystemGC(); + long afterUsedMemory = getUsedMemory(); + long retainedMemory = afterUsedMemory - beforeUsedMemory; + + List results = new ArrayList<>(); + results.add(new ScalarResult( + "mem.retained.total", retainedMemory / 1024.0 / 1024.0, "MB", AggregationPolicy.AVG)); + results.add(new ScalarResult( + "mem.retained.total.norm", + (1.0 * retainedMemory) / result.getMetadata().getAllOps(), + "B/op", + AggregationPolicy.AVG)); + results.add(new ScalarResult("mem.retained.count", retained.size(), "obj", AggregationPolicy.AVG)); + results.add(new ScalarResult( + "mem.retained.count.norm", + (1.0 * retained.size()) / result.getMetadata().getAllOps(), + "obj", + AggregationPolicy.AVG)); + return results; + } + + @Override + public String getDescription() { + return "Measures retained memory after each iteration"; + } + + private static final int MAX_WAIT_MSEC = 20 * 1000; + + @SuppressWarnings("checkstyle:CyclomaticComplexity") + // Same as BaseRunner#runSystemGC + private boolean runSystemGC() { + List enabledBeans = new ArrayList<>(); + + long beforeGcCount = 0; + for (GarbageCollectorMXBean bean : ManagementFactory.getGarbageCollectorMXBeans()) { + long count = bean.getCollectionCount(); + if (count != -1) { + enabledBeans.add(bean); + } + } + + for (GarbageCollectorMXBean bean : enabledBeans) { + beforeGcCount += bean.getCollectionCount(); + } + + // Run the GC twice, and force finalization before each GCs. + System.runFinalization(); + System.gc(); + System.runFinalization(); + System.gc(); + + // Now make sure GC actually happened. We have to wait for two things: + // a) That at least two collections happened, indicating GC work. + // b) That counter updates have not happened for a while, indicating GC work had ceased. + // + // Note there is an opportunity window for a concurrent GC to happen before the first + // System.gc() call, which would get counted towards our GCs. This race is unresolvable + // unless we have GC-specific information about the collection cycles, and verify those + // were indeed GCs triggered by us. + + if (enabledBeans.isEmpty()) { + System.out.println( + "WARNING: MXBeans can not report GC info. System.gc() invoked, pessimistically waiting " + + MAX_WAIT_MSEC + " msecs"); + try { + TimeUnit.MILLISECONDS.sleep(MAX_WAIT_MSEC); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return true; + } + + boolean gcHappened = false; + + long start = System.nanoTime(); + while (TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start) < MAX_WAIT_MSEC) { + try { + TimeUnit.MILLISECONDS.sleep(200); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + long afterGcCount = 0; + for (GarbageCollectorMXBean bean : enabledBeans) { + afterGcCount += bean.getCollectionCount(); + } + + if (!gcHappened) { + if (afterGcCount - beforeGcCount >= 2) { + gcHappened = true; + } + } else { + if (afterGcCount == beforeGcCount) { + // Stable! + return true; + } + beforeGcCount = afterGcCount; + } + } + + if (gcHappened) { + System.out.println("WARNING: System.gc() was invoked but unable to wait while GC stopped, is GC too" + + " asynchronous?"); + } else { + System.out.println("WARNING: System.gc() was invoked but couldn't detect a GC occurring, is System.gc()" + + " disabled?"); + } + return false; + } + + private long getUsedMemory() { + MemoryUsage heapUsage = memoryBean.getHeapMemoryUsage(); + return heapUsage.getUsed(); + } + } } From 4269a66b95d6cf3ead6beda762c98fb7e55d9311 Mon Sep 17 00:00:00 2001 From: Alexis Le Dantec Date: Mon, 8 Dec 2025 17:37:28 +0100 Subject: [PATCH 4/5] Properly implement memory profiler --- .../palantir/conjure/ConjureBenchmarks.java | 190 +++--------------- .../com/palantir/conjure/MemoryProfiler.java | 174 ++++++++++++++++ 2 files changed, 205 insertions(+), 159 deletions(-) create mode 100644 conjure-jmh/src/jmh/java/com/palantir/conjure/MemoryProfiler.java diff --git a/conjure-jmh/src/jmh/java/com/palantir/conjure/ConjureBenchmarks.java b/conjure-jmh/src/jmh/java/com/palantir/conjure/ConjureBenchmarks.java index 7a4f82d4a..547501c39 100644 --- a/conjure-jmh/src/jmh/java/com/palantir/conjure/ConjureBenchmarks.java +++ b/conjure-jmh/src/jmh/java/com/palantir/conjure/ConjureBenchmarks.java @@ -25,17 +25,10 @@ import com.palantir.conjure.java.serialization.ObjectMappers; import com.palantir.logsafe.Preconditions; import java.io.IOException; -import java.lang.management.GarbageCollectorMXBean; -import java.lang.management.ManagementFactory; -import java.lang.management.MemoryMXBean; -import java.lang.management.MemoryUsage; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.TimeUnit; @@ -49,16 +42,10 @@ import org.openjdk.jmh.annotations.OutputTimeUnit; import org.openjdk.jmh.annotations.Param; import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; import org.openjdk.jmh.annotations.Warmup; -import org.openjdk.jmh.infra.BenchmarkParams; -import org.openjdk.jmh.infra.IterationParams; import org.openjdk.jmh.profile.GCProfiler; -import org.openjdk.jmh.profile.InternalProfiler; -import org.openjdk.jmh.results.AggregationPolicy; -import org.openjdk.jmh.results.IterationResult; -import org.openjdk.jmh.results.Result; -import org.openjdk.jmh.results.ScalarResult; import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.options.Options; import org.openjdk.jmh.runner.options.OptionsBuilder; @@ -75,11 +62,37 @@ public class ConjureBenchmarks { private static final ObjectMapper mapper = ObjectMappers.newClientJsonMapper(); + @Setup + public void before() { + // Ensure we have enough retained capacity to avoid measuring resizing costs. + // Each operation takes at least ~100-200ns, so 10 million ensures we have enough for all benchmark iterations. + MemoryProfiler.ensureRetainedCapacity(10_000_000); + } + @SuppressWarnings("ImmutableEnumChecker") public enum RawJson { - EMPTY("{}"), - SINGLETON_MAP("{\"map\":{\"key1\":\"value1\"}}"), - NORMAL_MAP("{\"map\":{\"key1\":\"value1\",\"key2\":\"value2\",\"key3\":\"value3\"}}"); + NO_MAP("{}"), + EMPTY(generateMap(0)), + SINGLETON(generateMap(1)), + THREE(generateMap(3)), + SIX(generateMap(6)), + TEN(generateMap(10)), + HUNDRED(generateMap(100)), + THOUSAND(generateMap(1000)), + ; + + private static String generateMap(int count) { + StringBuilder sb = new StringBuilder(); + sb.append("{\"map\":{"); + for (int i = 1; i <= count; i++) { + sb.append("\"key").append(i).append("\":\"value").append(i).append("\""); + if (i < count) { + sb.append(","); + } + } + sb.append("}}"); + return sb.toString(); + } private final byte[] json; @@ -275,151 +288,10 @@ private void checkNotBuilt() { public static void main(String[] _args) throws Exception { Options opt = new OptionsBuilder() .include(ConjureBenchmarks.class.getSimpleName()) - .addProfiler(GCProfiler.class, "churn=true") + .addProfiler(GCProfiler.class) .addProfiler(MemoryProfiler.class) - .shouldDoGC(true) .build(); new Runner(opt).run(); } - - public static final class MemoryProfiler implements InternalProfiler { - private static final List retained = new ArrayList<>(10_000_000); - private static final MemoryMXBean memoryBean = ManagementFactory.getMemoryMXBean(); - - private long beforeUsedMemory = 0L; - - private static void addRetained(Object obj) { - retained.add(obj); - } - - private static void clearRetained() { - retained.clear(); - } - - @Override - public void beforeIteration(BenchmarkParams _benchmarkParams, IterationParams _iterationParams) { - clearRetained(); - runSystemGC(); - beforeUsedMemory = getUsedMemory(); - } - - @Override - public Collection afterIteration( - BenchmarkParams _benchmarkParams, IterationParams _iterationParams, IterationResult result) { - runSystemGC(); - long afterUsedMemory = getUsedMemory(); - long retainedMemory = afterUsedMemory - beforeUsedMemory; - - List results = new ArrayList<>(); - results.add(new ScalarResult( - "mem.retained.total", retainedMemory / 1024.0 / 1024.0, "MB", AggregationPolicy.AVG)); - results.add(new ScalarResult( - "mem.retained.total.norm", - (1.0 * retainedMemory) / result.getMetadata().getAllOps(), - "B/op", - AggregationPolicy.AVG)); - results.add(new ScalarResult("mem.retained.count", retained.size(), "obj", AggregationPolicy.AVG)); - results.add(new ScalarResult( - "mem.retained.count.norm", - (1.0 * retained.size()) / result.getMetadata().getAllOps(), - "obj", - AggregationPolicy.AVG)); - return results; - } - - @Override - public String getDescription() { - return "Measures retained memory after each iteration"; - } - - private static final int MAX_WAIT_MSEC = 20 * 1000; - - @SuppressWarnings("checkstyle:CyclomaticComplexity") - // Same as BaseRunner#runSystemGC - private boolean runSystemGC() { - List enabledBeans = new ArrayList<>(); - - long beforeGcCount = 0; - for (GarbageCollectorMXBean bean : ManagementFactory.getGarbageCollectorMXBeans()) { - long count = bean.getCollectionCount(); - if (count != -1) { - enabledBeans.add(bean); - } - } - - for (GarbageCollectorMXBean bean : enabledBeans) { - beforeGcCount += bean.getCollectionCount(); - } - - // Run the GC twice, and force finalization before each GCs. - System.runFinalization(); - System.gc(); - System.runFinalization(); - System.gc(); - - // Now make sure GC actually happened. We have to wait for two things: - // a) That at least two collections happened, indicating GC work. - // b) That counter updates have not happened for a while, indicating GC work had ceased. - // - // Note there is an opportunity window for a concurrent GC to happen before the first - // System.gc() call, which would get counted towards our GCs. This race is unresolvable - // unless we have GC-specific information about the collection cycles, and verify those - // were indeed GCs triggered by us. - - if (enabledBeans.isEmpty()) { - System.out.println( - "WARNING: MXBeans can not report GC info. System.gc() invoked, pessimistically waiting " - + MAX_WAIT_MSEC + " msecs"); - try { - TimeUnit.MILLISECONDS.sleep(MAX_WAIT_MSEC); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - return true; - } - - boolean gcHappened = false; - - long start = System.nanoTime(); - while (TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start) < MAX_WAIT_MSEC) { - try { - TimeUnit.MILLISECONDS.sleep(200); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - - long afterGcCount = 0; - for (GarbageCollectorMXBean bean : enabledBeans) { - afterGcCount += bean.getCollectionCount(); - } - - if (!gcHappened) { - if (afterGcCount - beforeGcCount >= 2) { - gcHappened = true; - } - } else { - if (afterGcCount == beforeGcCount) { - // Stable! - return true; - } - beforeGcCount = afterGcCount; - } - } - - if (gcHappened) { - System.out.println("WARNING: System.gc() was invoked but unable to wait while GC stopped, is GC too" - + " asynchronous?"); - } else { - System.out.println("WARNING: System.gc() was invoked but couldn't detect a GC occurring, is System.gc()" - + " disabled?"); - } - return false; - } - - private long getUsedMemory() { - MemoryUsage heapUsage = memoryBean.getHeapMemoryUsage(); - return heapUsage.getUsed(); - } - } } diff --git a/conjure-jmh/src/jmh/java/com/palantir/conjure/MemoryProfiler.java b/conjure-jmh/src/jmh/java/com/palantir/conjure/MemoryProfiler.java new file mode 100644 index 000000000..87cba8fa6 --- /dev/null +++ b/conjure-jmh/src/jmh/java/com/palantir/conjure/MemoryProfiler.java @@ -0,0 +1,174 @@ +/* + * (c) Copyright 2025 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.conjure; + +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryUsage; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.infra.IterationParams; +import org.openjdk.jmh.profile.InternalProfiler; +import org.openjdk.jmh.results.AggregationPolicy; +import org.openjdk.jmh.results.IterationResult; +import org.openjdk.jmh.results.Result; +import org.openjdk.jmh.results.ScalarResult; + +public final class MemoryProfiler implements InternalProfiler { + private static final List retained = new ArrayList<>(); + private static final MemoryMXBean memoryBean = ManagementFactory.getMemoryMXBean(); + + private long beforeUsedMemory = 0L; + + /** + * Used to pre-allocate retained objects positions in the list to avoid resizing overhead during the benchmark. + */ + public static void ensureRetainedCapacity(int capacity) { + ((ArrayList) retained).ensureCapacity(capacity); + } + + public static void addRetained(Object obj) { + retained.add(obj); + } + + private static void clearRetained() { + retained.clear(); + } + + @Override + public void beforeIteration(BenchmarkParams _benchmarkParams, IterationParams _iterationParams) { + clearRetained(); + runSystemGC(); + beforeUsedMemory = getUsedMemory(); + } + + @Override + public Collection afterIteration( + BenchmarkParams _benchmarkParams, IterationParams _iterationParams, IterationResult result) { + runSystemGC(); + long afterUsedMemory = getUsedMemory(); + long retainedMemory = afterUsedMemory - beforeUsedMemory; + + List results = new ArrayList<>(); + results.add( + new ScalarResult("mem.retained.total", retainedMemory / 1024.0 / 1024.0, "MB", AggregationPolicy.AVG)); + results.add(new ScalarResult( + "mem.retained.total.norm", + (1.0 * retainedMemory) / result.getMetadata().getAllOps(), + "B/op", + AggregationPolicy.AVG)); + results.add(new ScalarResult("mem.retained.count", retained.size(), "obj", AggregationPolicy.AVG)); + return results; + } + + @Override + public String getDescription() { + return "Measures retained memory after each iteration"; + } + + private static final int MAX_WAIT_MSEC = 20 * 1000; + + @SuppressWarnings("checkstyle:CyclomaticComplexity") + // Same as BaseRunner#runSystemGC + private boolean runSystemGC() { + List enabledBeans = new ArrayList<>(); + + long beforeGcCount = 0; + for (GarbageCollectorMXBean bean : ManagementFactory.getGarbageCollectorMXBeans()) { + long count = bean.getCollectionCount(); + if (count != -1) { + enabledBeans.add(bean); + } + } + + for (GarbageCollectorMXBean bean : enabledBeans) { + beforeGcCount += bean.getCollectionCount(); + } + + // Run the GC twice, and force finalization before each GCs. + System.runFinalization(); + System.gc(); + System.runFinalization(); + System.gc(); + + // Now make sure GC actually happened. We have to wait for two things: + // a) That at least two collections happened, indicating GC work. + // b) That counter updates have not happened for a while, indicating GC work had ceased. + // + // Note there is an opportunity window for a concurrent GC to happen before the first + // System.gc() call, which would get counted towards our GCs. This race is unresolvable + // unless we have GC-specific information about the collection cycles, and verify those + // were indeed GCs triggered by us. + + if (enabledBeans.isEmpty()) { + System.out.println("WARNING: MXBeans can not report GC info. System.gc() invoked, pessimistically waiting " + + MAX_WAIT_MSEC + " msecs"); + try { + TimeUnit.MILLISECONDS.sleep(MAX_WAIT_MSEC); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return true; + } + + boolean gcHappened = false; + + long start = System.nanoTime(); + while (TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start) < MAX_WAIT_MSEC) { + try { + TimeUnit.MILLISECONDS.sleep(200); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + long afterGcCount = 0; + for (GarbageCollectorMXBean bean : enabledBeans) { + afterGcCount += bean.getCollectionCount(); + } + + if (!gcHappened) { + if (afterGcCount - beforeGcCount >= 2) { + gcHappened = true; + } + } else { + if (afterGcCount == beforeGcCount) { + // Stable! + return true; + } + beforeGcCount = afterGcCount; + } + } + + if (gcHappened) { + System.out.println("WARNING: System.gc() was invoked but unable to wait while GC stopped, is GC too" + + " asynchronous?"); + } else { + System.out.println("WARNING: System.gc() was invoked but couldn't detect a GC occurring, is System.gc()" + + " disabled?"); + } + return false; + } + + private long getUsedMemory() { + MemoryUsage heapUsage = memoryBean.getHeapMemoryUsage(); + return heapUsage.getUsed(); + } +} From 721b3b48f421cce53c434ff8533408800ab7e078 Mon Sep 17 00:00:00 2001 From: Alexis Le Dantec Date: Mon, 8 Dec 2025 17:47:34 +0100 Subject: [PATCH 5/5] Fix build.gradle --- build.gradle | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/build.gradle b/build.gradle index 56be80b7c..9ff17b5c8 100644 --- a/build.gradle +++ b/build.gradle @@ -49,7 +49,7 @@ apply plugin: 'com.palantir.jdks.latest' javaVersions { libraryTarget = 17 - runtime = 25 + runtime = 21 } allprojects { @@ -100,11 +100,6 @@ afterEvaluate { } } -javaVersions { - libraryTarget = 17 - runtime = 21 -} - jdks { daemonTarget = 21 }