diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index 5288f92dbe3..21f2260897f 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -26,6 +26,7 @@ import datadog.instrument.utils.ClassLoaderValue; import datadog.metrics.api.statsd.StatsDClientManager; import datadog.trace.api.Config; +import datadog.trace.api.InstrumenterConfig; import datadog.trace.api.Platform; import datadog.trace.api.WithGlobalTracer; import datadog.trace.api.appsec.AppSecEventTracker; @@ -336,6 +337,11 @@ public static void start( StaticEventLogger.end("crashtracking"); } + Object codeCoverageTransformer = null; + if (InstrumenterConfig.get().isCodeCoverageEnabled()) { + codeCoverageTransformer = maybeStartCodeCoverage(inst); + } + startDatadogAgent(initTelemetry, inst); final EnumSet libraries = detectLibraries(log); @@ -390,7 +396,8 @@ public static void start( } InstallDatadogTracerCallback installDatadogTracerCallback = - new InstallDatadogTracerCallback(initTelemetry, inst, okHttpDelayMillis); + new InstallDatadogTracerCallback( + initTelemetry, inst, okHttpDelayMillis, codeCoverageTransformer); if (waitForJUL) { log.debug("Custom logger detected. Delaying Datadog Tracer initialization."); registerLogManagerCallback(installDatadogTracerCallback); @@ -645,11 +652,14 @@ protected static class InstallDatadogTracerCallback extends ClassLoadCallBack { private final Object sco; private final Class scoClass; private final int okHttpDelayMillis; + private final Object codeCoverageTransformer; public InstallDatadogTracerCallback( InitializationTelemetry initTelemetry, Instrumentation instrumentation, - int okHttpDelayMillis) { + int okHttpDelayMillis, + Object codeCoverageTransformer) { + this.codeCoverageTransformer = codeCoverageTransformer; this.okHttpDelayMillis = okHttpDelayMillis; this.instrumentation = instrumentation; try { @@ -696,6 +706,10 @@ public void execute() { if (flareEnabled) { startFlarePoller(scoClass, sco); } + + if (codeCoverageTransformer != null) { + startCodeCoverageCollector(codeCoverageTransformer, sco); + } } private void resumeRemoteComponents() { @@ -1124,6 +1138,34 @@ private static void maybeStartCiVisibility(Instrumentation inst, Class scoCla } } + private static Object maybeStartCodeCoverage(Instrumentation inst) { + StaticEventLogger.begin("Code Coverage"); + + try { + final Class systemClass = + AGENT_CLASSLOADER.loadClass("datadog.trace.codecoverage.CodeCoverageSystem"); + final Method startMethod = systemClass.getMethod("start", Instrumentation.class); + return startMethod.invoke(null, inst); + } catch (final Throwable e) { + log.warn("Not starting Code Coverage subsystem", e); + return null; + } finally { + StaticEventLogger.end("Code Coverage"); + } + } + + private static void startCodeCoverageCollector(Object transformer, Object sco) { + try { + final Class systemClass = + AGENT_CLASSLOADER.loadClass("datadog.trace.codecoverage.CodeCoverageSystem"); + final Method startCollectorMethod = + systemClass.getMethod("startCollector", Object.class, Object.class); + startCollectorMethod.invoke(null, transformer, sco); + } catch (final Throwable e) { + log.warn("Not starting Code Coverage collector", e); + } + } + private static void maybeStartLLMObs(Instrumentation inst, Class scoClass, Object sco) { if (llmObsEnabled) { StaticEventLogger.begin("LLM Observability"); diff --git a/dd-java-agent/agent-ci-visibility/build.gradle b/dd-java-agent/agent-ci-visibility/build.gradle index 7811fa39140..070c6e5c06b 100644 --- a/dd-java-agent/agent-ci-visibility/build.gradle +++ b/dd-java-agent/agent-ci-visibility/build.gradle @@ -27,6 +27,7 @@ dependencies { implementation project(':components:json') implementation project(':internal-api') implementation project(':internal-api:internal-api-9') + implementation project(':utils:coverage-utils') testImplementation project(':dd-java-agent:testing') testImplementation("com.google.jimfs:jimfs:1.1") // an in-memory file system for testing code that works with files diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityCoverageServices.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityCoverageServices.java index c83beb14283..0fa7dc81d4f 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityCoverageServices.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityCoverageServices.java @@ -10,9 +10,13 @@ import datadog.trace.civisibility.coverage.SkippableAwareCoverageStoreFactory; import datadog.trace.civisibility.coverage.file.FileCoverageStore; import datadog.trace.civisibility.coverage.line.LineCoverageStore; +import datadog.communication.http.OkHttpUtils; +import datadog.trace.api.civisibility.telemetry.CiVisibilityCountMetric; +import datadog.trace.api.civisibility.telemetry.CiVisibilityDistributionMetric; +import datadog.trace.civisibility.communication.TelemetryListener; import datadog.trace.civisibility.coverage.report.CoverageProcessor; -import datadog.trace.civisibility.coverage.report.CoverageReportUploader; import datadog.trace.civisibility.coverage.report.JacocoCoverageProcessor; +import datadog.trace.coverage.CoverageReportUploader; import datadog.trace.civisibility.coverage.report.child.ChildProcessCoverageReporter; import datadog.trace.civisibility.coverage.report.child.JacocoChildProcessCoverageReporter; import datadog.trace.civisibility.domain.buildsystem.ModuleSignalRouter; @@ -34,11 +38,20 @@ static class Parent { ExecutionSettings executionSettings = repoServices.executionSettingsFactory.create(JvmInfo.CURRENT_JVM, null); - CoverageReportUploader coverageReportUploader = - executionSettings.isCodeCoverageReportUploadEnabled() - ? new CoverageReportUploader( - services.ciIntake, repoServices.ciTags, services.metricCollector) - : null; + CoverageReportUploader coverageReportUploader; + if (executionSettings.isCodeCoverageReportUploadEnabled()) { + OkHttpUtils.CustomListener telemetryListener = + new TelemetryListener.Builder(services.metricCollector) + .requestCount(CiVisibilityCountMetric.COVERAGE_UPLOAD_REQUEST) + .requestBytes(CiVisibilityDistributionMetric.COVERAGE_UPLOAD_REQUEST_BYTES) + .requestErrors(CiVisibilityCountMetric.COVERAGE_UPLOAD_REQUEST_ERRORS) + .requestDuration(CiVisibilityDistributionMetric.COVERAGE_UPLOAD_REQUEST_MS) + .build(); + coverageReportUploader = + new CoverageReportUploader(services.ciIntake, repoServices.ciTags, telemetryListener); + } else { + coverageReportUploader = null; + } coverageProcessorFactory = new JacocoCoverageProcessor.Factory( diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/report/JacocoCoverageProcessor.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/report/JacocoCoverageProcessor.java index db630801e84..c87ca9a0d8b 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/report/JacocoCoverageProcessor.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/report/JacocoCoverageProcessor.java @@ -5,6 +5,9 @@ import datadog.trace.api.civisibility.domain.SourceSet; import datadog.trace.civisibility.config.ExecutionSettings; import datadog.trace.civisibility.domain.buildsystem.ModuleSignalRouter; +import datadog.trace.coverage.CoverageReportUploader; +import datadog.trace.coverage.LcovReportWriter; +import datadog.trace.coverage.LinesCoverage; import datadog.trace.civisibility.ipc.AckResponse; import datadog.trace.civisibility.ipc.ModuleCoverageDataJacoco; import datadog.trace.civisibility.ipc.SignalResponse; diff --git a/dd-java-agent/agent-code-coverage/build.gradle b/dd-java-agent/agent-code-coverage/build.gradle new file mode 100644 index 00000000000..6c86afcf1f7 --- /dev/null +++ b/dd-java-agent/agent-code-coverage/build.gradle @@ -0,0 +1,31 @@ +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar + +plugins { + id 'com.gradleup.shadow' +} + +apply from: "$rootDir/gradle/java.gradle" +apply from: "$rootDir/gradle/version.gradle" + +minimumBranchCoverage = 0.0 +minimumInstructionCoverage = 0.0 + +dependencies { + api libs.slf4j + + implementation group: 'org.jacoco', name: 'org.jacoco.core', version: '0.8.14' + + implementation project(':internal-api') + implementation project(':communication') + implementation project(':utils:coverage-utils') + + testImplementation project(':dd-java-agent:testing') +} + +tasks.named("shadowJar", ShadowJar) { + dependencies deps.excludeShared +} + +tasks.named("jar", Jar) { + archiveClassifier = 'unbundled' +} diff --git a/dd-java-agent/agent-code-coverage/src/main/java/datadog/trace/codecoverage/ClassProbeMapping.java b/dd-java-agent/agent-code-coverage/src/main/java/datadog/trace/codecoverage/ClassProbeMapping.java new file mode 100644 index 00000000000..71c20302ed4 --- /dev/null +++ b/dd-java-agent/agent-code-coverage/src/main/java/datadog/trace/codecoverage/ClassProbeMapping.java @@ -0,0 +1,28 @@ +package datadog.trace.codecoverage; + +import java.util.BitSet; + +/** + * Cached mapping from probe IDs to source lines for a single class. Built once per class version + * (identified by CRC64) and reused across collection cycles. + */ +final class ClassProbeMapping { + final long classId; + final String className; // "com/example/MyClass" + final String sourceFile; // "SourceFile.java" + final BitSet executableLines; + final int[][] probeToLines; // probeToLines[probeId] = sorted line numbers + + ClassProbeMapping( + long classId, + String className, + String sourceFile, + BitSet executableLines, + int[][] probeToLines) { + this.classId = classId; + this.className = className; + this.sourceFile = sourceFile; + this.executableLines = executableLines; + this.probeToLines = probeToLines; + } +} diff --git a/dd-java-agent/agent-code-coverage/src/main/java/datadog/trace/codecoverage/ClassProbeMappingBuilder.java b/dd-java-agent/agent-code-coverage/src/main/java/datadog/trace/codecoverage/ClassProbeMappingBuilder.java new file mode 100644 index 00000000000..0ba0c734f99 --- /dev/null +++ b/dd-java-agent/agent-code-coverage/src/main/java/datadog/trace/codecoverage/ClassProbeMappingBuilder.java @@ -0,0 +1,358 @@ +package datadog.trace.codecoverage; + +import java.util.ArrayList; +import java.util.BitSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.jacoco.core.internal.flow.ClassProbesAdapter; +import org.jacoco.core.internal.flow.ClassProbesVisitor; +import org.jacoco.core.internal.flow.IFrame; +import org.jacoco.core.internal.flow.LabelInfo; +import org.jacoco.core.internal.flow.MethodProbesVisitor; +import org.jacoco.core.internal.instr.InstrSupport; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.Handle; +import org.objectweb.asm.Label; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.tree.AbstractInsnNode; +import org.objectweb.asm.tree.MethodNode; +import org.objectweb.asm.tree.TryCatchBlockNode; + +/** + * Builds a {@link ClassProbeMapping} by parsing the class bytecode once using JaCoCo's {@link + * ClassProbesAdapter}, building a simplified instruction graph, and walking predecessor chains to + * determine which lines each probe covers. + * + *

This replaces the previous N+1 pass approach (one {@code Analyzer} pass per probe plus one for + * executable lines) with a single-pass design that is significantly faster for classes with many + * probes. + */ +final class ClassProbeMappingBuilder { + + static ClassProbeMapping build( + long classId, String className, int probeCount, byte[] classBytes) { + ClassReader reader = InstrSupport.classReaderFor(classBytes); + ProbeMappingVisitor visitor = new ProbeMappingVisitor(); + ClassProbesAdapter adapter = new ClassProbesAdapter(visitor, false); + reader.accept(adapter, 0); + return visitor.toMapping(classId, className, probeCount); + } + + /** Simplified instruction node with a line number and a single predecessor link. */ + static final class ProbeNode { + final int line; + ProbeNode predecessor; + + ProbeNode(int line) { + this.line = line; + } + } + + /** A deferred jump from a source instruction to a target label. */ + static final class Jump { + final ProbeNode source; + final Label target; + final int branch; + + Jump(ProbeNode source, Label target, int branch) { + this.source = source; + this.target = target; + this.branch = branch; + } + } + + /** + * Class-level visitor that collects source file info and delegates method visiting to {@link + * MethodMapper}. + */ + private static final class ProbeMappingVisitor extends ClassProbesVisitor { + private String sourceFile; + private final BitSet executableLines = new BitSet(); + private final Map probeToLines = new HashMap<>(); + + @Override + public void visitSource(String source, String debug) { + sourceFile = source; + } + + @Override + public MethodProbesVisitor visitMethod( + int access, String name, String desc, String signature, String[] exceptions) { + return new MethodMapper(executableLines, probeToLines); + } + + @Override + public void visitTotalProbeCount(int count) { + // no-op; we get probeCount from the caller + } + + ClassProbeMapping toMapping(long classId, String className, int probeCount) { + int[][] probeToLinesArray = new int[probeCount][]; + for (int p = 0; p < probeCount; p++) { + BitSet lines = probeToLines.get(p); + probeToLinesArray[p] = (lines != null) ? bitSetToArray(lines) : new int[0]; + } + return new ClassProbeMapping(classId, className, sourceFile, executableLines, probeToLinesArray); + } + + private static int[] bitSetToArray(BitSet bs) { + int[] result = new int[bs.cardinality()]; + int idx = 0; + for (int bit = bs.nextSetBit(0); bit >= 0; bit = bs.nextSetBit(bit + 1)) { + result[idx++] = bit; + } + return result; + } + } + + /** + * Method-level visitor that builds a simplified instruction graph (with predecessor links) and + * records probe-to-instruction associations. After all instructions are replayed, jump targets are + * wired and predecessor chains are walked to collect covered lines per probe. + * + *

This replicates the logic of JaCoCo's {@code InstructionsBuilder} and {@code + * MethodAnalyzer}, which are package-private and cannot be used directly from this package. + */ + private static final class MethodMapper extends MethodProbesVisitor { + private static final int UNKNOWN_LINE = -1; + + private final BitSet executableLines; + private final Map probeToLines; + + // Per-method state + private int currentLine = UNKNOWN_LINE; + private ProbeNode currentInsn; + private final List

On the first collection cycle (or when new classes appear), a classpath scan builds the + * cache. Subsequent cycles simply iterate boolean probe arrays and set bits -- no JaCoCo {@code + * Analyzer} pass is needed. + */ +public final class CodeCoverageCollector { + + private static final Logger log = LoggerFactory.getLogger(CodeCoverageCollector.class); + + private final CodeCoverageTransformer transformer; + private final CodeCoverageSender sender; + private final int intervalSeconds; + private final String explicitClasspath; + private final ProbeMappingCache probeCache = new ProbeMappingCache(); + private final AgentTaskScheduler scheduler = new AgentTaskScheduler(CODE_COVERAGE); + + /** + * @param transformer the transformer that holds runtime probe data + * @param sender the sender to deliver coverage results to + * @param intervalSeconds interval between collection cycles + * @param explicitClasspath explicit classpath override (nullable; if null, auto-detected) + */ + public CodeCoverageCollector( + CodeCoverageTransformer transformer, + CodeCoverageSender sender, + int intervalSeconds, + String explicitClasspath) { + this.transformer = transformer; + this.sender = sender; + this.intervalSeconds = intervalSeconds; + this.explicitClasspath = explicitClasspath; + } + + /** Starts the periodic collection scheduler. */ + public void start() { + scheduler.scheduleAtFixedRate( + this::collect, intervalSeconds, intervalSeconds, TimeUnit.SECONDS); + log.debug( + "Code coverage collector started with interval of {} seconds", intervalSeconds); + } + + /** Stops the periodic collection scheduler. */ + public void stop() { + scheduler.shutdown(5, TimeUnit.SECONDS); + } + + /** Performs a single collection cycle: collect probes, resolve via cache, and send. */ + void collect() { + try { + // 1. Collect and reset probes + ExecutionDataStore execStore = new ExecutionDataStore(); + SessionInfoStore sessionStore = new SessionInfoStore(); + transformer.collectAndReset(execStore, sessionStore); + + // 2. Separate cache hits from misses + Collection allEntries = execStore.getContents(); + List cacheMisses = new ArrayList<>(); + for (ExecutionData ed : allEntries) { + if (probeCache.get(ed.getId()) == null) { + cacheMisses.add(ed); + } + } + + // 3. Build cache entries for misses (scans classpath) + if (!cacheMisses.isEmpty()) { + List classpathEntries = resolveClasspath(); + probeCache.buildMissing(cacheMisses, classpathEntries); + log.debug("Built cache entries for {} new classes", cacheMisses.size()); + } + + // 4. Build coverage from cache + Map coverage = new HashMap<>(); + for (ExecutionData ed : allEntries) { + ClassProbeMapping mapping = probeCache.get(ed.getId()); + if (mapping == null || mapping.className == null || mapping.sourceFile == null) { + continue; // no mapping available + } + + CoverageKey key = new CoverageKey(mapping.sourceFile, mapping.className); + LinesCoverage lc = coverage.computeIfAbsent(key, k -> new LinesCoverage()); + lc.executableLines.or(mapping.executableLines); + + boolean[] probes = ed.getProbes(); + for (int p = 0; p < probes.length && p < mapping.probeToLines.length; p++) { + if (probes[p]) { + for (int line : mapping.probeToLines[p]) { + lc.coveredLines.set(line); + } + } + } + } + + // 5. Send if there is data + if (!coverage.isEmpty()) { + sender.upload(coverage); + } + } catch (Exception e) { + log.debug("Error during code coverage collection", e); + } + } + + /** + * Resolves classpath entries to analyze. If an explicit classpath is configured, it takes + * precedence. Otherwise, falls back to {@code java.class.path} system property. + */ + private List resolveClasspath() { + String cp; + if (explicitClasspath != null && !explicitClasspath.isEmpty()) { + cp = explicitClasspath; + } else { + cp = System.getProperty("java.class.path"); + } + List entries = new ArrayList<>(); + if (cp != null && !cp.isEmpty()) { + for (String path : cp.split(File.pathSeparator)) { + String trimmed = path.trim(); + if (!trimmed.isEmpty()) { + entries.add(new File(trimmed)); + } + } + } + return entries; + } +} diff --git a/dd-java-agent/agent-code-coverage/src/main/java/datadog/trace/codecoverage/CodeCoverageFilter.java b/dd-java-agent/agent-code-coverage/src/main/java/datadog/trace/codecoverage/CodeCoverageFilter.java new file mode 100644 index 00000000000..204335a6b8b --- /dev/null +++ b/dd-java-agent/agent-code-coverage/src/main/java/datadog/trace/codecoverage/CodeCoverageFilter.java @@ -0,0 +1,84 @@ +package datadog.trace.codecoverage; + +import java.util.function.Predicate; + +/** + * Determines whether a class should be instrumented for production code coverage based on + * include/exclude patterns. + */ +public final class CodeCoverageFilter implements Predicate { + + private final String[] includePrefixes; + private final String[] excludePrefixes; + private final boolean includeAll; + + /** + * @param includes include patterns (e.g. {@code ["com.example.*", "*"]}). A single {@code "*"} + * means include everything. + * @param excludes exclude patterns (e.g. {@code ["com.example.internal.*"]}) + */ + public CodeCoverageFilter(String[] includes, String[] excludes) { + this.includeAll = includes.length == 1 && "*".equals(includes[0]); + this.includePrefixes = toVmPrefixes(includes); + this.excludePrefixes = toVmPrefixes(excludes); + } + + /** + * @param className class name in VM format (e.g. {@code "com/example/MyClass"}) + * @return {@code true} if the class should be instrumented + */ + @Override + public boolean test(String className) { + // Always reject agent internals + if (className.startsWith("datadog/")) { + return false; + } + + // Check excludes first + for (String excludePrefix : excludePrefixes) { + if (className.startsWith(excludePrefix)) { + return false; + } + } + + if (includeAll) { + return true; + } + + // Check includes + for (String includePrefix : includePrefixes) { + if (className.startsWith(includePrefix)) { + return true; + } + } + + return false; + } + + /** + * Converts dot-separated patterns like {@code "com.example.*"} to VM-format prefixes like {@code + * "com/example/"}. + */ + private static String[] toVmPrefixes(String[] patterns) { + if (patterns == null || patterns.length == 0) { + return new String[0]; + } + String[] prefixes = new String[patterns.length]; + for (int i = 0; i < patterns.length; i++) { + String pattern = patterns[i].trim(); + if ("*".equals(pattern)) { + prefixes[i] = ""; + continue; + } + // Strip trailing wildcard + if (pattern.endsWith(".*") || pattern.endsWith("/*")) { + pattern = pattern.substring(0, pattern.length() - 1); + } else if (pattern.endsWith("*")) { + pattern = pattern.substring(0, pattern.length() - 1); + } + // Convert dots to slashes + prefixes[i] = pattern.replace('.', '/'); + } + return prefixes; + } +} diff --git a/dd-java-agent/agent-code-coverage/src/main/java/datadog/trace/codecoverage/CodeCoverageSender.java b/dd-java-agent/agent-code-coverage/src/main/java/datadog/trace/codecoverage/CodeCoverageSender.java new file mode 100644 index 00000000000..4178a906f12 --- /dev/null +++ b/dd-java-agent/agent-code-coverage/src/main/java/datadog/trace/codecoverage/CodeCoverageSender.java @@ -0,0 +1,30 @@ +package datadog.trace.codecoverage; + +import datadog.trace.coverage.CoverageKey; +import datadog.trace.coverage.CoverageReportUploader; +import datadog.trace.coverage.LinesCoverage; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class CodeCoverageSender { + private static final Logger log = LoggerFactory.getLogger(CodeCoverageSender.class); + private final CoverageReportUploader uploader; + + public CodeCoverageSender(CoverageReportUploader uploader) { + this.uploader = uploader; + } + + public void upload(Map coverage) { + try { + ByteArrayOutputStream buf = new ByteArrayOutputStream(); + CoverageBinaryEncoder.encode(coverage, buf); + uploader.upload("ddcov", new ByteArrayInputStream(buf.toByteArray())); + } catch (IOException e) { + log.debug("Failed to upload code coverage report", e); + } + } +} diff --git a/dd-java-agent/agent-code-coverage/src/main/java/datadog/trace/codecoverage/CodeCoverageSystem.java b/dd-java-agent/agent-code-coverage/src/main/java/datadog/trace/codecoverage/CodeCoverageSystem.java new file mode 100644 index 00000000000..59707c105a0 --- /dev/null +++ b/dd-java-agent/agent-code-coverage/src/main/java/datadog/trace/codecoverage/CodeCoverageSystem.java @@ -0,0 +1,155 @@ +package datadog.trace.codecoverage; + +import datadog.communication.BackendApi; +import datadog.communication.BackendApiFactory; +import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.trace.api.Config; +import datadog.trace.api.DDTags; +import datadog.trace.api.git.CommitInfo; +import datadog.trace.api.git.GitInfo; +import datadog.trace.api.git.GitInfoProvider; +import datadog.trace.api.git.PersonInfo; +import datadog.trace.api.intake.Intake; +import datadog.trace.coverage.CoverageReportUploader; +import java.lang.instrument.Instrumentation; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Predicate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Entry point for the production code coverage product module. + * + *

Follows the tracer's standard product system pattern with a two-phase start: + * + *

    + *
  1. {@link #start(Instrumentation)} — called during premain, before ByteBuddy's + * transformer is registered. Must not use logging, NIO, or JMX. + *
  2. {@link #startCollector(Object, Object)} — called from a deferred callback after premain, + * when logging and thread scheduling are safe. + *
+ */ +public final class CodeCoverageSystem { + + private static final Logger log = LoggerFactory.getLogger(CodeCoverageSystem.class); + + /** + * Phase 1: registers the coverage {@link java.lang.instrument.ClassFileTransformer}. + * + *

Called during premain, synchronously, before ByteBuddy. The returned object is an opaque + * handle to the transformer, passed to {@link #startCollector(Object, Object)} later. + * + * @param inst the JVM instrumentation service + * @return the transformer instance (opaque; passed to {@link #startCollector}) + * @throws Exception if JaCoCo runtime initialization fails + */ + public static Object start(Instrumentation inst) throws Exception { + Config config = Config.get(); + String[] includes = config.getCodeCoverageIncludes(); + String[] excludes = config.getCodeCoverageExcludes(); + Predicate filter = new CodeCoverageFilter(includes, excludes); + CodeCoverageTransformer transformer = new CodeCoverageTransformer(inst, filter); + inst.addTransformer(transformer); + return transformer; + } + + /** + * Phase 2: starts the periodic coverage collector. + * + *

Called from a deferred callback after premain. Safe to use logging and thread scheduling. + * + * @param transformerObj the opaque transformer handle returned by {@link #start} + * @param scoObj the SharedCommunicationObjects instance for backend communication + */ + public static void startCollector(Object transformerObj, Object scoObj) { + CodeCoverageTransformer transformer = (CodeCoverageTransformer) transformerObj; + Config config = Config.get(); + + // Build event tags from git info + Map tags = buildGitTags(); + if (!tags.containsKey("git.commit.sha")) { + log.warn( + "DD_GIT_COMMIT_SHA is not set; " + + "code coverage reports cannot be uploaded without a commit SHA"); + return; + } + + // Create BackendApi for coverage uploads + BackendApiFactory factory = + new BackendApiFactory(config, (SharedCommunicationObjects) scoObj); + BackendApi backendApi = factory.createBackendApi(Intake.CI_INTAKE); + if (backendApi == null) { + log.warn( + "Cannot create backend API for code coverage uploads; " + + "agent may not support EVP proxy"); + return; + } + + tags.put(DDTags.LANGUAGE_TAG_KEY, DDTags.LANGUAGE_TAG_VALUE); + String env = config.getEnv(); + if (env != null && !env.isEmpty()) { + tags.put("runtime.env", env); + } + String serviceName = config.getServiceName(); + if (serviceName != null && !serviceName.isEmpty()) { + tags.put("report.flags", Collections.singletonList("service:" + serviceName)); + } + + CoverageReportUploader uploader = new CoverageReportUploader(backendApi, tags, null); + CodeCoverageSender sender = new CodeCoverageSender(uploader); + + CodeCoverageCollector collector = + new CodeCoverageCollector( + transformer, + sender, + config.getCodeCoverageReportIntervalSeconds(), + config.getCodeCoverageClasspath()); + collector.start(); + } + + private static Map buildGitTags() { + Map tags = new HashMap<>(); + GitInfo gitInfo = GitInfoProvider.INSTANCE.getGitInfo(); + CommitInfo commit = gitInfo.getCommit(); + if (commit != null && commit.getSha() != null) { + tags.put("git.commit.sha", commit.getSha()); + } + if (gitInfo.getRepositoryURL() != null) { + tags.put("git.repository_url", gitInfo.getRepositoryURL()); + } + if (gitInfo.getBranch() != null) { + tags.put("git.branch", gitInfo.getBranch()); + } + // Add author/committer info if available + if (commit != null) { + PersonInfo author = commit.getAuthor(); + if (author.getName() != null) { + tags.put("git.commit.author.name", author.getName()); + } + if (author.getEmail() != null) { + tags.put("git.commit.author.email", author.getEmail()); + } + if (author.getIso8601Date() != null) { + tags.put("git.commit.author.date", author.getIso8601Date()); + } + PersonInfo committer = commit.getCommitter(); + if (committer.getName() != null) { + tags.put("git.commit.committer.name", committer.getName()); + } + if (committer.getEmail() != null) { + tags.put("git.commit.committer.email", committer.getEmail()); + } + if (committer.getIso8601Date() != null) { + tags.put("git.commit.committer.date", committer.getIso8601Date()); + } + if (commit.getFullMessage() != null) { + tags.put("git.commit.message", commit.getFullMessage()); + } + } + return tags; + } + + private CodeCoverageSystem() {} +} diff --git a/dd-java-agent/agent-code-coverage/src/main/java/datadog/trace/codecoverage/CodeCoverageTransformer.java b/dd-java-agent/agent-code-coverage/src/main/java/datadog/trace/codecoverage/CodeCoverageTransformer.java new file mode 100644 index 00000000000..7ad06a4e831 --- /dev/null +++ b/dd-java-agent/agent-code-coverage/src/main/java/datadog/trace/codecoverage/CodeCoverageTransformer.java @@ -0,0 +1,218 @@ +package datadog.trace.codecoverage; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.lang.instrument.ClassFileTransformer; +import java.lang.instrument.Instrumentation; +import java.security.ProtectionDomain; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.function.Predicate; +import org.jacoco.core.data.ExecutionDataReader; +import org.jacoco.core.data.ExecutionDataStore; +import org.jacoco.core.data.ExecutionDataWriter; +import org.jacoco.core.data.SessionInfoStore; +import org.jacoco.core.instr.Instrumenter; +import org.jacoco.core.runtime.IRuntime; +import org.jacoco.core.runtime.InjectedClassRuntime; +import org.jacoco.core.runtime.RuntimeData; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A {@link ClassFileTransformer} that uses JaCoCo's {@link Instrumenter} to insert boolean probes + * into class bytecode at load time. + * + *

Must be registered before ByteBuddy's transformer so that JaCoCo sees original class + * bytes (CRC64 must match the {@code .class} files on disk for analysis to work). + */ +public final class CodeCoverageTransformer implements ClassFileTransformer { + + private static final Logger log = LoggerFactory.getLogger(CodeCoverageTransformer.class); + + private final RuntimeData runtimeData; + private final Instrumenter instrumenter; + private final Predicate filter; + + /** + * Initializes the JaCoCo runtime and instrumenter. + * + *

This replicates the logic from JaCoCo's {@code AgentModule} and {@code PreMain}: it creates + * an isolated classloader, opens {@code java.lang} to it via {@code + * Instrumentation.redefineModule}, loads {@link InjectedClassRuntime} in that module, and starts + * the runtime. + * + * @param inst the JVM instrumentation service + * @param filter predicate that decides which classes to instrument (VM class name format) + * @throws Exception if the JaCoCo runtime cannot be initialized + */ + public CodeCoverageTransformer(Instrumentation inst, Predicate filter) throws Exception { + this.filter = filter; + this.runtimeData = new RuntimeData(); + + // Replicate AgentModule logic: create isolated classloader and open java.lang to it + Set scope = new HashSet<>(); + addToScopeWithInnerClasses(InjectedClassRuntime.class, scope); + + // Use the classloader that has the (shaded) JaCoCo classes as the resource source and parent. + // The parent provides access to AbstractRuntime, IRuntime, RuntimeData, ASM classes, etc. + // Scoped classes (InjectedClassRuntime and its inner classes) are re-defined in the isolated + // classloader so they belong to its distinct unnamed module — which has java.lang opened to it. + ClassLoader agentLoader = CodeCoverageTransformer.class.getClassLoader(); + + ClassLoader isolatedLoader = + new ClassLoader(agentLoader) { + @Override + protected Class loadClass(String name, boolean resolve) + throws ClassNotFoundException { + if (!scope.contains(name)) { + return super.loadClass(name, resolve); + } + InputStream resourceStream = + agentLoader.getResourceAsStream(name.replace('.', '/') + ".class"); + if (resourceStream == null) { + throw new ClassNotFoundException(name); + } + byte[] bytes; + try { + bytes = readAllBytes(resourceStream); + } catch (IOException e) { + throw new RuntimeException(e); + } + return defineClass( + name, bytes, 0, bytes.length, CodeCoverageTransformer.class.getProtectionDomain()); + } + }; + + // Open java.lang package to the isolated classloader's unnamed module + openPackage(inst, Object.class, isolatedLoader); + + // Load InjectedClassRuntime in the isolated module + @SuppressWarnings("unchecked") + Class rtClass = + (Class) isolatedLoader.loadClass(InjectedClassRuntime.class.getName()); + + IRuntime runtime = + rtClass + .getConstructor(Class.class, String.class) + .newInstance(Object.class, "$DDCov"); + + runtime.startup(runtimeData); + this.instrumenter = new Instrumenter(runtime); + } + + @Override + public byte[] transform( + ClassLoader loader, + String className, + Class classBeingRedefined, + ProtectionDomain pd, + byte[] classfileBuffer) { + if (classBeingRedefined != null) { + return null; // retransformation not supported (schema change) + } + if (className == null || loader == null) { + return null; // skip bootstrap classes and unnamed classes + } + if (!filter.test(className)) { + return null; + } + try { + return instrumenter.instrument(classfileBuffer, className); + } catch (Exception e) { + log.debug("Failed to instrument class {}", className, e); + return null; + } + } + + /** + * Collects current probe data and resets all probes to {@code false}. + * + *

Uses a serialize/deserialize round-trip to capture probe values before reset. This is + * necessary because {@code RuntimeData.collect()} passes references to the live {@code boolean[]} + * probe arrays to the visitor. If we passed an {@code ExecutionDataStore} directly, it would store + * references to the same arrays that {@code reset()} then zeroes out — destroying the collected + * data. The byte-stream approach (same as JaCoCo's own {@code Agent.getExecutionData()}) captures + * probe values into the stream before the reset runs. + * + * @param target store to receive the execution data + * @param sessionTarget store to receive session info + */ + public void collectAndReset(ExecutionDataStore target, SessionInfoStore sessionTarget) { + try { + // Serialize probe data to bytes (captures values before reset) + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + ExecutionDataWriter writer = new ExecutionDataWriter(buffer); + runtimeData.collect(writer, writer, true); + + // Deserialize into the target stores + ExecutionDataReader reader = + new ExecutionDataReader(new java.io.ByteArrayInputStream(buffer.toByteArray())); + reader.setExecutionDataVisitor(target); + reader.setSessionInfoVisitor(sessionTarget); + reader.read(); + } catch (IOException e) { + throw new RuntimeException("Failed to collect coverage data", e); + } + } + + /** + * Opens the package of {@code classInPackage} to the unnamed module of {@code targetLoader}. + * + *

This uses {@code Instrumentation.redefineModule} reflectively (same approach as JaCoCo's + * {@code AgentModule.openPackage}). + */ + private static void openPackage( + Instrumentation inst, Class classInPackage, ClassLoader targetLoader) throws Exception { + // module of the package to open (e.g. java.base for java.lang) + Object module = Class.class.getMethod("getModule").invoke(classInPackage); + + // unnamed module of the isolated classloader + Object unnamedModule = ClassLoader.class.getMethod("getUnnamedModule").invoke(targetLoader); + + Class moduleClass = Class.forName("java.lang.Module"); + + // Instrumentation.redefineModule(Module, Set, Map, Map>, Set, Map) + Instrumentation.class + .getMethod( + "redefineModule", + moduleClass, + Set.class, + Map.class, + Map.class, + Set.class, + Map.class) + .invoke( + inst, + module, // module to modify + Collections.emptySet(), // extraReads + Collections.emptyMap(), // extraExports + Collections.singletonMap( + classInPackage.getPackage().getName(), + Collections.singleton(unnamedModule)), // extraOpens + Collections.emptySet(), // extraUses + Collections.emptyMap()); // extraProvides + } + + /** Recursively adds the given class and all its declared inner classes to the scope set. */ + private static void addToScopeWithInnerClasses(Class clazz, Set scope) { + scope.add(clazz.getName()); + for (Class inner : clazz.getDeclaredClasses()) { + addToScopeWithInnerClasses(inner, scope); + } + } + + /** Reads all bytes from an input stream. */ + private static byte[] readAllBytes(InputStream is) throws IOException { + byte[] buf = new byte[1024]; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + int r; + while ((r = is.read(buf)) != -1) { + out.write(buf, 0, r); + } + return out.toByteArray(); + } +} diff --git a/dd-java-agent/agent-code-coverage/src/main/java/datadog/trace/codecoverage/CoverageBinaryEncoder.java b/dd-java-agent/agent-code-coverage/src/main/java/datadog/trace/codecoverage/CoverageBinaryEncoder.java new file mode 100644 index 00000000000..3a7dff733c3 --- /dev/null +++ b/dd-java-agent/agent-code-coverage/src/main/java/datadog/trace/codecoverage/CoverageBinaryEncoder.java @@ -0,0 +1,68 @@ +package datadog.trace.codecoverage; + +import datadog.trace.coverage.CoverageKey; +import datadog.trace.coverage.LinesCoverage; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.BitSet; +import java.util.Map; + +/** + * Encodes coverage data into the Coverage Binary Protocol v1 + */ +public final class CoverageBinaryEncoder { + + private static final int VERSION = 1; + private static final int NUM_EXTRA_FIELDS = 1; // className + + public static void encode(Map coverage, OutputStream out) + throws IOException { + out.write(VERSION); + writeUvarint(NUM_EXTRA_FIELDS, out); + writeUvarint(coverage.size(), out); + for (Map.Entry entry : coverage.entrySet()) { + writeRecord(entry.getKey(), entry.getValue(), out); + } + } + + private static void writeRecord(CoverageKey key, LinesCoverage lines, OutputStream out) + throws IOException { + writeString(key.sourceFile, out); + writeString(key.className, out); + + int maxLine = + Math.max(lines.executableLines.length(), lines.coveredLines.length()) - 1; + if (maxLine < 0) { + writeUvarint(0, out); + return; + } + int byteCount = (maxLine >> 3) + 1; + writeUvarint(byteCount, out); + writeBitVector(lines.executableLines, byteCount, out); + writeBitVector(lines.coveredLines, byteCount, out); + } + + private static void writeBitVector(BitSet bits, int byteCount, OutputStream out) + throws IOException { + byte[] data = bits.toByteArray(); + out.write(data, 0, Math.min(data.length, byteCount)); + for (int i = data.length; i < byteCount; i++) { + out.write(0); + } + } + + private static void writeString(String s, OutputStream out) throws IOException { + byte[] bytes = s.getBytes(StandardCharsets.UTF_8); + writeUvarint(bytes.length, out); + out.write(bytes); + } + + static void writeUvarint(int value, OutputStream out) throws IOException { + while (value >= 0x80) { + out.write((value & 0x7F) | 0x80); + value >>>= 7; + } + out.write(value); + } +} diff --git a/dd-java-agent/agent-code-coverage/src/main/java/datadog/trace/codecoverage/ProbeMappingCache.java b/dd-java-agent/agent-code-coverage/src/main/java/datadog/trace/codecoverage/ProbeMappingCache.java new file mode 100644 index 00000000000..62169796b3c --- /dev/null +++ b/dd-java-agent/agent-code-coverage/src/main/java/datadog/trace/codecoverage/ProbeMappingCache.java @@ -0,0 +1,224 @@ +package datadog.trace.codecoverage; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import org.jacoco.core.data.ExecutionData; +import org.jacoco.core.internal.data.CRC64; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Maintains a cache of {@link ClassProbeMapping} entries, keyed by CRC64 class ID. Builds entries + * lazily from classpath analysis when cache misses occur. + */ +final class ProbeMappingCache { + + private static final Logger log = LoggerFactory.getLogger(ProbeMappingCache.class); + + private final Map cache = new HashMap<>(); + + /** Returns the cached mapping for the given class ID, or null if not cached. */ + ClassProbeMapping get(long classId) { + return cache.get(classId); + } + + /** + * Populates cache entries for all classes in {@code missingClasses}. First attempts targeted + * lookup via the context classloader's {@code getResourceAsStream} (O(1) per class). Any classes + * that can't be resolved this way fall back to a full classpath scan. + * + * @param missingClasses execution data entries that have no cached mapping + * @param classpathEntries jars/directories to scan as fallback + */ + void buildMissing(Collection missingClasses, List classpathEntries) { + // Build a lookup: classId -> ExecutionData for the missing entries + Map needed = new HashMap<>(); + for (ExecutionData ed : missingClasses) { + needed.put(ed.getId(), ed); + } + + // Phase 1: targeted classloader lookup (fast path) + resolveViaClassloader(needed); + + // Phase 2: fall back to classpath scan for anything still unresolved + if (!needed.isEmpty()) { + resolveViaClasspathScan(needed, classpathEntries); + } + + // Any remaining entries couldn't be found anywhere. + // Mark them with a sentinel so we don't retry on subsequent cycles. + for (Map.Entry e : needed.entrySet()) { + cache.put( + e.getKey(), + new ClassProbeMapping(e.getKey(), null, null, new BitSet(), new int[0][])); + log.debug( + "Class {} (id {}) not found on classpath; skipping", + e.getValue().getName(), + Long.toHexString(e.getKey())); + } + } + + /** + * Attempts to resolve missing classes via the context classloader's resource lookup. This is O(1) + * per class — the classloader already knows where each class file lives. CRC64 is verified after + * reading to ensure the bytes match what JaCoCo instrumented. + */ + private void resolveViaClassloader(Map needed) { + // Try the system classloader (application classpath) first, then the context classloader. + // The dd-code-coverage thread inherits the agent's context classloader, which typically + // can't find application classes. The system classloader is the standard app classloader. + ClassLoader systemCl = ClassLoader.getSystemClassLoader(); + ClassLoader contextCl = Thread.currentThread().getContextClassLoader(); + + // Iterate over a copy since we modify 'needed' during iteration + for (ExecutionData ed : new ArrayList<>(needed.values())) { + String resource = ed.getName() + ".class"; + InputStream is = findResource(resource, systemCl, contextCl); + if (is == null) { + continue; // not found via any classloader — will try classpath scan + } + try (InputStream stream = is) { + byte[] bytes = readAllBytes(stream); + long crc = CRC64.classId(bytes); + if (crc != ed.getId()) { + // CRC64 mismatch — classloader returned different bytes than what was instrumented. + // Fall through to classpath scan. + log.debug( + "CRC64 mismatch for {} via classloader (expected {}, got {}); will try classpath scan", + ed.getName(), + Long.toHexString(ed.getId()), + Long.toHexString(crc)); + continue; + } + ClassProbeMapping mapping = + ClassProbeMappingBuilder.build( + ed.getId(), ed.getName(), ed.getProbes().length, bytes); + cache.put(ed.getId(), mapping); + needed.remove(ed.getId()); + } catch (Exception e) { + log.debug("Failed to resolve class {} via classloader", ed.getName(), e); + } + } + } + + /** + * Tries to find a class resource using the given classloaders, returning the first non-null + * InputStream. Returns null if no classloader can find the resource. + */ + private static InputStream findResource( + String resource, ClassLoader... classLoaders) { + for (ClassLoader cl : classLoaders) { + if (cl == null) { + continue; + } + InputStream is = cl.getResourceAsStream(resource); + if (is != null) { + return is; + } + } + return null; + } + + /** + * Falls back to scanning classpath jars/directories for classes that couldn't be resolved via the + * classloader. + */ + private void resolveViaClasspathScan( + Map needed, List classpathEntries) { + for (File entry : classpathEntries) { + if (needed.isEmpty()) { + break; + } + if (!entry.exists()) { + continue; + } + + try { + if (entry.isDirectory()) { + scanDirectory(entry, needed); + } else if (entry.getName().endsWith(".jar") || entry.getName().endsWith(".zip")) { + scanJar(entry, needed); + } + } catch (IOException e) { + log.debug("Failed to scan classpath entry for cache building: {}", entry, e); + } + } + } + + private void scanDirectory(File dir, Map needed) throws IOException { + File[] files = dir.listFiles(); + if (files == null) { + return; + } + for (File f : files) { + if (needed.isEmpty()) { + return; + } + if (f.isDirectory()) { + scanDirectory(f, needed); + } else if (f.getName().endsWith(".class")) { + // Use try-with-resources to avoid leaking the FileInputStream + try (FileInputStream fis = new FileInputStream(f)) { + byte[] bytes = readAllBytes(fis); + tryBuildMapping(bytes, needed); + } + } + } + } + + private void scanJar(File jarFile, Map needed) throws IOException { + try (ZipInputStream zis = new ZipInputStream(new FileInputStream(jarFile))) { + ZipEntry entry; + while ((entry = zis.getNextEntry()) != null && !needed.isEmpty()) { + if (entry.getName().endsWith(".class") && !entry.isDirectory()) { + // Do NOT close the ZipInputStream here -- readAllBytes reads without closing + byte[] bytes = readAllBytes(zis); + tryBuildMapping(bytes, needed); + } + } + } + } + + private void tryBuildMapping(byte[] classBytes, Map needed) { + long crc = CRC64.classId(classBytes); + ExecutionData ed = needed.get(crc); + if (ed == null) { + return; // this class isn't one we're looking for + } + + try { + ClassProbeMapping mapping = + ClassProbeMappingBuilder.build( + ed.getId(), ed.getName(), ed.getProbes().length, classBytes); + cache.put(ed.getId(), mapping); + needed.remove(crc); + } catch (Exception e) { + log.debug("Failed to build probe mapping for class {}", ed.getName(), e); + } + } + + /** + * Reads all bytes from an input stream WITHOUT closing it (important for ZipInputStream where + * closing the stream would close the zip). + */ + private static byte[] readAllBytes(InputStream is) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buf = new byte[4096]; + int r; + while ((r = is.read(buf)) != -1) { + out.write(buf, 0, r); + } + return out.toByteArray(); + } +} diff --git a/dd-java-agent/agent-code-coverage/src/test/java/datadog/trace/codecoverage/CoverageBinaryEncoderTest.java b/dd-java-agent/agent-code-coverage/src/test/java/datadog/trace/codecoverage/CoverageBinaryEncoderTest.java new file mode 100644 index 00000000000..1b6a26f8a35 --- /dev/null +++ b/dd-java-agent/agent-code-coverage/src/test/java/datadog/trace/codecoverage/CoverageBinaryEncoderTest.java @@ -0,0 +1,515 @@ +package datadog.trace.codecoverage; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import datadog.trace.coverage.CoverageKey; +import datadog.trace.coverage.LinesCoverage; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class CoverageBinaryEncoderTest { + + // --- uvarint encoding --- + + @Test + void uvarintZero() throws IOException { + assertUvarint(0, new byte[] {0x00}); + } + + @Test + void uvarintSingleByte() throws IOException { + assertUvarint(1, new byte[] {0x01}); + assertUvarint(0x7F, new byte[] {0x7F}); + } + + @Test + void uvarintTwoBytes() throws IOException { + // 128 = 0x80 → low 7 bits = 0x00 with continuation, then 0x01 + assertUvarint(128, new byte[] {(byte) 0x80, 0x01}); + // 16383 = 0x3FFF → 0xFF, 0x7F + assertUvarint(16383, new byte[] {(byte) 0xFF, 0x7F}); + } + + @Test + void uvarintThreeBytes() throws IOException { + // 16384 = 0x4000 → 0x80, 0x80, 0x01 + assertUvarint(16384, new byte[] {(byte) 0x80, (byte) 0x80, 0x01}); + } + + @Test + void uvarintLargeValue() throws IOException { + // 300 = 0x12C → low 7: 0x2C | 0x80 = 0xAC, remaining 2 → 0x02 + assertUvarint(300, new byte[] {(byte) 0xAC, 0x02}); + } + + // --- Empty coverage map --- + + @Test + void emptyMapProducesHeaderOnly() throws IOException { + Map coverage = new LinkedHashMap<>(); + byte[] result = encode(coverage); + // version=1, num_extra_fields=1, num_records=0 + assertArrayEquals(new byte[] {0x01, 0x01, 0x00}, result); + } + + // --- Single record with empty BitSets --- + + @Test + void singleRecordEmptyLines() throws IOException { + Map coverage = new LinkedHashMap<>(); + coverage.put(new CoverageKey("A.java", "A"), new LinesCoverage()); + + byte[] result = encode(coverage); + ByteArrayOutputStream expected = new ByteArrayOutputStream(); + expected.write(0x01); // version + expected.write(0x01); // num_extra_fields + expected.write(0x01); // num_records = 1 + writeExpectedString("A.java", expected); + writeExpectedString("A", expected); + expected.write(0x00); // bitvec_byte_count = 0, no bit vector data + + assertArrayEquals(expected.toByteArray(), result); + } + + // --- Bit vector encoding --- + + @Test + void singleLineSet() throws IOException { + // Line 1 only: byte_count = (1>>3)+1 = 1, exec byte 0 = 0x02, cov byte 0 = 0x02 + Map coverage = new LinkedHashMap<>(); + LinesCoverage lc = new LinesCoverage(); + lc.executableLines.set(1); + lc.coveredLines.set(1); + coverage.put(new CoverageKey("X.java", "X"), lc); + + byte[] result = encode(coverage); + ByteArrayOutputStream expected = new ByteArrayOutputStream(); + expected.write(0x01); // version + expected.write(0x01); // num_extra_fields + expected.write(0x01); // num_records + writeExpectedString("X.java", expected); + writeExpectedString("X", expected); + expected.write(0x01); // bitvec_byte_count = 1 + expected.write(0x02); // executable: line 1 → bit 1 of byte 0 + expected.write(0x02); // covered: line 1 → bit 1 of byte 0 + + assertArrayEquals(expected.toByteArray(), result); + } + + @Test + void linesSpanMultipleBytes() throws IOException { + // Lines {1, 8}: max_line=8, byte_count=(8>>3)+1=2 + // Line 1: byte 0, bit 1 → 0x02 + // Line 8: byte 1, bit 0 → 0x01 + Map coverage = new LinkedHashMap<>(); + LinesCoverage lc = new LinesCoverage(); + lc.executableLines.set(1); + lc.executableLines.set(8); + lc.coveredLines.set(8); + coverage.put(new CoverageKey("F.java", "F"), lc); + + byte[] result = encode(coverage); + ByteArrayOutputStream expected = new ByteArrayOutputStream(); + expected.write(0x01); + expected.write(0x01); + expected.write(0x01); + writeExpectedString("F.java", expected); + writeExpectedString("F", expected); + expected.write(0x02); // bitvec_byte_count = 2 + expected.write(0x02); // exec byte 0: line 1 + expected.write(0x01); // exec byte 1: line 8 + expected.write(0x00); // cov byte 0: no lines + expected.write(0x01); // cov byte 1: line 8 + + assertArrayEquals(expected.toByteArray(), result); + } + + @Test + void coveredLinesBitVectorPaddedWithZeros() throws IOException { + // executable has line 15 (byte 1), covered has only line 1 (byte 0) + // Both bit vectors must be 2 bytes (covered padded to match) + Map coverage = new LinkedHashMap<>(); + LinesCoverage lc = new LinesCoverage(); + lc.executableLines.set(1); + lc.executableLines.set(15); + lc.coveredLines.set(1); + coverage.put(new CoverageKey("P.java", "P"), lc); + + byte[] result = encode(coverage); + ByteArrayOutputStream expected = new ByteArrayOutputStream(); + expected.write(0x01); + expected.write(0x01); + expected.write(0x01); + writeExpectedString("P.java", expected); + writeExpectedString("P", expected); + expected.write(0x02); // bitvec_byte_count = 2 (max line 15: (15>>3)+1=2) + expected.write(0x02); // exec byte 0: line 1 + expected.write((byte) 0x80); // exec byte 1: line 15 → bit 7 + expected.write(0x02); // cov byte 0: line 1 + expected.write(0x00); // cov byte 1: padding + + assertArrayEquals(expected.toByteArray(), result); + } + + @Test + void executableLinesBitVectorPaddedWhenCoveredHasHigherLine() throws IOException { + // covered has line 10 (higher than executable's max of 3) + // This violates the spec constraint but encoder should still handle it + Map coverage = new LinkedHashMap<>(); + LinesCoverage lc = new LinesCoverage(); + lc.executableLines.set(3); + lc.coveredLines.set(10); + coverage.put(new CoverageKey("Q.java", "Q"), lc); + + byte[] result = encode(coverage); + ByteArrayOutputStream expected = new ByteArrayOutputStream(); + expected.write(0x01); + expected.write(0x01); + expected.write(0x01); + writeExpectedString("Q.java", expected); + writeExpectedString("Q", expected); + expected.write(0x02); // bitvec_byte_count = (10>>3)+1 = 2 + expected.write(0x08); // exec byte 0: line 3 → bit 3 + expected.write(0x00); // exec byte 1: padding + expected.write(0x00); // cov byte 0: no lines in lower byte + expected.write(0x04); // cov byte 1: line 10 → byte 1, bit 2 + + assertArrayEquals(expected.toByteArray(), result); + } + + // --- Multiple records --- + + @Test + void multipleRecords() throws IOException { + Map coverage = new LinkedHashMap<>(); + + LinesCoverage lc1 = new LinesCoverage(); + lc1.executableLines.set(1); + lc1.coveredLines.set(1); + coverage.put(new CoverageKey("A.java", "A"), lc1); + + LinesCoverage lc2 = new LinesCoverage(); + lc2.executableLines.set(2); + coverage.put(new CoverageKey("B.java", "B"), lc2); + + byte[] result = encode(coverage); + ByteArrayOutputStream expected = new ByteArrayOutputStream(); + expected.write(0x01); // version + expected.write(0x01); // num_extra_fields + expected.write(0x02); // num_records = 2 + + // Record 1 + writeExpectedString("A.java", expected); + writeExpectedString("A", expected); + expected.write(0x01); // bitvec_byte_count = 1 + expected.write(0x02); // exec: line 1 + expected.write(0x02); // cov: line 1 + + // Record 2 + writeExpectedString("B.java", expected); + writeExpectedString("B", expected); + expected.write(0x01); // bitvec_byte_count = 1 + expected.write(0x04); // exec: line 2 + expected.write(0x00); // cov: none + + assertArrayEquals(expected.toByteArray(), result); + } + + // --- String encoding --- + + @Test + void emptyStringEncoding() throws IOException { + Map coverage = new LinkedHashMap<>(); + coverage.put(new CoverageKey("", ""), new LinesCoverage()); + + byte[] result = encode(coverage); + ByteArrayOutputStream expected = new ByteArrayOutputStream(); + expected.write(0x01); // version + expected.write(0x01); // num_extra_fields + expected.write(0x01); // num_records + expected.write(0x00); // file_name: length 0 + expected.write(0x00); // extra_fields[0]: length 0 + expected.write(0x00); // bitvec_byte_count = 0 + + assertArrayEquals(expected.toByteArray(), result); + } + + @Test + void utf8MultiByteStringEncoding() throws IOException { + // UTF-8 multi-byte: "Ñ" is 2 bytes (0xC3 0x91) + Map coverage = new LinkedHashMap<>(); + coverage.put(new CoverageKey("Ñ.java", "Ñ"), new LinesCoverage()); + + byte[] result = encode(coverage); + byte[] fileName = "Ñ.java".getBytes(StandardCharsets.UTF_8); + byte[] className = "Ñ".getBytes(StandardCharsets.UTF_8); + + ByteArrayOutputStream expected = new ByteArrayOutputStream(); + expected.write(0x01); + expected.write(0x01); + expected.write(0x01); + // file_name length is byte count, not char count + writeExpectedUvarint(fileName.length, expected); + expected.write(fileName); + writeExpectedUvarint(className.length, expected); + expected.write(className); + expected.write(0x00); // bitvec_byte_count + + assertArrayEquals(expected.toByteArray(), result); + // Verify byte length != char length + assertEquals(7, fileName.length); // "Ñ" is 2 bytes + ".java" is 5 bytes + } + + // --- Spec example --- + + @Test + void specExampleTwoRecords() throws IOException { + Map coverage = new LinkedHashMap<>(); + + // Record 1: com/example/Foo.java, com.example.Foo, exec={1,2,3,5,8}, cov={1,3,5} + LinesCoverage lc1 = new LinesCoverage(); + for (int line : new int[] {1, 2, 3, 5, 8}) { + lc1.executableLines.set(line); + } + for (int line : new int[] {1, 3, 5}) { + lc1.coveredLines.set(line); + } + coverage.put(new CoverageKey("com/example/Foo.java", "com.example.Foo"), lc1); + + // Record 2: com/example/Bar.java, com.example.Bar, exec={2,4,6}, cov={4} + LinesCoverage lc2 = new LinesCoverage(); + for (int line : new int[] {2, 4, 6}) { + lc2.executableLines.set(line); + } + lc2.coveredLines.set(4); + coverage.put(new CoverageKey("com/example/Bar.java", "com.example.Bar"), lc2); + + byte[] result = encode(coverage); + + // Expected byte sequence from the spec + byte[] expected = { + 0x01, 0x01, 0x02, // header: version=1, extra_fields=1, records=2 + 0x14, // file_name length = 20 + 0x63, 0x6F, 0x6D, 0x2F, 0x65, 0x78, 0x61, 0x6D, // "com/exam" + 0x70, 0x6C, 0x65, 0x2F, 0x46, 0x6F, 0x6F, 0x2E, // "ple/Foo." + 0x6A, 0x61, 0x76, 0x61, // "java" + 0x0F, // extra_fields[0] length = 15 + 0x63, 0x6F, 0x6D, 0x2E, 0x65, 0x78, 0x61, 0x6D, // "com.exam" + 0x70, 0x6C, 0x65, 0x2E, 0x46, 0x6F, 0x6F, // "ple.Foo" + 0x02, // bitvec_byte_count = 2 + 0x2E, 0x01, // executable_lines + 0x2A, 0x00, // covered_lines + 0x14, // file_name length = 20 + 0x63, 0x6F, 0x6D, 0x2F, 0x65, 0x78, 0x61, 0x6D, // "com/exam" + 0x70, 0x6C, 0x65, 0x2F, 0x42, 0x61, 0x72, 0x2E, // "ple/Bar." + 0x6A, 0x61, 0x76, 0x61, // "java" + 0x0F, // extra_fields[0] length = 15 + 0x63, 0x6F, 0x6D, 0x2E, 0x65, 0x78, 0x61, 0x6D, // "com.exam" + 0x70, 0x6C, 0x65, 0x2E, 0x42, 0x61, 0x72, // "ple.Bar" + 0x01, // bitvec_byte_count = 1 + 0x54, // executable_lines + 0x10 // covered_lines + }; + + assertArrayEquals(expected, result); + } + + // --- Edge cases --- + + @Test + void highLineNumber() throws IOException { + // Line 1000: byte index = 1000>>3 = 125, bit = 1000&7 = 0 + // byte_count = (1000>>3)+1 = 126 + Map coverage = new LinkedHashMap<>(); + LinesCoverage lc = new LinesCoverage(); + lc.executableLines.set(1000); + lc.coveredLines.set(1000); + coverage.put(new CoverageKey("H.java", "H"), lc); + + byte[] result = encode(coverage); + + // Parse manually: skip header and strings, check bitvec + int offset = 0; + assertEquals(0x01, result[offset++] & 0xFF); // version + assertEquals(0x01, result[offset++] & 0xFF); // num_extra_fields + assertEquals(0x01, result[offset++] & 0xFF); // num_records + + // Skip file_name "H.java" (length 6) + assertEquals(0x06, result[offset++] & 0xFF); + offset += 6; + + // Skip extra_field "H" (length 1) + assertEquals(0x01, result[offset++] & 0xFF); + offset += 1; + + // bitvec_byte_count = 126 → varint encoding: (126 & 0x7F) = 0x7E, fits in 1 byte + assertEquals(126, result[offset++] & 0xFF); + + // executable_lines: 126 bytes, only byte 125 has bit 0 set + for (int i = 0; i < 126; i++) { + int expectedByte = (i == 125) ? 0x01 : 0x00; + assertEquals(expectedByte, result[offset + i] & 0xFF, "exec byte " + i); + } + offset += 126; + + // covered_lines: same pattern + for (int i = 0; i < 126; i++) { + int expectedByte = (i == 125) ? 0x01 : 0x00; + assertEquals(expectedByte, result[offset + i] & 0xFF, "cov byte " + i); + } + } + + @Test + void line7SetsHighBitOfByte0() throws IOException { + // Line 7: byte 0, bit 7 → 0x80 + Map coverage = new LinkedHashMap<>(); + LinesCoverage lc = new LinesCoverage(); + lc.executableLines.set(7); + coverage.put(new CoverageKey("S.java", "S"), lc); + + byte[] result = encode(coverage); + ByteArrayOutputStream expected = new ByteArrayOutputStream(); + expected.write(0x01); + expected.write(0x01); + expected.write(0x01); + writeExpectedString("S.java", expected); + writeExpectedString("S", expected); + expected.write(0x01); // bitvec_byte_count = 1 + expected.write((byte) 0x80); // exec: line 7 → bit 7 + expected.write(0x00); // cov: none + + assertArrayEquals(expected.toByteArray(), result); + } + + @Test + void allLinesInOneByte() throws IOException { + // Lines {1,2,3,4,5,6,7}: all in byte 0 + Map coverage = new LinkedHashMap<>(); + LinesCoverage lc = new LinesCoverage(); + for (int i = 1; i <= 7; i++) { + lc.executableLines.set(i); + } + lc.coveredLines.set(4); + coverage.put(new CoverageKey("Z.java", "Z"), lc); + + byte[] result = encode(coverage); + ByteArrayOutputStream expected = new ByteArrayOutputStream(); + expected.write(0x01); + expected.write(0x01); + expected.write(0x01); + writeExpectedString("Z.java", expected); + writeExpectedString("Z", expected); + expected.write(0x01); // bitvec_byte_count = 1 + // exec: bits 1-7 set → 0xFE + expected.write((byte) 0xFE); + // cov: bit 4 → 0x10 + expected.write(0x10); + + assertArrayEquals(expected.toByteArray(), result); + } + + @Test + void onlyExecutableLinesNoCoverage() throws IOException { + Map coverage = new LinkedHashMap<>(); + LinesCoverage lc = new LinesCoverage(); + lc.executableLines.set(1); + lc.executableLines.set(5); + coverage.put(new CoverageKey("N.java", "N"), lc); + + byte[] result = encode(coverage); + ByteArrayOutputStream expected = new ByteArrayOutputStream(); + expected.write(0x01); + expected.write(0x01); + expected.write(0x01); + writeExpectedString("N.java", expected); + writeExpectedString("N", expected); + expected.write(0x01); // bitvec_byte_count = 1 + expected.write(0x22); // exec: line 1 (0x02) | line 5 (0x20) = 0x22 + expected.write(0x00); // cov: empty + + assertArrayEquals(expected.toByteArray(), result); + } + + @Test + void stringLengthRequiresMultiByteUvarint() throws IOException { + // Create a string longer than 127 bytes so its length needs 2 uvarint bytes + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 130; i++) { + sb.append('a'); + } + String longName = sb.toString(); + + Map coverage = new LinkedHashMap<>(); + coverage.put(new CoverageKey(longName, "C"), new LinesCoverage()); + + byte[] result = encode(coverage); + + // Check the uvarint encoding of 130: (130 & 0x7F) | 0x80 = 0x82, 130 >> 7 = 1 → 0x01 + int offset = 3; // skip version + num_extra_fields + num_records + assertEquals((byte) 0x82, result[offset]); // low 7 bits of 130 with continuation + assertEquals((byte) 0x01, result[offset + 1]); // remaining bits + offset += 2; + // Verify string data + for (int i = 0; i < 130; i++) { + assertEquals((byte) 'a', result[offset + i]); + } + } + + @Test + void outputSizeMatchesExpectedForSpecExample() throws IOException { + // The spec says total message size is 85 bytes + Map coverage = new LinkedHashMap<>(); + + LinesCoverage lc1 = new LinesCoverage(); + for (int line : new int[] {1, 2, 3, 5, 8}) { + lc1.executableLines.set(line); + } + for (int line : new int[] {1, 3, 5}) { + lc1.coveredLines.set(line); + } + coverage.put(new CoverageKey("com/example/Foo.java", "com.example.Foo"), lc1); + + LinesCoverage lc2 = new LinesCoverage(); + for (int line : new int[] {2, 4, 6}) { + lc2.executableLines.set(line); + } + lc2.coveredLines.set(4); + coverage.put(new CoverageKey("com/example/Bar.java", "com.example.Bar"), lc2); + + byte[] result = encode(coverage); + assertEquals(85, result.length); + } + + // --- Helpers --- + + private static byte[] encode(Map coverage) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + CoverageBinaryEncoder.encode(coverage, out); + return out.toByteArray(); + } + + private static void assertUvarint(int value, byte[] expected) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + CoverageBinaryEncoder.writeUvarint(value, out); + assertArrayEquals(expected, out.toByteArray(), "uvarint(" + value + ")"); + } + + private static void writeExpectedUvarint(int value, ByteArrayOutputStream out) { + while (value >= 0x80) { + out.write((value & 0x7F) | 0x80); + value >>>= 7; + } + out.write(value); + } + + private static void writeExpectedString(String s, ByteArrayOutputStream out) throws IOException { + byte[] bytes = s.getBytes(StandardCharsets.UTF_8); + writeExpectedUvarint(bytes.length, out); + out.write(bytes); + } +} diff --git a/dd-java-agent/build.gradle b/dd-java-agent/build.gradle index a34b233cea9..8712defde15 100644 --- a/dd-java-agent/build.gradle +++ b/dd-java-agent/build.gradle @@ -229,6 +229,7 @@ includeSubprojShadowJar(project(':dd-java-agent:agent-aiguard'), 'aiguard', incl includeSubprojShadowJar(project(':dd-java-agent:agent-iast'), 'iast', includedJarFileTree) includeSubprojShadowJar(project(':dd-java-agent:agent-debugger'), 'debugger', includedJarFileTree) includeSubprojShadowJar(project(':dd-java-agent:agent-ci-visibility'), 'ci-visibility', includedJarFileTree) +includeSubprojShadowJar(project(':dd-java-agent:agent-code-coverage'), 'code-coverage', includedJarFileTree) includeSubprojShadowJar(project(':dd-java-agent:agent-llmobs'), 'llm-obs', includedJarFileTree) includeSubprojShadowJar(project(':dd-java-agent:agent-logs-intake'), 'logs-intake', includedJarFileTree) includeSubprojShadowJar(project(':dd-java-agent:cws-tls'), 'cws-tls', includedJarFileTree) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/CodeCoverageConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/CodeCoverageConfig.java new file mode 100644 index 00000000000..cee50521510 --- /dev/null +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/CodeCoverageConfig.java @@ -0,0 +1,14 @@ +package datadog.trace.api.config; + +/** Constant with names of configuration options for production code coverage. */ +public final class CodeCoverageConfig { + + public static final String CODE_COVERAGE_ENABLED = "code.coverage.enabled"; + public static final String CODE_COVERAGE_INCLUDES = "code.coverage.includes"; + public static final String CODE_COVERAGE_EXCLUDES = "code.coverage.excludes"; + public static final String CODE_COVERAGE_REPORT_INTERVAL_SECONDS = + "code.coverage.report.interval.seconds"; + public static final String CODE_COVERAGE_CLASSPATH = "code.coverage.classpath"; + + private CodeCoverageConfig() {} +} diff --git a/gradle.properties b/gradle.properties index 48d5ceb5b49..c51c2eadd4d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ org.gradle.parallel=true org.gradle.caching=true -org.gradle.jvmargs=-XX:MaxMetaspaceSize=1g +org.gradle.jvmargs=-Xmx4g -XX:MaxMetaspaceSize=1g # Toggle on to get more details during IJ sync #org.gradle.logging.level=info diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index fcaac7a9b55..6375241f53f 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -294,6 +294,10 @@ import static datadog.trace.api.config.CiVisibilityConfig.TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES; import static datadog.trace.api.config.CiVisibilityConfig.TEST_MANAGEMENT_ENABLED; import static datadog.trace.api.config.CiVisibilityConfig.TEST_SESSION_NAME; +import static datadog.trace.api.config.CodeCoverageConfig.CODE_COVERAGE_CLASSPATH; +import static datadog.trace.api.config.CodeCoverageConfig.CODE_COVERAGE_EXCLUDES; +import static datadog.trace.api.config.CodeCoverageConfig.CODE_COVERAGE_INCLUDES; +import static datadog.trace.api.config.CodeCoverageConfig.CODE_COVERAGE_REPORT_INTERVAL_SECONDS; import static datadog.trace.api.config.CrashTrackingConfig.CRASH_TRACKING_AGENTLESS; import static datadog.trace.api.config.CrashTrackingConfig.CRASH_TRACKING_AGENTLESS_DEFAULT; import static datadog.trace.api.config.CrashTrackingConfig.CRASH_TRACKING_ERRORS_INTAKE_ENABLED; @@ -1248,6 +1252,11 @@ public static String getHostName() { private final boolean cwsEnabled; private final int cwsTlsRefresh; + private final String[] codeCoverageIncludes; + private final String[] codeCoverageExcludes; + private final int codeCoverageReportIntervalSeconds; + private final String codeCoverageClasspath; + private final boolean dataJobsOpenLineageEnabled; private final boolean dataJobsOpenLineageTimeoutEnabled; private final boolean dataJobsParseSparkPlanEnabled; @@ -2816,6 +2825,20 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) cwsEnabled = configProvider.getBoolean(CWS_ENABLED, DEFAULT_CWS_ENABLED); cwsTlsRefresh = configProvider.getInteger(CWS_TLS_REFRESH, DEFAULT_CWS_TLS_REFRESH); + { + List includesList = configProvider.getList(CODE_COVERAGE_INCLUDES); + codeCoverageIncludes = + includesList == null || includesList.isEmpty() + ? new String[] {"*"} + : includesList.toArray(new String[0]); + List excludesList = configProvider.getList(CODE_COVERAGE_EXCLUDES); + codeCoverageExcludes = + excludesList == null ? new String[0] : excludesList.toArray(new String[0]); + } + codeCoverageReportIntervalSeconds = + configProvider.getInteger(CODE_COVERAGE_REPORT_INTERVAL_SECONDS, 900); + codeCoverageClasspath = configProvider.getString(CODE_COVERAGE_CLASSPATH); + dataJobsOpenLineageEnabled = configProvider.getBoolean( DATA_JOBS_OPENLINEAGE_ENABLED, DEFAULT_DATA_JOBS_OPENLINEAGE_ENABLED); @@ -4677,6 +4700,22 @@ public boolean isCwsEnabled() { return cwsEnabled; } + public String[] getCodeCoverageIncludes() { + return codeCoverageIncludes; + } + + public String[] getCodeCoverageExcludes() { + return codeCoverageExcludes; + } + + public int getCodeCoverageReportIntervalSeconds() { + return codeCoverageReportIntervalSeconds; + } + + public String getCodeCoverageClasspath() { + return codeCoverageClasspath; + } + public int getCwsTlsRefresh() { return cwsTlsRefresh; } diff --git a/internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java b/internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java index 00dd43f7197..d679b2812c4 100644 --- a/internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java +++ b/internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java @@ -28,6 +28,7 @@ import static datadog.trace.api.config.AppSecConfig.APPSEC_ENABLED; import static datadog.trace.api.config.AppSecConfig.APPSEC_RASP_ENABLED; import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_ENABLED; +import static datadog.trace.api.config.CodeCoverageConfig.CODE_COVERAGE_ENABLED; import static datadog.trace.api.config.GeneralConfig.AGENTLESS_LOG_SUBMISSION_ENABLED; import static datadog.trace.api.config.GeneralConfig.APP_LOGS_COLLECTION_ENABLED; import static datadog.trace.api.config.GeneralConfig.DATA_JOBS_ENABLED; @@ -215,6 +216,8 @@ public class InstrumenterConfig { private final boolean appLogsCollectionEnabled; private final boolean legacyContextManagerEnabled; + private final boolean codeCoverageEnabled; + static { // Bind telemetry collector to config module before initializing ConfigProvider OtelEnvMetricCollectorProvider.register(OtelEnvMetricCollectorImpl.getInstance()); @@ -367,6 +370,8 @@ private InstrumenterConfig() { configProvider.getBoolean(APP_LOGS_COLLECTION_ENABLED, DEFAULT_APP_LOGS_COLLECTION_ENABLED); legacyContextManagerEnabled = configProvider.getBoolean(LEGACY_CONTEXT_MANAGER_ENABLED, true); + + codeCoverageEnabled = configProvider.getBoolean(CODE_COVERAGE_ENABLED, false); } public boolean isCodeOriginEnabled() { @@ -690,6 +695,10 @@ public boolean isLegacyContextManagerEnabled() { return legacyContextManagerEnabled; } + public boolean isCodeCoverageEnabled() { + return codeCoverageEnabled; + } + // This has to be placed after all other static fields to give them a chance to initialize private static final InstrumenterConfig INSTANCE = new InstrumenterConfig( @@ -811,6 +820,8 @@ public String toString() { + apiSecurityEndpointCollectionEnabled + ", legacyContextManagerEnabled=" + legacyContextManagerEnabled + + ", codeCoverageEnabled=" + + codeCoverageEnabled + '}'; } } diff --git a/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java b/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java index 4aa40e06522..02364ec4d41 100644 --- a/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java +++ b/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java @@ -63,7 +63,9 @@ public enum AgentThread { LLMOBS_EVALS_PROCESSOR("dd-llmobs-evals-processor"), - FEATURE_FLAG_EXPOSURE_PROCESSOR("dd-ffe-exposure-processor"); + FEATURE_FLAG_EXPOSURE_PROCESSOR("dd-ffe-exposure-processor"), + + CODE_COVERAGE("dd-code-coverage"); public final String threadName; diff --git a/settings.gradle.kts b/settings.gradle.kts index dbe66b33670..497fe614654 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -128,6 +128,11 @@ include( ":dd-java-agent:agent-ci-visibility:civisibility-instrumentation-test-fixtures", ) +// code-coverage +include( + ":dd-java-agent:agent-code-coverage", +) + // llm-observability include( ":dd-java-agent:agent-llmobs", @@ -159,6 +164,7 @@ include( ":dd-java-agent:testing", ":utils:config-utils", ":utils:container-utils", + ":utils:coverage-utils", ":utils:filesystem-utils", ":utils:flare-utils", ":utils:logging-utils", diff --git a/utils/coverage-utils/build.gradle.kts b/utils/coverage-utils/build.gradle.kts new file mode 100644 index 00000000000..900b9da3aef --- /dev/null +++ b/utils/coverage-utils/build.gradle.kts @@ -0,0 +1,13 @@ +plugins { + `java-library` +} + +apply(from = "$rootDir/gradle/java.gradle") + +dependencies { + // For CoverageReportUploader + implementation(project(":communication")) + implementation(project(":internal-api")) + + testImplementation(project(":dd-java-agent:testing")) +} diff --git a/utils/coverage-utils/src/main/java/datadog/trace/coverage/CoverageKey.java b/utils/coverage-utils/src/main/java/datadog/trace/coverage/CoverageKey.java new file mode 100644 index 00000000000..560361f5960 --- /dev/null +++ b/utils/coverage-utils/src/main/java/datadog/trace/coverage/CoverageKey.java @@ -0,0 +1,27 @@ +package datadog.trace.coverage; + +import java.util.Objects; + +public class CoverageKey { + public final String sourceFile; + public final String className; + + public CoverageKey(String sourceFile, String className) { + this.sourceFile = sourceFile; + this.className = className; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + CoverageKey that = (CoverageKey) o; + return Objects.equals(sourceFile, that.sourceFile) && Objects.equals(className, that.className); + } + + @Override + public int hashCode() { + return Objects.hash(sourceFile, className); + } +} diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/report/CoverageReportUploader.java b/utils/coverage-utils/src/main/java/datadog/trace/coverage/CoverageReportUploader.java similarity index 67% rename from dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/report/CoverageReportUploader.java rename to utils/coverage-utils/src/main/java/datadog/trace/coverage/CoverageReportUploader.java index 0d0ffef37c9..9a60e477bf1 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/report/CoverageReportUploader.java +++ b/utils/coverage-utils/src/main/java/datadog/trace/coverage/CoverageReportUploader.java @@ -1,4 +1,4 @@ -package datadog.trace.civisibility.coverage.report; +package datadog.trace.coverage; import static datadog.communication.http.OkHttpUtils.jsonRequestBodyOf; @@ -7,10 +7,6 @@ import com.squareup.moshi.Types; import datadog.communication.BackendApi; import datadog.communication.http.OkHttpUtils; -import datadog.trace.api.civisibility.telemetry.CiVisibilityCountMetric; -import datadog.trace.api.civisibility.telemetry.CiVisibilityDistributionMetric; -import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector; -import datadog.trace.civisibility.communication.TelemetryListener; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.IOException; import java.io.InputStream; @@ -19,6 +15,7 @@ import java.util.HashMap; import java.util.Map; import java.util.zip.GZIPOutputStream; +import javax.annotation.Nullable; import okhttp3.MediaType; import okhttp3.MultipartBody; import okhttp3.RequestBody; @@ -27,25 +24,25 @@ public class CoverageReportUploader { private final BackendApi backendApi; - private final Map ciTags; - private final CiVisibilityMetricCollector metricCollector; - private final JsonAdapter> eventAdapter; + private final Map tags; + @Nullable private final OkHttpUtils.CustomListener requestListener; + private final JsonAdapter> eventAdapter; public CoverageReportUploader( BackendApi backendApi, - Map ciTags, - CiVisibilityMetricCollector metricCollector) { + Map tags, + @Nullable OkHttpUtils.CustomListener requestListener) { this.backendApi = backendApi; - this.ciTags = ciTags; - this.metricCollector = metricCollector; + this.tags = tags; + this.requestListener = requestListener; Moshi moshi = new Moshi.Builder().build(); - Type type = Types.newParameterizedType(Map.class, String.class, String.class); + Type type = Types.newParameterizedType(Map.class, String.class, Object.class); eventAdapter = moshi.adapter(type); } public void upload(String format, InputStream reportStream) throws IOException { - Map event = new HashMap<>(ciTags); + Map event = new HashMap<>(tags); event.put("format", format); event.put("type", "coverage_report"); String eventJson = eventAdapter.toJson(event); @@ -60,15 +57,7 @@ public void upload(String format, InputStream reportStream) throws IOException { .addFormDataPart("event", "event.json", eventBody) .build(); - OkHttpUtils.CustomListener telemetryListener = - new TelemetryListener.Builder(metricCollector) - .requestCount(CiVisibilityCountMetric.COVERAGE_UPLOAD_REQUEST) - .requestBytes(CiVisibilityDistributionMetric.COVERAGE_UPLOAD_REQUEST_BYTES) - .requestErrors(CiVisibilityCountMetric.COVERAGE_UPLOAD_REQUEST_ERRORS) - .requestDuration(CiVisibilityDistributionMetric.COVERAGE_UPLOAD_REQUEST_MS) - .build(); - - backendApi.post("cicovreprt", multipartBody, responseStream -> null, telemetryListener, false); + backendApi.post("cicovreprt", multipartBody, responseStream -> null, requestListener, false); } /** Request body that compresses a form data part */ diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/report/LcovReportWriter.java b/utils/coverage-utils/src/main/java/datadog/trace/coverage/LcovReportWriter.java similarity index 97% rename from dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/report/LcovReportWriter.java rename to utils/coverage-utils/src/main/java/datadog/trace/coverage/LcovReportWriter.java index b7fc48c8873..4146c2d42ed 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/report/LcovReportWriter.java +++ b/utils/coverage-utils/src/main/java/datadog/trace/coverage/LcovReportWriter.java @@ -1,4 +1,4 @@ -package datadog.trace.civisibility.coverage.report; +package datadog.trace.coverage; import java.io.IOException; import java.io.StringWriter; diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/report/LinesCoverage.java b/utils/coverage-utils/src/main/java/datadog/trace/coverage/LinesCoverage.java similarity index 76% rename from dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/report/LinesCoverage.java rename to utils/coverage-utils/src/main/java/datadog/trace/coverage/LinesCoverage.java index 962467439a1..b587f3a0f7e 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/report/LinesCoverage.java +++ b/utils/coverage-utils/src/main/java/datadog/trace/coverage/LinesCoverage.java @@ -1,4 +1,4 @@ -package datadog.trace.civisibility.coverage.report; +package datadog.trace.coverage; import java.util.BitSet; diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/coverage/report/CoverageReportUploaderTest.groovy b/utils/coverage-utils/src/test/groovy/datadog/trace/coverage/CoverageReportUploaderTest.groovy similarity index 93% rename from dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/coverage/report/CoverageReportUploaderTest.groovy rename to utils/coverage-utils/src/test/groovy/datadog/trace/coverage/CoverageReportUploaderTest.groovy index d1322d91c3b..0644a641e27 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/coverage/report/CoverageReportUploaderTest.groovy +++ b/utils/coverage-utils/src/test/groovy/datadog/trace/coverage/CoverageReportUploaderTest.groovy @@ -1,11 +1,10 @@ -package datadog.trace.civisibility.coverage.report +package datadog.trace.coverage import com.fasterxml.jackson.databind.ObjectMapper import datadog.communication.BackendApi import datadog.communication.IntakeApi import datadog.communication.http.HttpRetryPolicy import datadog.communication.http.OkHttpUtils -import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector import datadog.trace.api.intake.Intake import datadog.trace.test.util.MultipartRequestParser import okhttp3.HttpUrl @@ -70,8 +69,7 @@ class CoverageReportUploaderTest extends Specification { def "test upload coverage report"() { setup: def backendApi = givenIntakeApi() - def metricCollector = Stub(CiVisibilityMetricCollector) - def uploader = new CoverageReportUploader(backendApi, [(CI_TAG_KEY):CI_TAG_VALUE], metricCollector) + def uploader = new CoverageReportUploader(backendApi, [(CI_TAG_KEY):CI_TAG_VALUE], null) def report = new ByteArrayInputStream(COVERAGE_REPORT_BODY.getBytes()) expect: diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/coverage/report/LcovReportWriterTest.groovy b/utils/coverage-utils/src/test/groovy/datadog/trace/coverage/LcovReportWriterTest.groovy similarity index 98% rename from dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/coverage/report/LcovReportWriterTest.groovy rename to utils/coverage-utils/src/test/groovy/datadog/trace/coverage/LcovReportWriterTest.groovy index a1c2d3c0478..9bc2b145199 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/coverage/report/LcovReportWriterTest.groovy +++ b/utils/coverage-utils/src/test/groovy/datadog/trace/coverage/LcovReportWriterTest.groovy @@ -1,4 +1,4 @@ -package datadog.trace.civisibility.coverage.report +package datadog.trace.coverage import spock.lang.Specification