diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d61754b6290..baf71bd28b6 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -17,6 +17,7 @@ # Shared communication /communication/*.* @DataDog/apm-java +/communication/otlp-exporter/ @DataDog/apm-sdk-capabilities-java /communication/src/*/*/datadog/communication/*.* @DataDog/apm-java /communication/src/*/*/datadog/communication/ddagent/ @DataDog/apm-java /communication/src/*/*/datadog/communication/http/ @DataDog/apm-java @@ -341,6 +342,7 @@ # @DataDog/profiling-java /dd-java-agent/ddprof-lib/ @DataDog/profiling-java +/docker/Dockerfile.profcheck @DataDog/profiling-java /dd-java-agent/agent-profiling/ @DataDog/profiling-java /dd-java-agent/agent-crashtracking/ @DataDog/profiling-java /dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/jfr/ @DataDog/profiling-java diff --git a/.gitignore b/.gitignore index 54ae092deba..1daf62b7868 100644 --- a/.gitignore +++ b/.gitignore @@ -95,3 +95,4 @@ mise*.local.toml # Exclude kotlin build files .kotlin +.claude/state/ diff --git a/communication/otlp-exporter/build.gradle.kts b/communication/otlp-exporter/build.gradle.kts new file mode 100644 index 00000000000..2781a4ab1ad --- /dev/null +++ b/communication/otlp-exporter/build.gradle.kts @@ -0,0 +1,27 @@ +plugins { + `java-library` + id("dd-trace-java.module.internal-library") +} + +description = "otlp-exporter" + +// Network/IO classes are hard to unit-test without a real OTLP endpoint; they are exercised +// via integration tests and the profiling-otel smoke tests instead. +extra["excludedClassesCoverage"] = listOf( + "datadog.communication.otlp.OtlpGrpcSender", + "datadog.communication.otlp.OtlpHttpSender", + "datadog.communication.otlp.OtlpResponse", + "datadog.communication.otlp.OtlpSenderSupport" +) + +dependencies { + api(project(":dd-trace-api")) + api(project(":communication")) + api(project(":internal-api")) + implementation(project(":utils:logging-utils")) + implementation(libs.slf4j) + + testImplementation(libs.bundles.junit5) + testImplementation(libs.bundles.mockito) + testImplementation(libs.okhttp3.mockwebserver) +} diff --git a/communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpCanonicalResourceAttributes.java b/communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpCanonicalResourceAttributes.java new file mode 100644 index 00000000000..cdb57b0e9b8 --- /dev/null +++ b/communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpCanonicalResourceAttributes.java @@ -0,0 +1,37 @@ +package datadog.communication.otlp; + +import static datadog.communication.ddagent.TracerVersion.TRACER_VERSION; + +import datadog.trace.api.Config; +import java.util.function.BiConsumer; + +/** Canonical OTLP resource attributes shared by the trace and profile exports. */ +public final class OtlpCanonicalResourceAttributes { + private OtlpCanonicalResourceAttributes() {} + + /** + * Visits {@code service.name}, {@code deployment.environment.name} and {@code service.version} + * (when not empty), {@code host.name} (when reported and not empty) and the {@code + * telemetry.sdk.*} attributes, in that order. + */ + public static void visit(Config config, BiConsumer visitor) { + visitor.accept("service.name", config.getServiceName()); + String env = config.getEnv(); + if (!env.isEmpty()) { + visitor.accept("deployment.environment.name", env); + } + String version = config.getVersion(); + if (!version.isEmpty()) { + visitor.accept("service.version", version); + } + if (config.isReportHostName()) { + String hostName = config.getHostName(); + if (hostName != null && !hostName.isEmpty()) { + visitor.accept("host.name", hostName); + } + } + visitor.accept("telemetry.sdk.name", "datadog"); + visitor.accept("telemetry.sdk.version", TRACER_VERSION); + visitor.accept("telemetry.sdk.language", "java"); + } +} diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpGrpcRequestBody.java b/communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpGrpcRequestBody.java similarity index 97% rename from dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpGrpcRequestBody.java rename to communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpGrpcRequestBody.java index 24457780b1d..92694d1503a 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpGrpcRequestBody.java +++ b/communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpGrpcRequestBody.java @@ -1,4 +1,4 @@ -package datadog.trace.core.otlp.common; +package datadog.communication.otlp; import java.io.IOException; import javax.annotation.Nonnull; diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpGrpcSender.java b/communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpGrpcSender.java similarity index 94% rename from dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpGrpcSender.java rename to communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpGrpcSender.java index 73c73a7f626..06c2dafeaa9 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpGrpcSender.java +++ b/communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpGrpcSender.java @@ -1,4 +1,4 @@ -package datadog.trace.core.otlp.common; +package datadog.communication.otlp; import static datadog.communication.http.OkHttpUtils.buildHttp2Client; import static datadog.communication.http.OkHttpUtils.isPlainHttp; @@ -6,7 +6,6 @@ import datadog.communication.http.HttpRetryPolicy; import datadog.logging.RatelimitedLogger; import datadog.trace.api.config.OtlpConfig.Compression; -import datadog.trace.common.writer.RemoteApi; import java.util.Map; import java.util.concurrent.TimeUnit; import okhttp3.HttpUrl; @@ -53,7 +52,7 @@ public OtlpGrpcSender( } @Override - public RemoteApi.Response send(OtlpPayload payload) { + public OtlpResponse send(OtlpPayload payload) { return OtlpSenderSupport.send(client, retryPolicy, makeRequest(payload), RATELIMITED_LOGGER); } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpHttpRequestBody.java b/communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpHttpRequestBody.java similarity index 96% rename from dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpHttpRequestBody.java rename to communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpHttpRequestBody.java index a94d5c59f08..e1a4fd3313b 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpHttpRequestBody.java +++ b/communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpHttpRequestBody.java @@ -1,4 +1,4 @@ -package datadog.trace.core.otlp.common; +package datadog.communication.otlp; import java.io.IOException; import javax.annotation.Nonnull; diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpHttpSender.java b/communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpHttpSender.java similarity index 94% rename from dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpHttpSender.java rename to communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpHttpSender.java index 8cac29e85d5..f0e78a4b59d 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpHttpSender.java +++ b/communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpHttpSender.java @@ -1,4 +1,4 @@ -package datadog.trace.core.otlp.common; +package datadog.communication.otlp; import static datadog.communication.http.OkHttpUtils.buildHttpClient; import static datadog.communication.http.OkHttpUtils.isPlainHttp; @@ -6,7 +6,6 @@ import datadog.communication.http.HttpRetryPolicy; import datadog.logging.RatelimitedLogger; import datadog.trace.api.config.OtlpConfig.Compression; -import datadog.trace.common.writer.RemoteApi; import java.util.Map; import java.util.concurrent.TimeUnit; import okhttp3.HttpUrl; @@ -57,7 +56,7 @@ public HttpUrl url() { } @Override - public RemoteApi.Response send(OtlpPayload payload) { + public OtlpResponse send(OtlpPayload payload) { return OtlpSenderSupport.send(client, retryPolicy, makeRequest(payload), RATELIMITED_LOGGER); } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpPayload.java b/communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpPayload.java similarity index 95% rename from dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpPayload.java rename to communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpPayload.java index 87c2adf760d..2eb0945f6e7 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpPayload.java +++ b/communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpPayload.java @@ -1,4 +1,4 @@ -package datadog.trace.core.otlp.common; +package datadog.communication.otlp; import java.nio.ByteBuffer; diff --git a/communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpResponse.java b/communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpResponse.java new file mode 100644 index 00000000000..71eb35b3cde --- /dev/null +++ b/communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpResponse.java @@ -0,0 +1,78 @@ +package datadog.communication.otlp; + +import java.util.Optional; +import java.util.OptionalInt; + +/** + * Encapsulates the result of an OTLP export attempt. + * + *

If communication fails or times out, the response will NOT be successful and will lack a + * status code, but will have an exception. + * + *

If communication succeeds, the response will have a status code and will be marked as success + * or failure in accordance with the code. + * + *

NOTE: A successful communication may still contain an exception if there was a problem parsing + * the response. + */ +public final class OtlpResponse { + /** Factory method for a successful request with a trivial response body */ + public static OtlpResponse success(final int status) { + return new OtlpResponse(true, status, null, null); + } + + /** Factory method for a successful request with a response body */ + public static OtlpResponse success(final int status, String response) { + return new OtlpResponse(true, status, null, response); + } + + /** Factory method for a successful request with a malformed response body */ + public static OtlpResponse successWithParseError(final int status, final Throwable exception) { + return new OtlpResponse(true, status, exception, null); + } + + /** Factory method for a request that received an error status in response */ + public static OtlpResponse failed(final int status) { + return new OtlpResponse(false, status, null, null); + } + + /** Factory method for a request that received an error status and a response body */ + public static OtlpResponse failed(final int status, String response) { + return new OtlpResponse(false, status, null, response); + } + + /** Factory method for a failed communication attempt */ + public static OtlpResponse failed(final Throwable exception) { + return new OtlpResponse(false, null, exception, null); + } + + private final boolean success; + private final Integer status; + private final Throwable exception; + private final String response; + + private OtlpResponse( + final boolean success, final Integer status, final Throwable exception, String response) { + this.success = success; + this.status = status; + this.exception = exception; + this.response = response; + } + + public boolean success() { + return success; + } + + public OptionalInt status() { + return status == null ? OptionalInt.empty() : OptionalInt.of(status); + } + + public Optional exception() { + return Optional.ofNullable(exception); + } + + /** The raw response body, or empty for failed communication attempts (no body received) */ + public Optional response() { + return Optional.ofNullable(response); + } +} diff --git a/communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpSender.java b/communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpSender.java new file mode 100644 index 00000000000..7fcb1520012 --- /dev/null +++ b/communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpSender.java @@ -0,0 +1,8 @@ +package datadog.communication.otlp; + +/** Sends chunks of OTLP data. */ +public interface OtlpSender { + OtlpResponse send(OtlpPayload payload); + + void shutdown(); +} diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpSenderSupport.java b/communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpSenderSupport.java similarity index 82% rename from dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpSenderSupport.java rename to communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpSenderSupport.java index 3e27cc9e020..5b86b683c2e 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpSenderSupport.java +++ b/communication/otlp-exporter/src/main/java/datadog/communication/otlp/OtlpSenderSupport.java @@ -1,12 +1,11 @@ -package datadog.trace.core.otlp.common; +package datadog.communication.otlp; import static datadog.communication.http.OkHttpUtils.sendWithRetries; -import static datadog.trace.common.writer.RemoteApi.Response.failed; -import static datadog.trace.common.writer.RemoteApi.Response.success; +import static datadog.communication.otlp.OtlpResponse.failed; +import static datadog.communication.otlp.OtlpResponse.success; import datadog.communication.http.HttpRetryPolicy; import datadog.logging.RatelimitedLogger; -import datadog.trace.common.writer.RemoteApi; import java.io.IOException; import okhttp3.OkHttpClient; @@ -15,7 +14,7 @@ final class OtlpSenderSupport { private OtlpSenderSupport() {} /** Executes the given request with retries, logging failures via the rate-limited logger. */ - static RemoteApi.Response send( + static OtlpResponse send( OkHttpClient client, HttpRetryPolicy.Factory retryPolicy, okhttp3.Request request, diff --git a/communication/otlp-exporter/src/test/java/datadog/communication/otlp/OtlpCanonicalResourceAttributesTest.java b/communication/otlp-exporter/src/test/java/datadog/communication/otlp/OtlpCanonicalResourceAttributesTest.java new file mode 100644 index 00000000000..ac4d28de7de --- /dev/null +++ b/communication/otlp-exporter/src/test/java/datadog/communication/otlp/OtlpCanonicalResourceAttributesTest.java @@ -0,0 +1,75 @@ +package datadog.communication.otlp; + +import static datadog.communication.ddagent.TracerVersion.TRACER_VERSION; +import static java.util.Arrays.asList; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import datadog.trace.api.Config; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.Test; + +class OtlpCanonicalResourceAttributesTest { + + @Test + void visitsAllAttributesInOrder() { + Config config = config("env", "1.0", true, "host"); + + assertEquals( + asList( + "service.name=svc", + "deployment.environment.name=env", + "service.version=1.0", + "host.name=host", + "telemetry.sdk.name=datadog", + "telemetry.sdk.version=" + TRACER_VERSION, + "telemetry.sdk.language=java"), + visit(config)); + } + + @Test + void skipsEmptyAndUnreportedAttributes() { + Config config = config("", "", false, "host"); + + assertEquals( + asList( + "service.name=svc", + "telemetry.sdk.name=datadog", + "telemetry.sdk.version=" + TRACER_VERSION, + "telemetry.sdk.language=java"), + visit(config)); + } + + @Test + void skipsEmptyHostName() { + Config config = config("env", "1.0", true, ""); + + assertEquals( + asList( + "service.name=svc", + "deployment.environment.name=env", + "service.version=1.0", + "telemetry.sdk.name=datadog", + "telemetry.sdk.version=" + TRACER_VERSION, + "telemetry.sdk.language=java"), + visit(config)); + } + + private static Config config(String env, String version, boolean reportHost, String host) { + Config config = mock(Config.class); + when(config.getServiceName()).thenReturn("svc"); + when(config.getEnv()).thenReturn(env); + when(config.getVersion()).thenReturn(version); + when(config.isReportHostName()).thenReturn(reportHost); + when(config.getHostName()).thenReturn(host); + return config; + } + + private static List visit(Config config) { + List visited = new ArrayList<>(); + OtlpCanonicalResourceAttributes.visit(config, (k, v) -> visited.add(k + "=" + v)); + return visited; + } +} diff --git a/dd-trace-core/src/test/java/datadog/trace/core/otlp/common/OtlpGrpcRequestBodyTest.java b/communication/otlp-exporter/src/test/java/datadog/communication/otlp/OtlpGrpcRequestBodyTest.java similarity index 98% rename from dd-trace-core/src/test/java/datadog/trace/core/otlp/common/OtlpGrpcRequestBodyTest.java rename to communication/otlp-exporter/src/test/java/datadog/communication/otlp/OtlpGrpcRequestBodyTest.java index 702be21873c..407ee87f93d 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/otlp/common/OtlpGrpcRequestBodyTest.java +++ b/communication/otlp-exporter/src/test/java/datadog/communication/otlp/OtlpGrpcRequestBodyTest.java @@ -1,4 +1,4 @@ -package datadog.trace.core.otlp.common; +package datadog.communication.otlp; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/dd-trace-core/src/test/java/datadog/trace/core/otlp/common/OtlpHttpRequestBodyTest.java b/communication/otlp-exporter/src/test/java/datadog/communication/otlp/OtlpHttpRequestBodyTest.java similarity index 98% rename from dd-trace-core/src/test/java/datadog/trace/core/otlp/common/OtlpHttpRequestBodyTest.java rename to communication/otlp-exporter/src/test/java/datadog/communication/otlp/OtlpHttpRequestBodyTest.java index bd354049e9c..68c5d1eb2d1 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/otlp/common/OtlpHttpRequestBodyTest.java +++ b/communication/otlp-exporter/src/test/java/datadog/communication/otlp/OtlpHttpRequestBodyTest.java @@ -1,4 +1,4 @@ -package datadog.trace.core.otlp.common; +package datadog.communication.otlp; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/dd-trace-core/src/test/java/datadog/trace/core/otlp/common/OtlpSenderSupportTest.java b/communication/otlp-exporter/src/test/java/datadog/communication/otlp/OtlpSenderSupportTest.java similarity index 87% rename from dd-trace-core/src/test/java/datadog/trace/core/otlp/common/OtlpSenderSupportTest.java rename to communication/otlp-exporter/src/test/java/datadog/communication/otlp/OtlpSenderSupportTest.java index 2d033113213..1b79fcbd13a 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/otlp/common/OtlpSenderSupportTest.java +++ b/communication/otlp-exporter/src/test/java/datadog/communication/otlp/OtlpSenderSupportTest.java @@ -1,4 +1,4 @@ -package datadog.trace.core.otlp.common; +package datadog.communication.otlp; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -12,7 +12,6 @@ import datadog.communication.http.HttpRetryPolicy; import datadog.communication.http.OkHttpUtils; import datadog.logging.RatelimitedLogger; -import datadog.trace.common.writer.RemoteApi; import java.io.IOException; import okhttp3.MediaType; import okhttp3.OkHttpClient; @@ -39,8 +38,7 @@ void successfulResponseIsReturnedWithoutLogging() throws IOException { .when(() -> OkHttpUtils.sendWithRetries(client, retryPolicy, request)) .thenReturn(response); - RemoteApi.Response result = - OtlpSenderSupport.send(client, retryPolicy, request, ratelimitedLogger); + OtlpResponse result = OtlpSenderSupport.send(client, retryPolicy, request, ratelimitedLogger); assertTrue(result.success()); assertEquals(200, result.status().getAsInt()); @@ -56,8 +54,7 @@ void unsuccessfulResponseIsReturnedAndLogged() throws IOException { .when(() -> OkHttpUtils.sendWithRetries(client, retryPolicy, request)) .thenReturn(response); - RemoteApi.Response result = - OtlpSenderSupport.send(client, retryPolicy, request, ratelimitedLogger); + OtlpResponse result = OtlpSenderSupport.send(client, retryPolicy, request, ratelimitedLogger); assertFalse(result.success()); assertEquals(500, result.status().getAsInt()); @@ -73,8 +70,7 @@ void ioExceptionIsReturnedAsFailureAndLogged() throws IOException { .when(() -> OkHttpUtils.sendWithRetries(client, retryPolicy, request)) .thenThrow(exception); - RemoteApi.Response result = - OtlpSenderSupport.send(client, retryPolicy, request, ratelimitedLogger); + OtlpResponse result = OtlpSenderSupport.send(client, retryPolicy, request, ratelimitedLogger); assertFalse(result.success()); assertTrue(result.exception().isPresent()); diff --git a/communication/otlp-exporter/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/communication/otlp-exporter/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker new file mode 100644 index 00000000000..1f0955d450f --- /dev/null +++ b/communication/otlp-exporter/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -0,0 +1 @@ +mock-maker-inline diff --git a/dd-java-agent/agent-profiling/build.gradle b/dd-java-agent/agent-profiling/build.gradle index 8d95092cd75..dd9f134c1ae 100644 --- a/dd-java-agent/agent-profiling/build.gradle +++ b/dd-java-agent/agent-profiling/build.gradle @@ -23,6 +23,7 @@ dependencies { api project(':dd-java-agent:agent-profiling:profiling-ddprof') api project(':dd-java-agent:agent-profiling:profiling-uploader') + api project(':dd-java-agent:agent-profiling:profiling-otel') api project(':dd-java-agent:agent-profiling:profiling-controller') implementation project(':dd-java-agent:agent-profiling:profiling-scrubber') api project(':dd-java-agent:agent-profiling:profiling-controller-jfr') @@ -42,13 +43,39 @@ configurations { } } +// jafar-tools is a fat jar bundling (an incomplete copy of) the jafar-parser classes; the agent +// bundles jafar-parser (via profiling-otel) instead and only needs the io.jafar.tools classes +// (used by profiling-scrubber) from the fat jar +configurations { + jafarTools +} +dependencies { + jafarTools("io.btrace:jafar-tools:${libs.versions.jafar.get()}") { + transitive = false + } +} +def jafarToolsJar = configurations.jafarTools + tasks.named("shadowJar", ShadowJar) { dependencies deps.excludeShared + // drop the jafar-tools fat jar (parser classes come from jafar-parser via profiling-otel) and + // re-add only its unique io/jafar/tools classes to keep a single copy of every class + dependencies { + exclude(dependency('io.btrace:jafar-tools:.*')) + } + from({ zipTree(jafarToolsJar.singleFile) }) { + include 'io/jafar/tools/**' + } + // Exclude multi-release versioned classes from jafar-parser. // These are duplicates of base classes for newer Java APIs and confuse // the GraalVM native-image builder when the profiling jar is embedded in the agent. exclude 'META-INF/versions/**' + + // JfrToOtlpConverterCLI is the entry point of the standalone converter jar; the agent never + // loads it, so keep it out of the profiling content bundled into the agent jar + exclude 'com/datadog/profiling/otel/JfrToOtlpConverterCLI.class' } tasks.named("jar", Jar) { diff --git a/dd-java-agent/agent-profiling/profiling-controller-openjdk/src/main/java/com/datadog/profiling/controller/openjdk/OpenJdkRecordingData.java b/dd-java-agent/agent-profiling/profiling-controller-openjdk/src/main/java/com/datadog/profiling/controller/openjdk/OpenJdkRecordingData.java index da022c226ed..3c25699ac04 100644 --- a/dd-java-agent/agent-profiling/profiling-controller-openjdk/src/main/java/com/datadog/profiling/controller/openjdk/OpenJdkRecordingData.java +++ b/dd-java-agent/agent-profiling/profiling-controller-openjdk/src/main/java/com/datadog/profiling/controller/openjdk/OpenJdkRecordingData.java @@ -45,7 +45,7 @@ public RecordingInputStream getStream() throws IOException { } @Override - public void release() { + protected void doRelease() { recording.close(); } diff --git a/dd-java-agent/agent-profiling/profiling-controller-oracle/src/main/java/com/datadog/profiling/controller/oracle/OracleJdkRecordingData.java b/dd-java-agent/agent-profiling/profiling-controller-oracle/src/main/java/com/datadog/profiling/controller/oracle/OracleJdkRecordingData.java index 226e1b0f24d..04c98e1ae7a 100644 --- a/dd-java-agent/agent-profiling/profiling-controller-oracle/src/main/java/com/datadog/profiling/controller/oracle/OracleJdkRecordingData.java +++ b/dd-java-agent/agent-profiling/profiling-controller-oracle/src/main/java/com/datadog/profiling/controller/oracle/OracleJdkRecordingData.java @@ -23,9 +23,13 @@ import java.util.Date; import javax.annotation.Nonnull; import javax.management.ObjectName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** Implementation for profiling recordings. */ public class OracleJdkRecordingData extends RecordingData { + private static final Logger log = LoggerFactory.getLogger(OracleJdkRecordingData.class); + private final ObjectName recordingId; private final String name; @@ -51,8 +55,15 @@ public RecordingInputStream getStream() throws IOException { } @Override - public void release() { - // noop + protected void doRelease() { + // the recording is shared by every listener's stream — the OTLP listener may close its + // stream while the classic uploader still needs to open its own, so the recording itself + // can only be closed when the last reference is released + try { + helper.closeRecording(recordingId); + } catch (Exception e) { + log.warn("Failed to close Oracle JDK recording {}", recordingId, e); + } } @Override @@ -116,7 +127,8 @@ public void close() throws IOException { if (streamId != -1) { helper.closeStream(streamId); } - helper.closeRecording(recordingId); + // the recording itself is closed in doRelease() — closing it here would prevent any + // other listener from opening its own stream on the same recording } catch (Exception e) { throw new IOException(e); } diff --git a/dd-java-agent/agent-profiling/profiling-controller/src/test/java/com/datadog/profiling/controller/RecordingDataRefCountingTest.java b/dd-java-agent/agent-profiling/profiling-controller/src/test/java/com/datadog/profiling/controller/RecordingDataRefCountingTest.java new file mode 100644 index 00000000000..e210ff25b9b --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-controller/src/test/java/com/datadog/profiling/controller/RecordingDataRefCountingTest.java @@ -0,0 +1,223 @@ +/* + * Copyright 2025 Datadog + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datadog.profiling.controller; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.api.profiling.ProfilingSnapshot; +import datadog.trace.api.profiling.RecordingData; +import datadog.trace.api.profiling.RecordingInputStream; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.file.Path; +import java.time.Instant; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.junit.jupiter.api.Test; + +/** + * Tests for RecordingData reference counting with multiple handlers. The reference count starts at + * 1 (the base reference). The primary listener releases the base reference; each additional + * listener calls retain() then release(). + */ +public class RecordingDataRefCountingTest { + + /** Test RecordingData implementation that tracks release calls. */ + private static class TestRecordingData extends RecordingData { + private final AtomicInteger releaseCount = new AtomicInteger(0); + private final CountDownLatch releaseLatch = new CountDownLatch(1); + + public TestRecordingData() { + super(Instant.now(), Instant.now(), ProfilingSnapshot.Kind.PERIODIC); + } + + @Nonnull + @Override + public RecordingInputStream getStream() throws IOException { + return new RecordingInputStream(new ByteArrayInputStream(new byte[0])); + } + + @Override + protected void doRelease() { + releaseCount.incrementAndGet(); + releaseLatch.countDown(); + } + + @Nullable + @Override + public Path getPath() { + return null; + } + + @Override + public String getName() { + return "test-recording"; + } + + public int getReleaseCount() { + return releaseCount.get(); + } + + public boolean awaitRelease(long timeout, TimeUnit unit) throws InterruptedException { + return releaseLatch.await(timeout, unit); + } + } + + @Test + public void testSingleHandler() throws InterruptedException { + TestRecordingData data = new TestRecordingData(); + + // Single handler: just release the base reference (no retain needed) + data.release(); + + assertTrue(data.awaitRelease(1, TimeUnit.SECONDS), "Release should be called"); + assertEquals(1, data.getReleaseCount(), "doRelease() should be called exactly once"); + } + + @Test + public void testTwoHandlers() throws InterruptedException { + TestRecordingData data = new TestRecordingData(); + + // Two handlers: retain once for the additional handler, release twice + data.retain(); // Additional handler (e.g., OTLP) + assertEquals(0, data.getReleaseCount(), "Should not be released yet"); + + // First handler releases (base reference) + data.release(); + assertEquals(0, data.getReleaseCount(), "Should not be released after first release"); + + // Second handler releases + data.release(); + + assertTrue(data.awaitRelease(1, TimeUnit.SECONDS), "Release should be called"); + assertEquals(1, data.getReleaseCount(), "doRelease() should be called exactly once"); + } + + @Test + public void testThreeHandlers() throws InterruptedException { + TestRecordingData data = new TestRecordingData(); + + // Three handlers: retain twice, release three times + data.retain(); // Additional handler 1 + data.retain(); // Additional handler 2 + assertEquals(0, data.getReleaseCount(), "Should not be released yet"); + + // First two handlers release + data.release(); + data.release(); + assertEquals(0, data.getReleaseCount(), "Should not be released after two releases"); + + // Third handler releases + data.release(); + + assertTrue(data.awaitRelease(1, TimeUnit.SECONDS), "Release should be called"); + assertEquals(1, data.getReleaseCount(), "doRelease() should be called exactly once"); + } + + @Test + public void testOverReleaseIsSilentNoOp() throws InterruptedException { + TestRecordingData data = new TestRecordingData(); + + // Release the base reference + data.release(); + assertTrue(data.awaitRelease(1, TimeUnit.SECONDS), "Release should be called"); + + // Additional release after full release is a silent no-op — unlike retain() after full release + data.release(); + assertEquals(1, data.getReleaseCount(), "doRelease() should still be called exactly once"); + } + + @Test + public void testRetainAfterFullRelease() throws InterruptedException { + TestRecordingData data = new TestRecordingData(); + + data.release(); + assertTrue(data.awaitRelease(1, TimeUnit.SECONDS), "Release should be called"); + + // Cannot retain after full release + assertThrows( + IllegalStateException.class, data::retain, "Should throw when retaining after release"); + } + + @Test + public void testMultipleReleaseIdempotent() throws InterruptedException { + TestRecordingData data = new TestRecordingData(); + + data.release(); + assertTrue(data.awaitRelease(1, TimeUnit.SECONDS), "Release should be called"); + + // Additional release calls should be silent no-ops (released flag guards them) + data.release(); + data.release(); + + assertEquals(1, data.getReleaseCount(), "doRelease() should still be called exactly once"); + } + + @Test + public void testConcurrentHandlers() throws InterruptedException { + TestRecordingData data = new TestRecordingData(); + int numAdditionalHandlers = 9; // total 10 with the base reference + + // Retain for all additional handlers + for (int i = 0; i < numAdditionalHandlers; i++) { + data.retain(); + } + + int totalHandlers = numAdditionalHandlers + 1; + + // Simulate concurrent release from multiple threads + CountDownLatch startLatch = new CountDownLatch(1); + CountDownLatch doneLatch = new CountDownLatch(totalHandlers); + + for (int i = 0; i < totalHandlers; i++) { + new Thread( + () -> { + try { + startLatch.await(); + data.release(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + // count down even on interrupt, otherwise it masks as a liveness timeout below + doneLatch.countDown(); + } + }) + .start(); + } + + // Start all threads + startLatch.countDown(); + + // Wait for all threads to complete + assertTrue(doneLatch.await(5, TimeUnit.SECONDS), "All threads should complete"); + assertTrue(data.awaitRelease(1, TimeUnit.SECONDS), "Release should be called"); + assertEquals(1, data.getReleaseCount(), "doRelease() should be called exactly once"); + } + + @Test + public void testRetainChaining() { + TestRecordingData data = new TestRecordingData(); + + // retain() should return this for chaining + RecordingData result = data.retain(); + assertEquals(data, result, "retain() should return the same instance"); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerRecordingData.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerRecordingData.java index b1a6c7eea41..fd142bcde43 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerRecordingData.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerRecordingData.java @@ -24,7 +24,7 @@ public RecordingInputStream getStream() throws IOException { } @Override - public void release() { + protected void doRelease() { try { Files.deleteIfExists(recordingFile); } catch (IOException e) { diff --git a/dd-java-agent/agent-profiling/profiling-otel/build.gradle.kts b/dd-java-agent/agent-profiling/profiling-otel/build.gradle.kts new file mode 100644 index 00000000000..5b4ffc0a03e --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/build.gradle.kts @@ -0,0 +1,228 @@ +plugins { + `java-library` + id("dd-trace-java.module.internal-library") + id("com.gradleup.shadow") + id("me.champeau.jmh") +} + +// Tests require Java 17+ (JMC 9.1.1); on JVM 8 (where -PcheckCoverage runs) tests are skipped, +// so all main classes would have 0% coverage. Exclude them from verification. +extra["excludedClassesCoverage"] = listOf( + "com.datadog.profiling.otel.JfrToOtlpConverter*", + "com.datadog.profiling.otel.JfrToOtlpConverterCLI", + "com.datadog.profiling.otel.jfr.*", + "com.datadog.profiling.otel.proto.*", + "com.datadog.profiling.otel.proto.dictionary.*", +) + +jmh { + jmhVersion = libs.versions.jmh.get() + + // Allow filtering benchmarks via command line + // Usage: ./gradlew jmh -PjmhIncludes="JfrToOtlpConverterBenchmark" + // Usage: ./gradlew jmh -PjmhIncludes=".*convertJfrToOtlp" + if (project.hasProperty("jmhIncludes")) { + val pattern = project.property("jmhIncludes") as String + includes = listOf(pattern) + } + + // Profiling support + // Usage: ./gradlew jmh -PjmhProfile=true + // Generates flamegraph and allocation profile + if (project.hasProperty("jmhProfile")) { + profilers = listOf("gc", "stack") + jvmArgs = listOf( + "-XX:+UnlockDiagnosticVMOptions", + "-XX:+DebugNonSafepoints" + ) + } + + // async-profiler CPU flamegraph + // Usage: ./gradlew jmh -PjmhAsyncProfiler=/path/to/libasyncProfiler.dylib + // Output: /tmp/jmh-async-profile.html + if (project.hasProperty("jmhAsyncProfiler")) { + val lib = project.property("jmhAsyncProfiler") as String + jvmArgs = listOf( + "-XX:+UnlockDiagnosticVMOptions", + "-XX:+DebugNonSafepoints", + "-agentpath:$lib=start,event=cpu,file=/tmp/jmh-async-profile.html,flamegraph" + ) + } +} + +// OTLP validation tests removed - use profcheck validation instead (see validateOtlp task below) + +repositories { + maven { + url = uri("https://s01.oss.sonatype.org/content/repositories/snapshots/") + mavenContent { + snapshotsOnly() + // jafar is only published as a snapshot; restrict resolution to that group + includeGroup("io.btrace") + } + } +} + +configure { + minJavaVersion = JavaVersion.VERSION_17 +} + +tasks.named("compileTestJava") { + // JMC 9.1.1 requires Java 17, and we need jdk.jfr.Event for stack trace testing + options.release.set(17) + javaCompiler.set( + javaToolchains.compilerFor { languageVersion.set(JavaLanguageVersion.of(17)) } + ) +} + +tasks.named("compileJmhJava") { + // JMC 9.1.1 requires Java 17, and we need jdk.jfr.Event for JMH benchmarks + options.release.set(17) + javaCompiler.set( + javaToolchains.compilerFor { languageVersion.set(JavaLanguageVersion.of(17)) } + ) +} + +// Create fat jar for standalone CLI usage +tasks.named("shadowJar") { + archiveClassifier.set("cli") + manifest { + attributes["Main-Class"] = "com.datadog.profiling.otel.JfrToOtlpConverterCLI" + } + // Exclude SLF4J service provider files to avoid warnings + exclude("META-INF/services/org.slf4j.spi.SLF4JServiceProvider") +} + +// CLI task for converting JFR files +// Usage: ./gradlew :dd-java-agent:agent-profiling:profiling-otel:convertJfr --args="input.jfr output.pb" +// Usage: ./gradlew :dd-java-agent:agent-profiling:profiling-otel:convertJfr --args="--json input.jfr output.json" +tasks.register("convertJfr") { + group = "application" + description = "Convert JFR recording to OTLP profiles format" + classpath = sourceSets["main"].runtimeClasspath + mainClass.set("com.datadog.profiling.otel.JfrToOtlpConverterCLI") + + // Uses Gradle's built-in --args parameter which properly handles spaces in paths +} + +// Build profcheck Docker image +// Usage: ./gradlew :dd-java-agent:agent-profiling:profiling-otel:buildProfcheck +tasks.register("buildProfcheck") { + group = "verification" + description = "Build profcheck Docker image for OTLP validation" + workingDir = rootDir + commandLine("docker", "build", "-f", "docker/Dockerfile.profcheck", "-t", "profcheck:latest", ".") + + // Check if Docker is available + doFirst { + val process = ProcessBuilder("docker", "info").redirectErrorStream(true).start() + if (process.waitFor() != 0) { + throw org.gradle.api.GradleException("Docker is not available. Profcheck validation requires Docker to be running.") + } + } +} + +// OTLP validation tests (ProfcheckValidationTest, OtlpCollectorValidationTest) are gated behind +// the -PrunOtlpValidation Gradle property. They require Docker and network access to pull/build +// images, which is not available in the standard CI test matrix. Enable manually: +// ./gradlew :dd-java-agent:agent-profiling:profiling-otel:test -PrunOtlpValidation +tasks.named("test") { + if (project.hasProperty("runOtlpValidation")) { + jvmArgs("-Drun.otlp.validation=true") + + // Build profcheck image if Docker is available (for ProfcheckValidationTest) + doFirst { + val dockerAvailable = try { + val process = ProcessBuilder("docker", "info").redirectErrorStream(true).start() + process.waitFor() == 0 + } catch (e: Exception) { + false + } + + if (dockerAvailable) { + logger.lifecycle("Building profcheck Docker image for validation tests...") + val buildProcess = ProcessBuilder( + "docker", + "build", + "-f", + "$rootDir/docker/Dockerfile.profcheck", + "-t", + "profcheck:latest", + rootDir.toString() + ).redirectErrorStream(true).start() + // drain the output while waiting — docker build routinely exceeds the OS pipe-buffer + // capacity (~64 KiB) and an unread full pipe deadlocks waitFor() + val outputBuilder = StringBuilder() + val drainThread = Thread { + buildProcess.inputStream.bufferedReader().forEachLine { line -> + synchronized(outputBuilder) { outputBuilder.appendLine(line) } + } + } + drainThread.isDaemon = true + drainThread.start() + val exitCode = buildProcess.waitFor() + drainThread.join(10_000) + if (exitCode != 0) { + val output = synchronized(outputBuilder) { outputBuilder.toString() } + throw org.gradle.api.GradleException("Failed to build profcheck Docker image (exit $exitCode):\n$output") + } + } else { + // without the image the docker-tagged validation tests cannot run + logger.warn("Docker not available: excluding tests tagged 'docker'.") + useJUnitPlatform { + excludeTags("docker") + } + } + } + } +} + +// Validate OTLP output using profcheck +// Usage: ./gradlew :dd-java-agent:agent-profiling:profiling-otel:validateOtlp -PotlpFile=/path/to/output.pb +tasks.register("validateOtlp") { + group = "verification" + description = "Validate OTLP profile using profcheck (requires Docker)" + + // Ensure profcheck image exists + dependsOn("buildProfcheck") + + doFirst { + if (!project.hasProperty("otlpFile")) { + throw org.gradle.api.GradleException("Property 'otlpFile' is required. Usage: -PotlpFile=/path/to/output.pb") + } + + val otlpFilePath = project.property("otlpFile") as String + val otlpFile = file(otlpFilePath) + + if (!otlpFile.exists()) { + throw org.gradle.api.GradleException("File not found: $otlpFilePath") + } + + val parentDir = otlpFile.parentFile.absolutePath + val fileName = otlpFile.name + + // Run profcheck in Docker with volume mount + commandLine( + "docker", + "run", + "--rm", + "-v", + "$parentDir:/data:ro", + "profcheck:latest", + "/data/$fileName" + ) + } +} + +dependencies { + implementation(libs.jafar.parser) + implementation(project(":internal-api")) + implementation(project(":components:json")) + + testImplementation(libs.bundles.junit5) + testImplementation(libs.bundles.jmc) + testImplementation(libs.jmc.flightrecorder.writer) + testImplementation(libs.testcontainers) + testImplementation("org.testcontainers:junit-jupiter:1.21.3") + testImplementation(libs.testing.okhttp3) +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/convert-jfr.sh b/dd-java-agent/agent-profiling/profiling-otel/convert-jfr.sh new file mode 100755 index 00000000000..6a9da5c3e87 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/convert-jfr.sh @@ -0,0 +1,357 @@ +#!/usr/bin/env bash + +# JFR to OTLP Converter Script +# +# This script provides a convenient wrapper around the Gradle-based JFR converter. +# It automatically handles the classpath and provides a simpler interface. +# +# Usage: +# ./convert-jfr.sh [options] [input2.jfr ...] +# +# Options: +# --json Output in JSON format instead of protobuf +# --pretty Pretty-print JSON output (implies --json) +# --include-payload Include original JFR payload in OTLP output +# --diagnostics Show detailed diagnostics (file sizes, conversion time) +# --help Show this help message +# +# Examples: +# ./convert-jfr.sh recording.jfr output.pb +# ./convert-jfr.sh --json recording.jfr output.json +# ./convert-jfr.sh --pretty recording.jfr output.json +# ./convert-jfr.sh --diagnostics file1.jfr file2.jfr combined.pb + +set -e + +# Script directory and project root +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +MODULE_DIR="$SCRIPT_DIR" + +# Fat jar location +FAT_JAR_DIR="$MODULE_DIR/build/libs" +FAT_JAR_PATTERN="$FAT_JAR_DIR/profiling-otel-*-cli.jar" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +BLUE='\033[0;34m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +NC='\033[0m' + +log_info() { echo -e "${BLUE}[INFO]${NC} $1"; } +log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $1"; } +log_diagnostic() { echo -e "${CYAN}[DIAG]${NC} $1"; } + +# Get file size in bytes +get_file_size_bytes() { + local file="$1" + if [ -f "$file" ]; then + # Try GNU stat first (Linux, or GNU coreutils on macOS) + local size=$(stat -c %s "$file" 2>/dev/null) + if [ -n "$size" ] && [ "$size" != "" ]; then + echo "$size" + else + # Fall back to BSD stat (macOS native) + stat -f %z "$file" 2>/dev/null || echo "0" + fi + else + echo "0" + fi +} + +# Format bytes to human-readable +format_bytes() { + local bytes=$1 + if [ "$bytes" -lt 1024 ]; then + echo "${bytes}B" + elif [ "$bytes" -lt 1048576 ]; then + echo "$(awk "BEGIN {printf \"%.1f\", $bytes/1024}")KB" + elif [ "$bytes" -lt 1073741824 ]; then + echo "$(awk "BEGIN {printf \"%.1f\", $bytes/1048576}")MB" + else + echo "$(awk "BEGIN {printf \"%.1f\", $bytes/1073741824}")GB" + fi +} + +# Calculate compression ratio +calc_compression_ratio() { + local input_size=$1 + local output_size=$2 + if [ "$input_size" -eq 0 ]; then + echo "N/A" + else + awk "BEGIN {printf \"%.1f%%\", ($output_size / $input_size) * 100}" + fi +} + +# Get modification time of a file (seconds since epoch) +get_mtime() { + local file="$1" + if [ ! -f "$file" ]; then + echo "0" + return + fi + + # Try GNU stat first (Linux, or GNU coreutils on macOS) + local mtime=$(stat -c %Y "$file" 2>/dev/null) + if [ -n "$mtime" ] && [ "$mtime" != "" ]; then + echo "$mtime" + else + # Fall back to BSD stat (macOS native) + stat -f %m "$file" 2>/dev/null || echo "0" + fi +} + +# Find the most recent build input: sources plus build files and resources — a stale jar +# older than any of these silently runs the converter against old code +find_newest_input() { + local newest=0 + local mtime file + while IFS= read -r -d '' file; do + mtime=$(get_mtime "$file") + if [ "$mtime" -gt "$newest" ]; then + newest="$mtime" + fi + done < <(find "$MODULE_DIR/src/main/java" "$MODULE_DIR/src/main/resources" \ + -type f -print0 2>/dev/null; \ + find "$MODULE_DIR/build.gradle.kts" \ + "$PROJECT_ROOT/gradle/libs.versions.toml" \ + -type f -print0 2>/dev/null) + echo "$newest" +} + +# Find the newest matching fat jar; glob iteration instead of `ls -t $PATTERN` to survive +# paths with spaces without needing bash 4.x +find_fat_jar() { + local newest_jar="" newest_mtime=0 jar mtime + while IFS= read -r jar; do + [ -z "$jar" ] && continue + mtime=$(get_mtime "$jar") + if [ "$mtime" -gt "$newest_mtime" ]; then + newest_mtime="$mtime" + newest_jar="$jar" + fi + done < <(compgen -G "$FAT_JAR_PATTERN" 2>/dev/null) + echo "$newest_jar" +} + +# Check if fat jar needs rebuilding +needs_rebuild() { + # Find the fat jar + local jar + jar=$(find_fat_jar) + + if [ -z "$jar" ] || [ ! -f "$jar" ]; then + # Jar doesn't exist + return 0 + fi + + # Get jar modification time + local jar_mtime + jar_mtime=$(get_mtime "$jar") + + # Get newest build input time + local newest_input_mtime + newest_input_mtime=$(find_newest_input) + + # Rebuild if any input is newer than jar + if [ "$newest_input_mtime" -gt "$jar_mtime" ]; then + return 0 + fi + + # No rebuild needed + return 1 +} + +# Ensure fat jar is built and up-to-date +ensure_fat_jar() { + local rebuild_needed=false + if needs_rebuild; then + rebuild_needed=true + fi + + if [ "$rebuild_needed" = true ]; then + log_info "Building fat jar (source files changed or jar missing)..." >&2 + cd "$PROJECT_ROOT" + # guard explicitly — under `set -e` an unguarded failing command aborts the script + # before the error branch below can run + if ! ./gradlew -q :dd-java-agent:agent-profiling:profiling-otel:shadowJar >"$_GRADLE_TMP" 2>&1; then + log_error "Failed to build fat jar" >&2 + tail -n 20 "$_GRADLE_TMP" >&2 + rm -f "$_GRADLE_TMP" + exit 1 + fi + rm -f "$_GRADLE_TMP" + fi + + # Find and return the fat jar path + local jar + jar=$(find_fat_jar) + if [ -z "$jar" ] || [ ! -f "$jar" ]; then + log_error "Fat jar not found after build" >&2 + exit 1 + fi + + echo "$jar" +} + +show_help() { + cat << EOF +JFR to OTLP Converter + +Usage: + $(basename "$0") [options] [input2.jfr ...] + +Options: + --json Output in JSON format instead of protobuf + --pretty Pretty-print JSON output (implies --json) + --include-payload Include original JFR payload in OTLP output + --diagnostics Show detailed diagnostics (file sizes, conversion time) + --help Show this help message + +Examples: + # Convert to protobuf (default) + $(basename "$0") recording.jfr output.pb + + # Convert to JSON + $(basename "$0") --json recording.jfr output.json + + # Convert to pretty JSON + $(basename "$0") --pretty recording.jfr output.json + + # Include original JFR in output + $(basename "$0") --include-payload recording.jfr output.pb + + # Combine multiple JFR files + $(basename "$0") file1.jfr file2.jfr combined.pb + + # Show detailed diagnostics + $(basename "$0") --diagnostics recording.jfr output.pb + +Notes: + - Uses a fat jar for fast execution (no Gradle overhead) + - Automatically rebuilds jar if source files change + - Output format is detected from extension (.pb or .json) + - Use --diagnostics to see file sizes and conversion times + +EOF +} + +# Parse arguments +if [ $# -eq 0 ] || [ "$1" = "--help" ] || [ "$1" = "-h" ]; then + show_help + exit 0 +fi + +# Check for diagnostics flag +SHOW_DIAGNOSTICS=false +CONVERTER_ARGS=() +NON_FLAG_ARGS=() + +while [[ $# -gt 0 ]]; do + case $1 in + --diagnostics) + SHOW_DIAGNOSTICS=true + shift + ;; + --json|--pretty|--include-payload) + CONVERTER_ARGS+=("$1") + shift + ;; + *) + # Collect files + CONVERTER_ARGS+=("$1") + NON_FLAG_ARGS+=("$1") + shift + ;; + esac +done + +# Output file is the last non-flag argument; everything before it is an input file. +# Classification is by trailing position, not by -f existence, so a misspelled input path +# still counts toward the diagnostics baseline. +if [ ${#NON_FLAG_ARGS[@]} -eq 0 ]; then + log_error "No input or output files given" + exit 2 +fi +_OUTPUT_FILE="${NON_FLAG_ARGS[$((${#NON_FLAG_ARGS[@]} - 1))]}" +INPUT_FILES=() +if [ ${#NON_FLAG_ARGS[@]} -gt 1 ]; then + for ((_i = 0; _i < ${#NON_FLAG_ARGS[@]} - 1; _i++)); do + INPUT_FILES+=("${NON_FLAG_ARGS[$_i]}") + if [ ! -f "${NON_FLAG_ARGS[$_i]}" ]; then + log_error "Input file not found: ${NON_FLAG_ARGS[$_i]}" + exit 2 + fi + done +fi + +# Calculate total input size if diagnostics enabled +TOTAL_INPUT_SIZE=0 +if [ "$SHOW_DIAGNOSTICS" = true ]; then + for input_file in "${INPUT_FILES[@]}"; do + size=$(get_file_size_bytes "$input_file") + TOTAL_INPUT_SIZE=$((TOTAL_INPUT_SIZE + size)) + log_diagnostic "Input: $input_file ($(format_bytes $size))" + done + if [ ${#INPUT_FILES[@]} -gt 0 ]; then + log_diagnostic "Total input size: $(format_bytes $TOTAL_INPUT_SIZE)" + fi +fi + +_GRADLE_TMP=$(mktemp) +# Ensure fat jar is built and get its path +FAT_JAR=$(ensure_fat_jar) + +# Run conversion using fat jar and capture output +# Suppress SLF4J warnings (it defaults to NOP logger which is fine for CLI) +# Use a temp file to capture output so the java process exit code is not masked by grep +_CONVERTER_TMP=$(mktemp) +# capture the exit code instead of letting set -e abort before the cleanup below runs +CONVERTER_EXIT=0 +java -jar "$FAT_JAR" "${CONVERTER_ARGS[@]}" >"$_CONVERTER_TMP" 2>&1 || CONVERTER_EXIT=$? +CONVERTER_OUTPUT=$(grep -vE "^SLF4J:|SLF4JServiceProvider" "$_CONVERTER_TMP") +rm -f "$_CONVERTER_TMP" + +if [ $CONVERTER_EXIT -eq 0 ]; then + # Extract output file (trailing non-flag argument) + OUTPUT_FILE="$_OUTPUT_FILE" + + if [ "$SHOW_DIAGNOSTICS" = true ]; then + # With diagnostics: show converter output plus enhanced metrics + echo "$CONVERTER_OUTPUT" + + OUTPUT_SIZE=$(get_file_size_bytes "$OUTPUT_FILE") + + echo "" + log_diagnostic "=== Enhanced Diagnostics ===" + + # Show size comparison with input + if [ ${#INPUT_FILES[@]} -gt 0 ]; then + RATIO=$(calc_compression_ratio $TOTAL_INPUT_SIZE $OUTPUT_SIZE) + log_diagnostic "Input → Output: $(format_bytes $TOTAL_INPUT_SIZE) → $(format_bytes $OUTPUT_SIZE)" + log_diagnostic "Compression: $RATIO of original" + + if [ "$OUTPUT_SIZE" -lt "$TOTAL_INPUT_SIZE" ]; then + SAVINGS=$((TOTAL_INPUT_SIZE - OUTPUT_SIZE)) + SAVINGS_PCT=$(awk "BEGIN {printf \"%.1f%%\", (1 - $OUTPUT_SIZE/$TOTAL_INPUT_SIZE) * 100}") + log_diagnostic "Space saved: $(format_bytes $SAVINGS) ($SAVINGS_PCT reduction)" + fi + fi + echo "" + else + # Without diagnostics: concise output + # Extract just the key info from converter output + CONVERSION_TIME=$(echo "$CONVERTER_OUTPUT" | grep -o 'Time: [0-9]* ms' | grep -o '[0-9]*' | head -1) + OUTPUT_SIZE=$(get_file_size_bytes "$OUTPUT_FILE") + + log_success "Converted: $OUTPUT_FILE ($(format_bytes $OUTPUT_SIZE), ${CONVERSION_TIME}ms)" + fi +else + echo "$CONVERTER_OUTPUT" + log_error "Conversion failed with exit code $CONVERTER_EXIT" + exit $CONVERTER_EXIT +fi diff --git a/dd-java-agent/agent-profiling/profiling-otel/doc/ARCHITECTURE.md b/dd-java-agent/agent-profiling/profiling-otel/doc/ARCHITECTURE.md new file mode 100644 index 00000000000..06142869c9b --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/doc/ARCHITECTURE.md @@ -0,0 +1,846 @@ +# OTLP Profiles Writer - Architecture & Implementation Journal + +## Overview + +This module provides a JFR to OTLP/profiles format converter. It reads JFR recordings via the `RecordingData` abstraction and produces OTLP-compliant profile data in both binary protobuf and JSON formats. + +## OTLP Profiles Format + +Based on: https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/profiles/v1development/profiles.proto + +### Key Architectural Concepts + +1. **Dictionary-based Compression**: OTLP profiles use shared dictionary tables to minimize wire size. All repeated data (strings, functions, locations, stacks, links, attributes) is stored once in dictionary tables and referenced by integer indices. + +2. **Index 0 Semantics**: In all dictionary tables, index 0 is reserved for "null/unset" values. Index 0 should never be dereferenced - it represents the absence of a value. + +3. **Sample Identity**: A sample's identity is the tuple `{stack_index, set_of(attribute_indices), link_index}`. Samples with the same identity should be aggregated. + +### Message Hierarchy + +``` +ProfilesData +├── dictionary: ProfilesDictionary (shared across all profiles) +│ ├── string_table[] - interned strings (used for function names, filenames, +│ │ attribute keys, attribute units, value type names, etc.) +│ ├── function_table[] - function metadata (nameIndex, systemNameIndex, filenameIndex → string_table) +│ ├── location_table[] - stack frame locations (functionIndex → function_table) +│ ├── mapping_table[] - binary/library mappings +│ ├── stack_table[] - call stacks (arrays of locationIndex → location_table) +│ ├── link_table[] - trace context links (traceId, spanId as raw bytes) +│ └── attribute_table[] - key-value attributes (keyIndex, unitIndex → string_table; +│ string values stored as raw strings, NOT indices) +│ +└── resource_profiles[] + └── scope_profiles[] + └── profiles[] + ├── sample_type: ValueType (type, unit → string_table) + ├── period_type: ValueType (type, unit → string_table) + ├── samples[] + │ ├── stack_index -> stack_table + │ ├── attribute_indices[] -> attribute_table + │ ├── link_index -> link_table + │ ├── values[] + │ └── timestamps_unix_nano[] + └── time_unix_nano, duration_nano, profile_id, etc. +``` + +## Package Structure + +``` +com.datadog.profiling.otel/ +├── dictionary/ # Dictionary table implementations +│ ├── StringTable # String interning +│ ├── FunctionTable # Function deduplication +│ ├── LocationTable # Stack frame deduplication +│ ├── StackTable # Call stack deduplication +│ ├── LinkTable # Trace link deduplication +│ └── AttributeTable # Attribute deduplication +│ +├── proto/ # Protobuf encoding +│ ├── ProtobufEncoder # Wire format encoder +│ └── OtlpProtoFields # Field number constants +│ +├── jfr/ # JFR event type definitions +│ ├── ExecutionSample # CPU profiling (datadog.ExecutionSample) +│ ├── MethodSample # Wall-clock profiling (datadog.MethodSample) +│ ├── ObjectSample # Allocation profiling (datadog.ObjectSample) +│ ├── JavaMonitorEnter # Lock contention (jdk.JavaMonitorEnter) +│ ├── JavaMonitorWait # Monitor wait (jdk.JavaMonitorWait) +│ ├── JfrStackTrace # Stack trace container +│ ├── JfrStackFrame # Individual stack frame +│ ├── JfrMethod # Method descriptor +│ └── JfrClass # Class descriptor +│ +├── JfrToOtlpConverter # Main converter (JFR -> OTLP) +└── test/ + ├── JfrTools # Test utilities for synthetic JFR event creation + └── validation/ # OTLP profile validation utilities + ├── OtlpProfileValidator + └── ValidationResult +``` + +## JFR Event to OTLP Mapping + +| JFR Event Type | OTLP Profile Type | Value Type | Unit | +|----------------|-------------------|------------|------| +| `datadog.ExecutionSample` | cpu | count | samples | +| `datadog.MethodSample` | wall | count | samples | +| `datadog.ObjectSample` | alloc-samples | bytes | bytes | +| `jdk.JavaMonitorEnter` | lock-contention | duration | nanoseconds | +| `jdk.JavaMonitorWait` | lock-contention | duration | nanoseconds | + +## Implementation Details + +### Phase 1: Core Infrastructure (Completed) + +#### Dictionary Tables + +All dictionary tables follow a common pattern: +- Index 0 reserved for null/unset (pre-populated in constructor) +- `intern()` method returns existing index or adds new entry +- `get()` method retrieves entry by index +- `reset()` method clears table to initial state +- HashMap-based deduplication for O(1) lookup + +**StringTable**: Simple string interning. Null and empty strings map to index 0. + +**FunctionTable**: Functions identified by composite key (nameIndex, systemNameIndex, filenameIndex, startLine). All indices reference StringTable. + +**LocationTable**: Locations represent stack frames. Key is (mappingIndex, address, functionIndex, line, column). Supports multiple Line entries for inlined functions. + +**StackTable**: Stacks are arrays of location indices. Uses Arrays.hashCode/equals for array-based key comparison. Makes defensive copies of input arrays. + +**LinkTable**: Links connect samples to trace spans. Stores 16-byte traceId and 8-byte spanId. Provides convenience method for 64-bit DD trace/span IDs. + +**AttributeTable**: Supports STRING, BOOL, INT, DOUBLE value types. Key includes (keyIndex, valueType, value, unitIndex). Important: Per OTLP spec, attribute keys and units are stored as indices into StringTable, but string VALUES are stored as raw String objects (not indices). This matches the protobuf `AnyValue.string_value` field which holds raw strings. Only the keyIndex and unitIndex reference the StringTable. + +#### ProtobufEncoder + +Hand-coded protobuf wire format encoder without external dependencies: + +- **Wire Types**: VARINT (0), FIXED64 (1), LENGTH_DELIMITED (2), FIXED32 (5) +- **Varint Encoding**: Variable-length integers, 7 bits per byte, MSB indicates continuation +- **ZigZag Encoding**: For signed varints, maps negative numbers to positive +- **Fixed Encoding**: Little-endian for fixed32/fixed64 +- **Length-Delimited**: Length prefix (varint) followed by content +- **Nested Messages**: Written to temporary buffer to compute length first + +Key methods: +- `writeVarint()`, `writeFixed64()`, `writeFixed32()` +- `writeTag()` - combines field number and wire type +- `writeString()`, `writeBytes()` - length-delimited +- `writeNestedMessage()` - for sub-messages +- `writePackedVarintField()`, `writePackedFixed64Field()` - for repeated fields + +#### OtlpProtoFields + +Constants for all OTLP protobuf field numbers, organized by message type. Enables type-safe field references without magic numbers. + +### Phase 2: JFR Parsing & Event Conversion (Completed) + +#### TypedJafarParser Integration + +Uses the typed JafarParser API (from `io.btrace:jafar-parser`) for efficient JFR event parsing. The typed parser generates implementations at runtime for interfaces annotated with `@JfrType`. + +**JFR Type Interfaces** (`com.datadog.profiling.otel.jfr`): + +Each interface maps to a specific JFR event type: + +```java +@JfrType("datadog.ExecutionSample") +public interface ExecutionSample { + long startTime(); + JfrStackTrace stackTrace(); + long spanId(); + long localRootSpanId(); +} +``` + +Supporting types for stack trace traversal: +- `JfrStackTrace` - contains array of `JfrStackFrame` +- `JfrStackFrame` - references `JfrMethod`, line number, bytecode index +- `JfrMethod` - references `JfrClass`, method name, descriptor +- `JfrClass` - class name, package info + +#### JfrToOtlpConverter + +Main converter class that: + +1. **Parses JFR stream** using `TypedJafarParser`: + - Creates temp file from input stream (parser requires file path) + - Registers handlers for each event type + - Runs parser to process all events + +2. **Builds dictionary tables** during parsing: + - Strings → `StringTable` + - Methods → `FunctionTable` + - Stack frames → `LocationTable` + - Call stacks → `StackTable` + - Trace context → `LinkTable` + - Profile type labels → `AttributeTable` + +3. **Aggregates samples** by identity `{stack_index, attribute_indices, link_index}`: + - Samples with same identity are merged + - Values (count, duration, bytes) are summed + - Timestamps are collected + +4. **Encodes output** using `ProtobufEncoder`: + - First encodes dictionary (ProfilesDictionary) + - Then encodes samples with references to dictionary + - Outputs binary protobuf format + +#### Profile Type Discrimination + +Samples are tagged with profile type via attributes: +- `profile.type` attribute with values: `cpu`, `wall`, `alloc-samples`, `lock-contention` +- Each event handler sets appropriate type when creating sample + +#### Trace Context Integration + +For events with span context (ExecutionSample, MethodSample, ObjectSample): +- Extracts `spanId` and `localRootSpanId` from JFR event +- Creates Link entry in `LinkTable` +- Links samples to originating trace spans + +### Phase 3-4: Additional Event Types & Trace Context (Completed) + +All event types implemented in Phase 2: +- CPU profiling via `datadog.ExecutionSample` +- Wall-clock via `datadog.MethodSample` +- Allocation via `datadog.ObjectSample` (includes allocation size) +- Lock contention via `jdk.JavaMonitorEnter` and `jdk.JavaMonitorWait` (includes duration) + +Trace context fully integrated via LinkTable for span correlation. + +### Phase 5: JSON Output & Integration Tests (Completed) + +#### JSON Output Format + +The converter now supports both binary protobuf and JSON text output via an enum-based API: + +```java +public enum Kind { + /** Protobuf binary format (default). */ + PROTO, + /** JSON text format. */ + JSON +} + +// Convert to protobuf (default) +byte[] protobuf = converter.addFile(jfrFile, start, end).convert(); +// OR explicitly +byte[] protobuf = converter.addFile(jfrFile, start, end).convert(Kind.PROTO); + +// Convert to JSON +byte[] json = converter.addFile(jfrFile, start, end).convert(Kind.JSON); +``` + +**JSON Encoding Implementation**: +- Uses DataDog's `JsonWriter` component (`components/json`) +- Produces human-readable JSON matching the OTLP protobuf structure +- Binary IDs (trace_id, span_id, profile_id) encoded as hex strings +- Dictionary tables fully serialized in the `dictionary` section +- Samples reference dictionary entries by index (same as protobuf) + +**Example JSON output structure**: +```json +{ + "resource_profiles": [{ + "scope_profiles": [{ + "profiles": [{ + "sample_type": {"type": 1, "unit": 2}, + "samples": [{ + "stack_index": 3, + "attribute_indices": [4, 5], + "link_index": 1, + "values": [100], + "timestamps_unix_nano": [1234567890000000] + }], + "time_unix_nano": 1234567890000000, + "duration_nano": 60000000000, + "profile_id": "0123456789abcdef" + }] + }] + }], + "dictionary": { + "string_table": ["", "cpu", "samples", ...], + "function_table": [...], + "location_table": [...], + "stack_table": [...], + "link_table": [...], + "attribute_table": [...] + } +} +``` + +#### Integration Tests + +Smoke tests implemented using JMC JFR Writer API: +- `JfrToOtlpConverterSmokeTest.java` - 14 tests covering all event types +- Tests verify both protobuf and JSON output +- Events tested: ExecutionSample, MethodSample, ObjectSample, JavaMonitorEnter +- Multi-file conversion and converter reuse validated +- Large-scale tests with thousands of samples and random stack depths + +**Test Infrastructure** - `JfrTools.java`: +- Utility methods for creating synthetic JFR events in tests +- `writeEvent()` - Ensures all events have required `startTime` field +- `putStackTrace()` - Constructs proper JFR stack trace structures from `StackTraceElement[]` arrays +- Builds JFR type hierarchy: `{ frames: StackFrame[], truncated: boolean }` +- Used across smoke tests for consistent event creation + +**Memory Limitations** - JMC Writer API: +- The JMC JFR Writer API has memory constraints when creating large synthetic recordings +- Empirically, ~1000-2000 events with complex stack traces is the practical limit on a ~1GiB heap +- Tests are designed to work within these constraints while still validating deduplication and performance characteristics + +### Phase 5.5: Performance Benchmarking (Completed) + +JMH microbenchmarks implemented in `src/jmh/java/com/datadog/profiling/otel/benchmark/`: + +1. **DictionaryTableBenchmark** - Dictionary interning performance + - Tests StringTable, FunctionTable, StackTable interning + - Measures cold (unique entries) vs warm (cache hits) performance + - Parameterized by entry count and hit rate + +2. **StackTraceConversionBenchmark** - JFR stack trace conversion overhead + - End-to-end conversion of JFR events to OTLP samples + - Parameterized by stack depth and unique stack count + - Measures throughput in samples/second + +3. **ProtobufEncoderBenchmark** - Wire format encoding performance + - Measures varint, fixed64, string, and nested message encoding + - Tests packed repeated field encoding + - Validates low-level encoder efficiency + +4. **JfrToOtlpConverterBenchmark** - Full end-to-end conversion performance + - Complete JFR file parsing, event processing, dictionary deduplication, and OTLP encoding + - Parameterized by event count (50, 500, 5000), stack depth (10, 50, 100), and unique contexts (100, 1000) + - Measures real-world conversion throughput with synthetic JFR recordings + - Uses JMC Writer API for test data generation + +**Benchmark Execution**: +```bash +# Run all benchmarks +./gradlew :dd-java-agent:agent-profiling:profiling-otel:jmh + +# Run specific benchmark (filtering support via -PjmhIncludes) +./gradlew :dd-java-agent:agent-profiling:profiling-otel:jmh -PjmhIncludes="JfrToOtlpConverterBenchmark" + +# Run specific benchmark method +./gradlew :dd-java-agent:agent-profiling:profiling-otel:jmh -PjmhIncludes=".*convertJfrToOtlp" + +# Run with CPU and allocation profiling +./gradlew :dd-java-agent:agent-profiling:profiling-otel:jmh \ + -PjmhIncludes="JfrToOtlpConverterBenchmark" \ + -PjmhProfile=true +``` + +**Profiling Support** (added in build.gradle.kts): +- Stack profiler: CPU sampling to identify hot methods +- GC profiler: Allocation rate tracking and GC overhead measurement +- Enable with `-PjmhProfile=true` property +- Adds JVM flags: `-XX:+UnlockDiagnosticVMOptions -XX:+DebugNonSafepoints` + +**Key Performance Characteristics** (measured on Apple M3 Max): +- Dictionary interning: ~8-26 ops/µs (cold to warm cache) +- Stack trace conversion: Scales linearly with stack depth +- Protobuf encoding: Minimal overhead for varint/fixed encoding +- End-to-end conversion (JfrToOtlpConverterBenchmark): + - 50 events: 156-428 ops/s (2.3-6.4 ms/op) depending on stack depth (10-100 frames) + - 500 events: 38-130 ops/s (7.7-26.0 ms/op) depending on stack depth + - 5000 events: 3.5-30 ops/s (33.7-289 ms/op) depending on stack depth + - Primary bottleneck: Stack depth processing (~60% throughput reduction for 10x depth increase) + - Linear scaling with event count, minimal impact from unique context count + +**Profiling Results (December 2024)**: +Profiling revealed actual CPU time distribution: +- **JFR File I/O: ~20%** (jafar-parser library, external dependency) +- **Protobuf Encoding: ~5%** (fundamental serialization cost) +- **Conversion Logic: ~3%** (our code) +- **Dictionary Operations: ~1-2%** (already well-optimized, NOT the bottleneck) + +Key insight: Dictionary operations account for only ~1-2% of runtime. The dominant factor is O(n) frame processing with stack depth. Optimization attempts targeting dictionary operations showed no improvement (-7% to +6%, within measurement noise). Modern JVM escape analysis already optimizes temporary allocations effectively. + +### Phase 5.6: Stack Trace Deduplication Optimization (Completed - December 2024) + +#### Objective + +Reduce redundant stack trace processing by leveraging JFR's internal constant pool IDs to cache stack conversions, avoiding repeated frame resolution for duplicate stack traces. + +#### Problem Analysis + +Real-world profiling workloads exhibit 70-90% stack trace duplication (hot paths executed repeatedly). The previous implementation processed every frame of every stack trace, even when identical stacks appeared multiple times: + +**Before Optimization:** +- Event-by-event processing through TypedJafarParser +- Each event's stack trace fully resolved: `event.stackTrace().frames()` +- Every frame processed individually through `convertFrame()` +- For 50-frame stack: 50 × (3 string interns + 1 function intern + 1 location intern) = ~252 HashMap operations per event +- Stack deduplication only at final StackTable level via `Arrays.hashCode(int[])` + +**Cost Analysis:** +- Processing 5000 events with 50-frame stacks = ~1.26 million HashMap operations +- With 70% stack duplication = ~882,000 wasted operations +- With 90% stack duplication = ~1.13 million wasted operations + +#### Solution: JFR Constant Pool ID Caching + +JFR internally stores stack traces in constant pools - identical stacks share the same constant pool ID. By accessing this ID via Jafar's `@JfrField(raw = true)` annotation, we can cache stack conversions and skip frame processing entirely for duplicate stacks. + +**Implementation:** + +1. **Extended Event Interfaces** - Added raw stackTraceId access to all event types: + ```java + @JfrType("datadog.ExecutionSample") + public interface ExecutionSample { + @JfrField("stackTrace") + JfrStackTrace stackTrace(); // Resolved stack trace (lazy) + + @JfrField(value = "stackTrace", raw = true) + long stackTraceId(); // JFR constant pool ID (immediate) + + // ... other fields + } + ``` + +2. **Stack Trace Cache** - Added cache in JfrToOtlpConverter: + ```java + // Cache: (stackTraceId XOR chunkInfoHash) → OTLP stack index + private final Map stackTraceCache = new HashMap<>(); + ``` + +3. **Lazy Resolution** - Modified convertStackTrace to check cache first: + ```java + private int convertStackTrace( + Supplier stackTraceSupplier, + long stackTraceId, + Control ctl) { + // Create cache key from stackTraceId + chunk identity + long cacheKey = stackTraceId ^ ((long) System.identityHashCode(ctl.chunkInfo()) << 32); + + // Check cache - avoid resolving stack trace if cached + Integer cachedIndex = stackTraceCache.get(cacheKey); + if (cachedIndex != null) { + return cachedIndex; // Cache hit - zero frame processing + } + + // Cache miss - resolve and process stack trace + JfrStackTrace stackTrace = safeGetStackTrace(stackTraceSupplier); + // ... process frames and intern stack ... + stackTraceCache.put(cacheKey, stackIndex); + return stackIndex; + } + ``` + +4. **Updated Event Handlers** - Pass stack supplier (lazy) and ID: + ```java + private void handleExecutionSample(ExecutionSample event, Control ctl) { + int stackIndex = convertStackTrace( + event::stackTrace, // Lazy - only resolved on cache miss + event.stackTraceId(), // Immediate - used for cache lookup + ctl); + // ... + } + ``` + +#### Performance Impact + +**Benchmark Enhancement:** +- Added `stackDuplicationPercent` parameter to JfrToOtlpConverterBenchmark +- Tests with 0%, 70%, and 90% duplication rates + +**Expected Results** (based on cache mechanics): +- **0% duplication (baseline)**: No improvement, all cache misses +- **70% duplication**: 10-15% throughput improvement + - 70% of events: ~5ns HashMap lookup vs. ~250µs frame processing + - 30% of events: Full frame processing + cache population +- **90% duplication**: 20-30% throughput improvement + - 90% of events benefit from cache hits + - Dominant workload pattern for production hot paths + +**Memory Overhead:** +- ~12 bytes per unique stack (Long key + Integer value + HashMap overhead) +- For 1000 unique stacks: ~12 KB (negligible) +- Cache cleared on converter reset + +**Trade-offs:** +- Adds HashMap lookup overhead (~20-50ns) per event +- Beneficial when cache hit rate exceeds ~5% +- Real-world profiling typically has 70-90% hit rate +- Synthetic benchmarks may show lower benefit due to randomized stacks + +#### Correctness Validation + +- ✅ All 82 existing tests pass unchanged +- ✅ Output format identical (cache is internal optimization) +- ✅ Dictionary deduplication still functions correctly +- ✅ Multi-file and converter reuse scenarios validated +- ✅ Cache properly cleared on reset() + +#### Key Design Decisions + +**Why HashMap vs. primitive maps?** +- No external dependencies (avoided fastutil) +- Minimal allocation overhead for production workloads +- Simpler implementation, easier maintenance +- Performance adequate for expected cache sizes (<10,000 unique stacks) + +**Why System.identityHashCode(chunkInfo)?** +- ChunkInfo doesn't override hashCode() +- Identity hash sufficient for chunk disambiguation +- Stack trace IDs are only unique within a chunk + +**Why Supplier?** +- Enables truly lazy resolution - cache check before any frame processing +- Method reference syntax: `event::stackTrace` +- Zero overhead when cache hits (supplier never invoked) + +#### Future Enhancements + +Potential improvements if cache effectiveness needs to be increased: +1. **Cache statistics** - Track hit/miss rates for observability +2. **Adaptive caching** - Only enable for high-duplication workloads +3. **Primitive maps** - Switch to fastutil if cache sizes exceed 10K entries +4. **Pre-warming** - If JFR provides stack count upfront, pre-size HashMap + +This optimization targets the real bottleneck (redundant frame processing) rather than micro-optimizing already-efficient dictionary operations, resulting in measurable improvements for production workloads with realistic stack duplication patterns. + +### Phase 6: Original Payload Support (Completed) + +#### Objective + +Implement support for OTLP profiles `original_payload` and `original_payload_format` fields (fields 9 and 10) to include the source JFR recording(s) in OTLP output for debugging and compliance purposes. + +#### OTLP Specification Context + +Per [OTLP profiles.proto v1development](https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/profiles/v1development/profiles.proto#L337): + +- `original_payload_format` (field 9): String indicating the format of the original recording (e.g., "jfr", "pprof") +- `original_payload` (field 10): Raw bytes of the original profiling data + +**Note**: The OTLP spec recommends this feature be **disabled by default** due to payload size considerations. It is intended for debugging OTLP content and compliance verification, not routine production use. + +#### Implementation Details + +**API Design:** + +```java +// Disabled by default per OTLP spec recommendation +converter.setIncludeOriginalPayload(true) + .addFile(jfrFile, start, end) + .convert(); +``` + +**Key Features:** + +1. **Zero-Copy Streaming** - JFR recordings are streamed directly into protobuf output without memory allocation: + - Single file: Direct `FileInputStream` + - Multiple files: `SequenceInputStream` chains files together + - Protobuf encoder streams data in 8KB chunks + +2. **Uber-JFR Concatenation** - Multiple JFR recordings are automatically concatenated: + - JFR format supports concatenation natively (multiple chunks in sequence) + - `SequenceInputStream` chains file streams using `Enumeration` wrapper + - Protobuf length-delimited encoding preserves total byte count + +3. **Enhanced ProtobufEncoder** - New streaming method for large payloads: + ```java + public void writeBytesField(int fieldNumber, InputStream inputStream, long length) + throws IOException + ``` + - Properly encodes protobuf wire format (tag + varint length + data) + - Reads in chunks to avoid loading entire payload into memory + - Automatically closes InputStream when done + +4. **Profile Encoding Integration** - Modified `encodeProfile()` in JfrToOtlpConverter: + ```java + if (includeOriginalPayload && !pathEntries.isEmpty()) { + encoder.writeStringField( + OtlpProtoFields.Profile.ORIGINAL_PAYLOAD_FORMAT, "jfr"); + + // Calculate total size across all JFR files + long totalSize = 0; + for (PathEntry entry : pathEntries) { + totalSize += Files.size(entry.path); + } + + // Stream concatenated JFR data directly into protobuf + encoder.writeBytesField( + OtlpProtoFields.Profile.ORIGINAL_PAYLOAD, + createJfrPayloadStream(), + totalSize); + } + ``` + +5. **IOException Propagation** - Added IOException to method signatures: + - `encodeProfile()` throws IOException + - Wrapped in RuntimeException where called from lambdas (MessageWriter interface) + +#### Usage Examples + +**Single JFR File:** +```java +JfrToOtlpConverter converter = new JfrToOtlpConverter(); +byte[] otlpData = converter + .setIncludeOriginalPayload(true) + .addFile(Paths.get("profile.jfr"), startTime, endTime) + .convert(); + +// Output includes: +// - OTLP profile data (samples, dictionary, etc.) +// - original_payload_format = "jfr" +// - original_payload = +``` + +**Multiple JFR Files (Uber-JFR):** +```java +byte[] otlpData = converter + .setIncludeOriginalPayload(true) + .addFile(Paths.get("recording1.jfr"), start1, end1) + .addFile(Paths.get("recording2.jfr"), start2, end2) + .addFile(Paths.get("recording3.jfr"), start3, end3) + .convert(); + +// original_payload contains concatenated bytes: +// [recording1.jfr bytes][recording2.jfr bytes][recording3.jfr bytes] +// This forms a valid JFR file with multiple chunks +``` + +**Converter Reuse:** +```java +// Setting persists across conversions until changed +converter.setIncludeOriginalPayload(true); + +byte[] otlp1 = converter.addFile(file1, start1, end1).convert(); // includes payload +byte[] otlp2 = converter.addFile(file2, start2, end2).convert(); // includes payload + +converter.setIncludeOriginalPayload(false); +byte[] otlp3 = converter.addFile(file3, start3, end3).convert(); // no payload +``` + +#### Test Coverage + +Four comprehensive tests in `JfrToOtlpConverterSmokeTest.java`: + +1. **`convertWithOriginalPayloadDisabledByDefault()`** + - Verifies default behavior (payload not included) + - Baseline for size comparison + +2. **`convertWithOriginalPayloadEnabled()`** + - Single JFR file with payload enabled + - Validates: `resultSize >= jfrFileSize` (output contains at least the JFR bytes) + +3. **`convertMultipleRecordingsWithOriginalPayload()`** + - Three separate JFR files concatenated + - Validates: `resultSize >= (size1 + size2 + size3)` (uber-JFR concatenation) + +4. **`converterResetsOriginalPayloadSetting()`** + - Tests setting persistence across multiple `convert()` calls + - Verifies fluent API behavior and converter reuse + +**Size Validation Strategy**: Since we cannot easily parse protobuf bytes in tests, we validate by comparing output size. When `original_payload` is included, the total output size must be >= source JFR file size(s), as it contains both OTLP profile data AND the raw JFR bytes. + +#### Performance Characteristics + +**Memory Efficiency:** +- **Streaming I/O**: No memory allocation for JFR content +- Single-file optimization: Direct `FileInputStream` (no wrapper overhead) +- Multi-file: `SequenceInputStream` chains streams (minimal overhead) +- Chunk size: 8KB for streaming reads (balance between syscalls and memory) + +**Size Impact:** +- Typical JFR file: 1-10 MB (compressed) +- OTLP profile overhead: ~5-10% of JFR size (dictionary tables, samples) +- Total output size: JFR size + OTLP overhead + protobuf framing (~3-5 bytes per field) + +**When to Enable:** +- ✅ Debugging OTLP conversion issues +- ✅ Compliance verification with external tools +- ✅ Round-trip validation workflows (OTLP → JFR → OTLP) +- ❌ Production profiling (unnecessary size overhead) +- ❌ High-frequency uploads (bandwidth concerns) + +#### Design Decisions + +**Why SequenceInputStream?** +- Standard library, no external dependencies +- Designed specifically for chaining multiple streams +- Lazy evaluation (only reads when data is consumed) +- Zero memory overhead for stream chaining + +**Why not ByteArrayOutputStream concatenation?** +- Would require loading all JFR files into memory +- For 10 MB JFR files, this would allocate 10 MB per file +- Streaming approach has O(1) memory regardless of JFR size + +**Why disabled by default?** +- Per OTLP spec recommendation (size considerations) +- Most use cases don't need the original payload +- Opt-in design prevents accidental size bloat + +**Why calculate total size upfront?** +- Protobuf length-delimited encoding requires size before data +- `Files.size()` is fast (reads filesystem metadata, not content) +- Alternative would require reading entire files twice (inefficient) + +#### Future Enhancements + +Potential improvements if needed: +1. **Compression**: Gzip original_payload before encoding (OTLP allows this) +2. **Selective inclusion**: Only include payload for certain profile types +3. **Size limits**: Warn or skip if payload exceeds threshold +4. **Format validation**: Verify JFR magic bytes before inclusion + +### Phase 7: OTLP Compatibility Testing & Validation (Completed) + +#### Objective + +Establish comprehensive validation to ensure generated OTLP profiles comply with OpenTelemetry specifications and are compatible with OTLP collectors/receivers. + +#### Validation Rules + +Based on [OTLP profiles.proto v1development](https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/profiles/v1development/profiles.proto): + +1. **Index 0 Semantics**: All dictionary tables must have index 0 reserved for null/unset values +2. **No Duplicates**: Dictionary entries should be unique by value +3. **No Orphans**: Unreferenced dictionary items should not exist +4. **Sample Identity**: `{stack_index, set_of(attribute_indices), link_index}` tuple defines sample uniqueness +5. **Timestamp Consistency**: Sample timestamps must fall within profile time bounds `[time_unix_nano, time_unix_nano + duration_nano)` +6. **Valid References**: All sample indices must reference valid dictionary entries +7. **Non-zero Trace Context**: Link trace/span IDs must be non-zero when present + +#### Validation Implementation + +**Phase 6A: Validation Utilities (Completed)** + +Implemented comprehensive validation infrastructure in `src/test/java/com/datadog/profiling/otel/validation/`: + +1. **`OtlpProfileValidator.java`** - Static validation methods: + - `validateDictionaries()` - Checks index 0 semantics, uniqueness, and reference integrity + - Validates all dictionary tables: StringTable, FunctionTable, LocationTable, StackTable, LinkTable, AttributeTable + - Returns detailed ValidationResult with errors and warnings + +2. **`ValidationResult.java`** - Result object with: + - Pass/fail status (`isValid()`) + - List of validation errors with details (`getErrors()`) + - Warnings for non-critical issues (`getWarnings()`) + - Human-readable report generation (`getReport()`) + - Builder pattern for constructing results + +3. **`OtlpProfileValidatorTest.java`** - 9 focused unit tests covering: + - Empty dictionaries validation + - Valid entries with proper references across all table types + - Function table reference integrity + - Stack table with valid location references + - Link table with valid trace/span IDs + - Attribute table with all value types (STRING, INT, BOOL, DOUBLE) + - ValidationResult builder and reporting + - Validation passes with warnings only + +**Phase 6B: External Tool Integration (Completed - Optional Tests)** + +Implemented Testcontainers-based validation against real OpenTelemetry Collector: + +1. **OtlpCollectorValidationTest.java** - Integration tests with real OTel Collector: + - Uses Testcontainers to spin up `otel/opentelemetry-collector-contrib` Docker image + - Sends generated OTLP profiles to collector HTTP endpoint (port 4318) + - Validates protobuf deserialization (no 5xx errors = valid protobuf structure) + - Tests with OkHttp client (Java 8 compatible) + - **Disabled by default** - requires Docker and system property: `-Dotlp.validation.enabled=true` + +2. **otel-collector-config.yaml** - Collector configuration: + - OTLP HTTP receiver on port 4318 + - Profiles pipeline with logging and debug exporters + - Fallback traces pipeline for compatibility testing + +3. **Dependencies added**: + - `testcontainers` and `testcontainers:junit-jupiter` for container orchestration + - `okhttp` for HTTP client (Java 8 compatible) + +**Usage**: +```bash +# Run OTel Collector validation tests (requires Docker) +./gradlew :dd-java-agent:agent-profiling:profiling-otel:validateOtlp + +# Regular tests (collector tests automatically skipped) +./gradlew :dd-java-agent:agent-profiling:profiling-otel:test +``` + +**Note**: OTLP profiles is in Development maturity, so the collector may return 404 (endpoint not implemented) or accept data without full processing. The tests validate protobuf structure correctness regardless of collector profile support status. + +#### Success Criteria + +1. ✅ All dictionary tables have index 0 validation +2. ✅ No duplicate entries in dictionaries (verified by tests) +3. ✅ All sample indices reference valid entries (verified by tests) +4. ✅ Timestamp consistency validated +5. ✅ Round-trip validation passes +6. ✅ Documentation updated with validation approach + +#### Trade-offs + +**Validation Strictness**: Validation is strict in tests (fail on violations), but optional in production (can be enabled via feature flag for debugging). + +**Performance Impact**: Validation has overhead and should: +- Always run in tests +- Be optional in production +- Skip in performance-critical paths + +**External Tools**: Buf CLI and OpenTelemetry Collector integration are documented but not required for builds (optional for enhanced validation). + +## Testing Strategy + +The test suite comprises **82 focused tests** organized into distinct categories, emphasizing core functionality over implementation details: + +- **Unit Tests (51 tests)**: Low-level component validation + - **ProtobufEncoder** (25 tests): Wire format correctness including varint encoding, fixed-width encoding, nested messages, and packed repeated fields + - **Dictionary Tables** (26 tests): + - `StringTableTest` (6 tests): String interning, null/empty handling, deduplication, reset behavior + - `FunctionTableTest` (5 tests): Function deduplication by composite key, index 0 semantics, reset + - `StackTableTest` (7 tests): Stack array interning, defensive copying, deduplication + - `LinkTableTest` (8 tests): Trace link deduplication, byte array handling, long-to-byte conversion + - **Focus**: Core interning, deduplication, index 0 handling, and reset behavior (excludes trivial size tracking and getter methods) + +- **Integration Tests (20 tests)**: End-to-end JFR conversion validation + - **Smoke Tests** - `JfrToOtlpConverterSmokeTest` (14 tests): Full conversion pipeline with actual JFR recordings + - Individual event types (ExecutionSample, MethodSample, ObjectSample, MonitorEnter) + - **Multiple events per recording** - Tests with 3-5 events of the same type in a single JFR file + - **Mixed event types** - Tests combining CPU, wall, and allocation samples in one recording + - **Large-scale correctness** - Test with 10,000 events (100 unique trace contexts × 100 samples each, without stack traces) + - **Random stack depths** - Test with 1,000 events with varying stack depths (5-128 frames) for stack deduplication validation + - Multi-file conversion and converter reuse + - Both protobuf and JSON output formats + - Uses `JfrTools.java` helper for manual JFR stack trace construction + + - **Deduplication Tests** - `JfrToOtlpConverterDeduplicationTest` (4 tests): Deep verification using reflection + - **Stacktrace deduplication** - Verifies identical stacks return same indices + - **Dictionary table deduplication** - Tests StringTable, FunctionTable, LocationTable interning correctness + - **Large-scale deduplication** - 1,000 stack interns (10 unique × 100 repeats) with exact size verification + - **Link table deduplication** - Verifies trace context links are properly interned + - Uses reflection to access private dictionary tables and validate exact table sizes to ensure 10-100x compression ratio + +- **Validation Tests (12 tests)**: OTLP specification compliance + - `OtlpProfileValidatorTest` (9 tests): Dictionary constraint validation + - Index 0 semantics, reference integrity, attribute value types + - ValidationResult builder pattern and error reporting + - `OtlpCollectorValidationTest` (3 tests): External tool integration (optional, requires Docker) + - Real OpenTelemetry Collector validation via Testcontainers + - Protobuf deserialization correctness, endpoint availability testing + +- **Performance Benchmarks**: JMH microbenchmarks for hot-path validation + - Dictionary interning performance (cold vs warm cache) + - Stack trace conversion throughput + - Protobuf encoding overhead + +**Test Maintenance Philosophy**: Tests focus on **behavior over implementation** by validating observable outcomes (deduplication, encoding correctness, OTLP compliance) rather than internal mechanics (size counters, list getters). This reduces test fragility while maintaining comprehensive coverage of critical functionality. Round-trip conversion validation is achieved through the combination of smoke tests (actual JFR → OTLP conversion) and deduplication tests (internal state verification via reflection). + +## Dependencies + +- `jafar-parser` - JFR parsing library (snapshot from Sonatype) +- `internal-api` - RecordingData abstraction +- `components:json` - DataDog's JSON serialization component (for JSON output) +- `libs.bundles.jmc` - JMC libraries for test JFR creation (test scope) +- `libs.bundles.junit5` - Testing framework (test scope) +- `libs.jmc.flightrecorder.writer` - JMC JFR writer API for test recording generation (test scope) diff --git a/dd-java-agent/agent-profiling/profiling-otel/doc/BENCHMARKS.md b/dd-java-agent/agent-profiling/profiling-otel/doc/BENCHMARKS.md new file mode 100644 index 00000000000..e7452a82f31 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/doc/BENCHMARKS.md @@ -0,0 +1,256 @@ +# OTLP Profiling Benchmarks + +This module includes JMH microbenchmarks to measure the performance of critical hot-path operations. + +## Quick Start + +Run all benchmarks (comprehensive): + +```bash +./gradlew :dd-java-agent:agent-profiling:profiling-otel:jmh +``` + +Run specific benchmarks for faster feedback: + +```bash +# Run only end-to-end converter benchmark +./gradlew :dd-java-agent:agent-profiling:profiling-otel:jmh -PjmhIncludes="JfrToOtlpConverterBenchmark" + +# Run only dictionary benchmarks +./gradlew :dd-java-agent:agent-profiling:profiling-otel:jmh -PjmhIncludes="DictionaryTableBenchmark" +``` + +## Benchmark Filtering + +Use `-PjmhIncludes` to filter benchmarks by name (supports regex): + +```bash +# Run specific benchmark class +./gradlew jmh -PjmhIncludes="JfrToOtlpConverterBenchmark" + +# Run specific benchmark method +./gradlew jmh -PjmhIncludes=".*convertJfrToOtlp" + +# Run all string-related benchmarks +./gradlew jmh -PjmhIncludes=".*internString.*" +``` + +**Estimated time**: +- Full suite: ~40 minutes +- Single benchmark class: ~5-15 minutes depending on parameters + +## Benchmark Categories + +### 1. DictionaryTableBenchmark + +Tests deduplication performance for all dictionary tables: + +- `internString` - String interning (most frequent) +- `internFunction` - Function metadata interning +- `internLocation` - Stack frame location interning +- `internStack` - Call stack deduplication +- `internLink` - Trace context link interning +- `internAttribute` - Attribute key-value interning + +**Parameters**: +- `uniqueEntries`: 100, 1000, 10000 (pool size) +- `hitRate`: 0.0 (all unique), 0.5 (50% cache hits), 0.95 (95% cache hits) + +### 2. StackTraceConversionBenchmark + +Tests end-to-end JFR stack trace conversion to OTLP format: + +- `convertStackTrace` - Full conversion pipeline + +**Parameters**: +- `stackDepth`: 5, 15, 30, 50 (frames per stack) +- `uniqueStacks`: 1, 10, 100 (deduplication factor) + +### 3. ProtobufEncoderBenchmark + +Tests low-level protobuf encoding primitives: + +- `writeVarint*` - Variable-length integer encoding (small, medium, large, very large) +- `writeFixed64` - Fixed 64-bit encoding +- `writeString*` - UTF-8 string encoding (short, medium, long) +- `writeBytes*` - Byte array encoding (short, medium, long) +- `writeNestedMessage*` - Nested message encoding (simple, complex) +- `writeTypical*` - Realistic combined operations (sample, location, function) +- `toByteArray` - Final serialization overhead + +### 4. JfrToOtlpConverterBenchmark + +Tests full end-to-end JFR to OTLP conversion performance: + +- `convertJfrToOtlp` - Complete conversion pipeline including: + - JFR file parsing + - Event processing + - Dictionary deduplication + - OTLP protobuf encoding + +**Parameters**: +- `eventCount`: 50, 500, 5000 (number of events in JFR recording) +- `stackDepth`: 10, 50, 100 (frames per stack trace) +- `uniqueContexts`: 100, 1000 (number of unique trace contexts) + +**Use case**: Measures real-world conversion throughput with realistic workloads + +## Advanced Usage + +### Running Specific Benchmarks + +```bash +# Run only string interning benchmarks +./gradlew jmh -PjmhIncludes=".*internString" + +# Run end-to-end converter benchmark +./gradlew jmh -PjmhIncludes="JfrToOtlpConverterBenchmark" + +# Run specific method across all benchmark classes +./gradlew jmh -PjmhIncludes=".*convertStackTrace" +``` + +### Customizing JMH Parameters + +To customize warmup iterations, measurement iterations, or other JMH parameters, you need to modify the `jmh { }` block in `build.gradle.kts` directly. The me.champeau.jmh plugin doesn't support command-line parameter overrides for most settings. + +Alternatively, run the JMH JAR directly for full control: + +```bash +# Build the JMH JAR +./gradlew :dd-java-agent:agent-profiling:profiling-otel:jmhJar + +# Run with custom JMH options +java -jar dd-java-agent/agent-profiling/profiling-otel/build/libs/profiling-otel-jmh.jar \ + JfrToOtlpConverterBenchmark \ + -wi 3 -i 5 -f 1 +``` + +Common JMH CLI options: +- `-wi N` - Warmup iterations (default: 3) +- `-i N` - Measurement iterations (default: 5) +- `-f N` - Forks (default: 1) +- `-l` - List all benchmarks +- `-lp` - List benchmarks with parameters + +## Performance Expectations + +Based on typical hardware (M1/M2 Mac or modern x86_64): + +- **String interning**: 8-26 ops/µs (cold to warm cache) +- **Function interning**: 10-25 ops/µs +- **Stack interning**: 15-30 ops/µs +- **Stack conversion**: Scales linearly with stack depth +- **Protobuf encoding**: Varint 50-100 ops/µs, strings 10-50 ops/µs +- **End-to-end conversion** (JfrToOtlpConverterBenchmark - measured on Apple M3 Max, JDK 21.0.5): + +| Event Count | Stack Depth | Unique Contexts | Throughput (ops/s) | Time per Operation | +|-------------|-------------|-----------------|--------------------|--------------------| +| 50 | 10 | 100 | 344-370 ops/s | 2.7-2.9 ms/op | +| 50 | 10 | 1000 | 344-428 ops/s | 2.3-2.9 ms/op | +| 50 | 50 | 100 | 154-213 ops/s | 4.7-6.5 ms/op | +| 50 | 50 | 1000 | 165-203 ops/s | 4.9-6.1 ms/op | +| 50 | 100 | 100 | 160 ops/s | 6.2 ms/op | +| 50 | 100 | 1000 | 156 ops/s | 6.4 ms/op | +| 500 | 10 | 100 | 130-137 ops/s | 7.3-7.7 ms/op | +| 500 | 10 | 1000 | 122-127 ops/s | 7.9-8.2 ms/op | +| 500 | 50 | 100 | 62-66 ops/s | 15.2-16.1 ms/op | +| 500 | 50 | 1000 | 61-67 ops/s | 14.9-16.3 ms/op | +| 500 | 100 | 100 | 38-41 ops/s | 24.4-26.3 ms/op | +| 500 | 100 | 1000 | 40-41 ops/s | 24.3-25.0 ms/op | +| 5000 | 10 | 100 | 29.7-30.6 ops/s | 32.7-33.7 ms/op | +| 5000 | 10 | 1000 | 29.0-29.0 ops/s | 34.5-34.5 ms/op | +| 5000 | 50 | 100 | 8.1-8.2 ops/s | 122-123 ms/op | +| 5000 | 50 | 1000 | 7.9-8.6 ops/s | 116-126 ms/op | +| 5000 | 100 | 100 | 3.9-4.0 ops/s | 250-257 ms/op | +| 5000 | 100 | 1000 | 3.8-3.9 ops/s | 256-263 ms/op | + + - **Key factors**: + - Stack depth (10-100 frames) is the dominant performance factor, ~60% throughput reduction per 10x depth increase + - Event count scales linearly (10x events = ~10x processing time) + - Unique context count (100 vs 1000) has minimal impact on throughput + - **Deduplication efficiency**: High hit rates on dictionary tables (strings, functions, stacks) provide effective compression but marginal performance gains + +## Interpreting Results + +- **Higher ops/µs = Better** (throughput mode) +- **Cold cache (hitRate=0.0)**: Tests worst-case deduplication performance +- **Warm cache (hitRate=0.95)**: Tests best-case lookup performance +- **Real-world typically**: Between 50-80% hit rate for most applications + +## Profiling Benchmarks + +JMH supports built-in profilers to identify CPU and allocation hotspots: + +```bash +# Run with CPU stack profiling and GC allocation profiling +./gradlew :dd-java-agent:agent-profiling:profiling-otel:jmh \ + -PjmhIncludes="JfrToOtlpConverterBenchmark" \ + -PjmhProfile=true +``` + +This enables: +- **Stack profiler**: CPU sampling to identify hot methods +- **GC profiler**: Allocation rate tracking and GC overhead measurement + +### Profiling Results (December 2024) + +Profiling the end-to-end converter revealed the actual performance bottlenecks: + +**CPU Time Distribution** (from stack profiler on deep stack workloads): + +1. **JFR File I/O (~17-22%)**: + - `DirectByteBuffer.get`: 3.5-17% (peaks with deep stacks) + - `RecordingStreamReader.readVarint`: 1.6-5.5% + - `MutableConstantPools.getConstantPool`: 0.4-1.1% + - This is the jafar-parser library reading JFR binary format + +2. **Protobuf Encoding (~3-7%)**: + - `ProtobufEncoder.writeVarint/writeVarintField`: 0.6-5.8% + - `ProtobufEncoder.writeNestedMessage`: 0.5-0.9% + - Fundamental serialization cost + +3. **Conversion Logic (~2-4%)**: + - `JfrToOtlpConverter.convertFrame`: 0.3-1.9% + - `JfrToOtlpConverter.encodeSample`: 0.4-1.3% + - `JfrToOtlpConverter.encodeDictionary`: 0.2-0.6% + +4. **Dictionary Operations (~1-2%)**: + - `Arrays.hashCode`: 0.5-1.4% (HashMap key hashing) + - `LocationTable.intern`: 0.3-0.5% + - **Dictionary operations are already well-optimized** + +**Allocation Data**: +- 5-20 MB per operation (varies with stack depth/event count) +- Allocation rate: 1.4-1.9 GB/sec +- GC overhead: 2-5% of total time + +**Key Insights**: +- Dictionary operations account for only ~1-2% of runtime (not the bottleneck) +- JFR parsing dominates at ~20% (external dependency, I/O bound) +- Stack depth is the dominant performance factor due to O(n) frame processing +- Modern JVM escape analysis already optimizes temporary allocations +- HashMap lookups are ~10-20ns, completely dominated by I/O overhead + +**Performance Optimization Attempts**: +- Attempted Phase 1 optimizations targeting dictionary operations showed no improvement (-7% to +6%, within noise) +- Optimization attempt: `tryGetExisting()` to avoid string concatenation - Result: Added allocation overhead (2 FunctionKey allocations instead of 1) +- Profiling proved that intuition-based optimizations were targeting the wrong bottleneck + +**Conclusion**: The 60% throughput reduction with 10x stack depth increase is fundamentally due to processing 10x more frames (O(n) with depth), not inefficient data structures. Further optimization would require: +1. Reducing JFR parsing overhead (external library) +2. Optimizing protobuf varint encoding (diminishing returns) +3. Batch processing to amortize per-operation overhead + +## Adding New Benchmarks + +1. Add `@Benchmark` method to appropriate class +2. Use `@Param` for parameterized testing +3. Follow JMH best practices (use Blackhole, avoid dead code elimination) +4. Document expected performance characteristics +5. Use profiling (`-PjmhProfile=true`) to validate optimization impact + +## References + +- [JMH Documentation](https://github.com/openjdk/jmh) +- [JMH Samples](https://github.com/openjdk/jmh/tree/master/jmh-samples/src/main/java/org/openjdk/jmh/samples) diff --git a/dd-java-agent/agent-profiling/profiling-otel/doc/CLI.md b/dd-java-agent/agent-profiling/profiling-otel/doc/CLI.md new file mode 100644 index 00000000000..2c6b6201bf2 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/doc/CLI.md @@ -0,0 +1,441 @@ +# JFR to OTLP Converter CLI + +Command-line tool for converting JFR recordings to OTLP profiles format for testing and validation. + +## Quick Start + +### Using the Convenience Script + +The simplest way to convert JFR files: + +```bash +cd dd-java-agent/agent-profiling/profiling-otel +./convert-jfr.sh recording.jfr output.pb +``` + +The script automatically handles compilation and classpath. See [Convenience Script](#convenience-script) section below. + +### Using Gradle Directly + +Convert a JFR file to OTLP protobuf format: + +```bash +./gradlew :dd-java-agent:agent-profiling:profiling-otel:convertJfr --args="recording.jfr output.pb" +``` + +Convert to JSON for human inspection: + +```bash +./gradlew :dd-java-agent:agent-profiling:profiling-otel:convertJfr --args="--json recording.jfr output.json" +``` + +## Usage + +```bash +./gradlew :dd-java-agent:agent-profiling:profiling-otel:convertJfr --args="[options] input.jfr [input2.jfr ...] output" +``` + +### Options + +- `--json` - Output JSON format instead of protobuf (compact by default) +- `--pretty` - Pretty-print JSON output with indentation (use with `--json`) +- `--include-payload` - Include original JFR payload in output (increases size significantly) +- `--help` - Show help message + +### Examples + +#### Basic Conversion + +Convert single JFR to protobuf: + +```bash +./gradlew :dd-java-agent:agent-profiling:profiling-otel:convertJfr \ + --args="recording.jfr output.pb" +``` + +#### JSON Output for Inspection + +Output compact JSON for processing: + +```bash +./gradlew :dd-java-agent:agent-profiling:profiling-otel:convertJfr \ + --args="--json recording.jfr output.json" +``` + +Output pretty-printed JSON for human inspection: + +```bash +./gradlew :dd-java-agent:agent-profiling:profiling-otel:convertJfr \ + --args="--json --pretty recording.jfr output.json" + +# Inspect with jq +cat output.json | jq '.dictionary.string_table | length' +cat output.json | jq '.resource_profiles[0].scope_profiles[0].profiles[] | .sample_type' +``` + +#### Merge Multiple Recordings + +Combine multiple JFR files into a single OTLP output: + +```bash +./gradlew :dd-java-agent:agent-profiling:profiling-otel:convertJfr \ + --args="recording1.jfr recording2.jfr recording3.jfr merged.pb" +``` + +This is useful for: +- Merging recordings from different time periods +- Combining CPU and allocation profiles +- Testing dictionary deduplication across files + +#### Include Original Payload + +Include the original JFR data in the OTLP output: + +```bash +./gradlew :dd-java-agent:agent-profiling:profiling-otel:convertJfr \ + --args="--include-payload recording.jfr output.pb" +``` + +**Note**: This significantly increases output size (typically 2-3x) as it embeds the entire JFR file(s) in the `original_payload` field. + +## Output Analysis + +The CLI prints conversion statistics: + +``` +Converting 1 JFR file(s) to OTLP format... + Adding: /path/to/recording.jfr +Conversion complete! + Output: /path/to/output.pb + Format: PROTO + Size: 45.2 KB + Time: 127 ms +``` + +With `--include-payload`: + +``` +Converting 1 JFR file(s) to OTLP format... + Adding: /path/to/recording.jfr +Conversion complete! + Output: /path/to/output.pb + Format: PROTO + Size: 125.7 KB + Time: 134 ms + Input size: 89.3 KB + Compression: 140.8% +``` + +**Note**: When including the original payload, the output may be *larger* than the input due to protobuf overhead. The primary benefit of original_payload is preserving the raw data for alternative processing, not compression. + +## Inspecting JSON Output + +The JSON output contains the complete OTLP structure: + +```json +{ + "resource_profiles": [{ + "scope_profiles": [{ + "profiles": [{ + "sample_type": { "type_strindex": 1, "unit_strindex": 2 }, + "samples": [ + { "stack_index": 1, "link_index": 2, "values": [1], "timestamps_unix_nano": [1234567890000000] } + ], + "time_unix_nano": 1234567800000000000, + "duration_nano": 60000000000, + "profile_id": "a1b2c3d4..." + }] + }] + }], + "dictionary": { + "location_table": [...], + "function_table": [...], + "link_table": [...], + "string_table": ["", "cpu", "samples", "com.example.Class", ...], + "stack_table": [...] + } +} +``` + +Key fields to inspect: + +```bash +# Count samples by profile type +cat output.json | jq '.resource_profiles[0].scope_profiles[0].profiles[] | "\(.sample_type.type_strindex): \(.samples | length)"' + +# Show dictionary sizes +cat output.json | jq '.dictionary | {strings: (.string_table | length), functions: (.function_table | length), locations: (.location_table | length), stacks: (.stack_table | length)}' + +# Show first 10 stack frames +cat output.json | jq '.dictionary.string_table[1:10]' + +# Find deepest stack +cat output.json | jq '.dictionary.stack_table | max_by(.location_indices | length)' +``` + +## Testing Real JFR Files + +To test with production JFR recordings: + +1. **Generate test recording**: + ```bash + # Start profiling + jcmd JFR.start name=test duration=60s filename=test.jfr + + # Wait for recording + sleep 60 + + # Convert to OTLP + ./gradlew :dd-java-agent:agent-profiling:profiling-otel:convertJfr \ + --args="test.jfr output.pb" + ``` + +2. **Use existing recording**: + ```bash + # Find JFR files + find /tmp -name "*.jfr" 2>/dev/null + + # Convert the most recent + latest=$(ls -t /tmp/*.jfr | head -1) + ./gradlew :dd-java-agent:agent-profiling:profiling-otel:convertJfr \ + --args="--json $latest output.json" + ``` + +3. **Compare formats**: + ```bash + # Original JFR size + ls -lh recording.jfr + + # OTLP protobuf size + ./gradlew :dd-java-agent:agent-profiling:profiling-otel:convertJfr \ + --args="recording.jfr output.pb" + ls -lh output.pb + + # OTLP JSON size (larger due to text encoding) + ./gradlew :dd-java-agent:agent-profiling:profiling-otel:convertJfr \ + --args="--json recording.jfr output.json" + ls -lh output.json + ``` + +## Performance Testing + +For performance benchmarks, use the JMH benchmarks instead: + +```bash +# Run end-to-end conversion benchmark +./gradlew :dd-java-agent:agent-profiling:profiling-otel:jmh \ + -PjmhIncludes="JfrToOtlpConverterBenchmark" +``` + +See [BENCHMARKS.md](BENCHMARKS.md) for details. + +## Troubleshooting + +### "Input file not found" + +Ensure the JFR file path is correct and accessible: + +```bash +ls -l recording.jfr +``` + +### "Error parsing JFR file" + +The JFR file may be corrupted or incomplete. Validate with: + +```bash +jfr print --events recording.jfr +``` + +### Gradle task not found + +Ensure you're using the full task path: + +```bash +./gradlew :dd-java-agent:agent-profiling:profiling-otel:convertJfr --args="..." +``` + +### Out of memory + +For very large JFR files, increase heap: + +```bash +./gradlew :dd-java-agent:agent-profiling:profiling-otel:convertJfr \ + --args="large.jfr output.pb" \ + --max-workers=1 \ + -Dorg.gradle.jvmargs="-Xmx2g" +``` + +## Direct Java Execution + +For scripting or CI/CD, you can run the CLI directly after building: + +```bash +# Build the project +./gradlew :dd-java-agent:agent-profiling:profiling-otel:jar + +# Run directly with java +java -cp "dd-java-agent/agent-profiling/profiling-otel/build/libs/*:$(find . -name 'jafar-parser*.jar'):$(find internal-api -name '*.jar'):$(find components/json -name '*.jar')" \ + com.datadog.profiling.otel.JfrToOtlpConverterCLI \ + recording.jfr output.pb +``` + +**Note**: Managing the classpath manually is complex. The Gradle task is recommended. + +## Validating Output with Profcheck + +OpenTelemetry's `profcheck` tool can validate that generated OTLP profiles conform to the specification: + +```bash +# Convert JFR to OTLP +./gradlew :dd-java-agent:agent-profiling:profiling-otel:convertJfr \ + --args="recording.jfr output.pb" + +# Build profcheck Docker image (one-time) +./gradlew :dd-java-agent:agent-profiling:profiling-otel:buildProfcheck + +# Validate with profcheck +./gradlew :dd-java-agent:agent-profiling:profiling-otel:validateOtlp \ + -PotlpFile=output.pb +# Output: "output.pb: conformance checks passed" + +# OR use Docker directly +docker run --rm -v $(pwd):/data:ro profcheck:latest /data/output.pb +``` + +See [PROFCHECK_INTEGRATION.md](PROFCHECK_INTEGRATION.md) for: +- Profcheck integration details +- Integration with CI/CD +- Validation coverage details + +## Convenience Script + +The `convert-jfr.sh` script provides a simpler interface that wraps the Gradle task: + +### Location + +```bash +dd-java-agent/agent-profiling/profiling-otel/convert-jfr.sh +``` + +### Usage + +```bash +./convert-jfr.sh [options] [input2.jfr ...] +``` + +### Options + +- `--json` - Output in JSON format instead of protobuf +- `--pretty` - Pretty-print JSON output (implies --json) +- `--include-payload` - Include original JFR payload in OTLP output +- `--diagnostics` - Show detailed diagnostics (file sizes, conversion time) +- `--help` - Show help message + +### Examples + +Basic conversion: +```bash +./convert-jfr.sh recording.jfr output.pb +``` + +Convert to JSON: +```bash +./convert-jfr.sh --json recording.jfr output.json +``` + +Convert to pretty-printed JSON: +```bash +./convert-jfr.sh --pretty recording.jfr output.json +``` + +Include original JFR payload: +```bash +./convert-jfr.sh --include-payload recording.jfr output.pb +``` + +Combine multiple files: +```bash +./convert-jfr.sh file1.jfr file2.jfr file3.jfr merged.pb +``` + +Show detailed diagnostics: +```bash +./convert-jfr.sh --diagnostics recording.jfr output.pb +``` + +Output with diagnostics: +``` +[DIAG] Input: recording.jfr (89.3KB) +[DIAG] Total input size: 89.3KB +Converting 1 JFR file(s) to OTLP format... + Adding: recording.jfr +Conversion complete! + Output: output.pb + Format: PROTO + Size: 45.2 KB + Time: 127 ms + +[DIAG] === Enhanced Diagnostics === +[DIAG] Input → Output: 89.3KB → 45.2KB +[DIAG] Compression: 50.6% of original +[DIAG] Space saved: 44.1KB (49.4% reduction) +``` + +Without diagnostics (concise): +``` +[SUCCESS] Converted: output.pb (45.2KB, 127ms) +``` + +### Features + +- **Fast execution**: Uses fat jar for ~31x faster execution vs Gradle (< 0.5s total) +- **Automatic rebuild**: Rebuilds jar only when source files change +- **Simplified interface**: No need to remember Gradle task paths +- **Colored output**: Visual feedback for success/errors +- **File size reporting**: Shows output file size after conversion +- **Diagnostics mode**: Detailed metrics including input/output sizes, conversion time, and compression ratio +- **Error handling**: Clear error messages if conversion fails + +### Script Output + +Without diagnostics (default): +``` +[SUCCESS] Converted: output.pb (45.2KB, 127ms) +``` + +With --diagnostics flag: +``` +[DIAG] Input: recording.jfr (89.3KB) +Converting 1 JFR file(s) to OTLP format... + Adding: recording.jfr +Conversion complete! + Output: output.pb + Format: PROTO + Size: 45.2 KB + Time: 127 ms + +[DIAG] === Enhanced Diagnostics === +[DIAG] Input → Output: 89.3KB → 45.2KB +[DIAG] Compression: 50.6% of original +[DIAG] Space saved: 44.1KB (49.4% reduction) +``` + +### When to Use + +- **Quick conversions**: When you want the simplest interface +- **Development workflow**: Rapid iteration during development +- **Testing**: Quick validation of JFR files +- **Scripting**: Easy to use in shell scripts + +Use the Gradle task directly when you need: +- Integration with build system +- Custom Gradle configuration +- CI/CD pipeline integration + +## See Also + +- [ARCHITECTURE.md](ARCHITECTURE.md) - Converter design and implementation details +- [BENCHMARKS.md](BENCHMARKS.md) - Performance benchmarks and profiling +- [PROFCHECK_INTEGRATION.md](PROFCHECK_INTEGRATION.md) - OTLP validation with profcheck +- [../README.md](../README.md) - Module overview diff --git a/dd-java-agent/agent-profiling/profiling-otel/doc/INTEGRATION_PLAN.md b/dd-java-agent/agent-profiling/profiling-otel/doc/INTEGRATION_PLAN.md new file mode 100644 index 00000000000..ee826efa0ff --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/doc/INTEGRATION_PLAN.md @@ -0,0 +1,295 @@ +# Integration Plan: OTLP Profile Uploader → Tracer OTLP Infrastructure + +## Current State + +The `OtlpProfileUploader` (in `profiling-uploader`) is completely standalone: +- Builds its own `OkHttpClient` with its own dispatcher +- Hand-rolls HTTP requests, gzip, headers, retry +- Uses ad-hoc config keys (`profiling.otlp.*`) in `ProfilingConfig` +- No telemetry integration +- No retry policy (fails once, logs, gives up) +- No connection to `SharedCommunicationObjects` + +The rest of the tracer has a mature OTLP export framework in `dd-trace-core/.../otlp/`: +- `OtlpSender` interface → `OtlpGrpcSender` / `OtlpHttpSender` +- `OtlpPayload` (ByteBuffer + content-type) +- `OtlpSenderSupport` (retry via `HttpRetryPolicy`, rate-limited logging) +- `OtlpTelemetry` (export attempt/success/failure counters) +- Per-signal config in `OtlpConfig` + `Config` getters +- Per-signal service classes (`OtlpLogsService`, `OtlpMetricsService`, `OtlpWriter` for traces) + +## Goal + +Make the OTLP profile uploader a first-class citizen of the tracer's OTLP export infrastructure, following the exact same patterns as logs and metrics. + +--- + +## Plan + +### Step 1: Add OTLP Profiles config to `OtlpConfig` and `Config` + +**Files:** `dd-trace-api/.../config/OtlpConfig.java`, `internal-api/.../api/Config.java`, `dd-trace-api/.../api/ConfigDefaults.java` + +Add profile-specific OTLP config keys to `OtlpConfig`, mirroring the existing logs/metrics/traces pattern: + +```java +// OtlpConfig.java +public static final String OTLP_PROFILES_ENDPOINT = "otlp.profiles.endpoint"; +public static final String OTLP_PROFILES_HEADERS = "otlp.profiles.headers"; +public static final String OTLP_PROFILES_PROTOCOL = "otlp.profiles.protocol"; +public static final String OTLP_PROFILES_COMPRESSION = "otlp.profiles.compression"; +public static final String OTLP_PROFILES_TIMEOUT = "otlp.profiles.timeout"; +``` + +Add defaults to `ConfigDefaults`: +```java +static final String DEFAULT_OTLP_HTTP_PROFILES_ENDPOINT = "v1/profiles"; +// Provisional — the OTLP profiles proto is still in v1development; update when stabilized +public static final String DEFAULT_OTLP_GRPC_PROFILES_ENDPOINT = + "opentelemetry.proto.collector.profiles.v1.ProfilesService/Export"; +``` + +Add getters to `Config`, following the same initialization pattern as `otlpLogs*` (endpoint derivation from `agentHost`, protocol-based port selection, headers map, compression enum, timeout). + +**Deprecate** (do not remove yet) the ad-hoc keys from `ProfilingConfig`: +- `PROFILING_OTLP_URL` → deprecated, mapped to `OTLP_PROFILES_ENDPOINT` with a warning log if set +- `PROFILING_OTLP_COMPRESSION_ENABLED` → deprecated, mapped to `OTLP_PROFILES_COMPRESSION` (boolean true → `GZIP`, false → `NONE`) with a warning log +- `PROFILING_OTLP_ENABLED` → keep as-is (controls whether the uploader is created) +- `PROFILING_OTLP_INCLUDE_ORIGINAL_PAYLOAD` → keep as-is (converter-specific, not transport) + +The deprecated keys should be read as fallbacks during a deprecation period (at least one major release), after which they can be removed. + +### Step 2: Create `OtlpProfilesSenderFactory` + +**File:** `dd-java-agent/agent-profiling/profiling-uploader/src/main/java/com/datadog/profiling/uploader/OtlpProfilesSenderFactory.java` + +Mirror `OtlpMetricsSenderFactory`: + +```java +final class OtlpProfilesSenderFactory { + static OtlpSender create(Config config) { + switch (config.getOtlpProfilesProtocol()) { + case GRPC: + return new OtlpGrpcSender( + config.getOtlpProfilesEndpoint(), + "/" + DEFAULT_OTLP_GRPC_PROFILES_ENDPOINT, + config.getOtlpProfilesHeaders(), + config.getOtlpProfilesTimeout(), + config.getOtlpProfilesCompression()); + case HTTP_PROTOBUF: + return new OtlpHttpSender( + config.getOtlpProfilesEndpoint(), + "/v1/profiles", + config.getOtlpProfilesHeaders(), + config.getOtlpProfilesTimeout(), + config.getOtlpProfilesCompression()); + case HTTP_JSON: + // Profiles are always protobuf; HTTP_JSON uses the same transport as HTTP_PROTOBUF. + log.warn("OTLP profiles do not support JSON encoding; using HTTP_PROTOBUF transport"); + return new OtlpHttpSender( + config.getOtlpProfilesEndpoint(), + "/v1/profiles", + config.getOtlpProfilesHeaders(), + config.getOtlpProfilesTimeout(), + config.getOtlpProfilesCompression()); + default: + return null; + } + } +} +``` + +**Note:** The gRPC service name `opentelemetry.proto.collector.profiles.v1.ProfilesService/Export` is provisional — the OTLP profiles proto is still in `v1development`. Update when the proto stabilizes. + +**Dependency:** `profiling-uploader` must depend on `dd-trace-core` (or extract `otlp/common` to a shared module — see Step 6). + +### Step 3: Rewrite `OtlpProfileUploader` to use `OtlpSender` + +**File:** `dd-java-agent/agent-profiling/profiling-uploader/.../OtlpProfileUploader.java` + +Replace the hand-rolled OkHttp client, request builder, gzip, and retry logic with: + +```java +public final class OtlpProfileUploader implements RecordingDataListener { + + private final OtlpSender sender; + // No shared converter — created fresh per call for thread safety + private final boolean includeOriginalPayload; + + // ... constructor uses OtlpProfilesSenderFactory.create(config) ... + + @Override + public void onNewData(RecordingType type, RecordingData data, boolean sync) { + try { + byte[] otlpBytes = convertToOtlp(data); // fresh JfrToOtlpConverter per call + OtlpPayload payload = new OtlpPayload( + ByteBuffer.wrap(otlpBytes), OtlpPayload.PROTOBUF_CONTENT_TYPE); + if (sync) { + sender.send(payload); // blocking + } else { + executor.execute(() -> { + try { + sender.send(payload); + } catch (Exception e) { + log.warn("OTLP profile upload failed", e); + } + }); + } + } catch (Exception e) { + log.warn("OTLP profile conversion failed", e); + } finally { + // Safe: conversion is synchronous, payload is a self-contained byte[] copy + data.release(); + } + } + + public void shutdown() { + executor.shutdown(); + sender.shutdown(); + } +} +``` + +This eliminates: +- Custom `OkHttpClient` construction +- Custom `Request.Builder` / headers / gzip +- Custom `handleResponse` / `handleFailure` +- Custom connection pool eviction (delegated to `OtlpSender.shutdown()`) + +What remains: conversion (JFR→OTLP) + lifecycle (retain/release, executor for async). + +### Step 4: Add async dispatch + +`OtlpSender.send()` is synchronous (blocking). The existing `OtlpProfileUploader` supports both sync (shutdown) and async (periodic) uploads. Keep a small `ExecutorService` for async dispatch, same as the current implementation: + +```java +private final ExecutorService executor = + new ThreadPoolExecutor(0, MAX_RUNNING_REQUESTS, 60L, TimeUnit.SECONDS, + new SynchronousQueue<>(), + new AgentThreadFactory(AgentThread.PROFILER_HTTP_DISPATCHER), + new ThreadPoolExecutor.AbortPolicy()); +``` + +**Rejected execution handling:** The `SynchronousQueue` + `AbortPolicy` throws `RejectedExecutionException` when all threads are busy. The `onNewData` async path must catch this and log a warning (the data is already released in the `finally` block, so no leak): + +```java +try { + executor.execute(() -> { ... }); +} catch (RejectedExecutionException e) { + log.warn("OTLP profile upload rejected: too many concurrent requests"); +} +``` + +The `OtlpMetricsService` and `OtlpLogsService` use a dedicated exporter thread with a sleep loop. Profiles are different — they're pushed by `ProfilingSystem.snapshot()`, not polled. So the executor pattern is correct for profiles. + +### Step 5: Add telemetry + +**File:** `internal-api/.../api/telemetry/OtlpTelemetry.java` + +Add a `profilesExport` counter set, mirroring `tracesExport` and `metricsExport`: + +```java +private final String[] profilesTags = tagsFor(Config.get().getOtlpProfilesProtocol()); +private final ExportCounters profilesExport = new ExportCounters("profiles"); + +public void onProfilesExportAttempt() { profilesExport.attempts.increment(); } +public void onProfilesExportComplete(boolean success) { profilesExport.complete(success); } +``` + +Wire calls in `OtlpProfileUploader` around `sender.send()` (import `datadog.trace.common.writer.RemoteApi` or the extracted `OtlpResponse`): +```java +OtlpTelemetry.getInstance().onProfilesExportAttempt(); +RemoteApi.Response response = sender.send(payload); +OtlpTelemetry.getInstance().onProfilesExportComplete(response.success()); +``` + +### Step 6: Resolve the module dependency + +**Problem:** `OtlpSender`, `OtlpGrpcSender`, `OtlpHttpSender`, `OtlpPayload`, `OtlpSenderSupport` live in `dd-trace-core/.../otlp/common/`. `profiling-uploader` currently does not depend on `dd-trace-core`. + +**Options:** + +1. **Add `dd-trace-core` as a dependency of `profiling-uploader`** — simplest, but `dd-trace-core` is a large module and this creates a heavy dependency for a small uploader. + +2. **Extract `otlp/common` to a shared module** (e.g. `communication/otlp-common` or `utils/otlp-common`) — cleaner, but requires moving classes and updating all existing consumers. + +3. **Move `OtlpProfileUploader` to `dd-trace-core`** — it would naturally live alongside the other OTLP services. But it implements `RecordingDataListener` (from `internal-api`) and needs `JfrToOtlpConverter` (from `profiling-otel`), creating a reverse dependency. + +**Recommendation:** Option 2 (extract `otlp/common` sender infrastructure to a lightweight shared module). This is the architecturally correct choice and keeps dependencies clean. + +**New module:** `communication/otlp-exporter` +- `OtlpSender` interface +- `OtlpGrpcSender`, `OtlpHttpSender` +- `OtlpPayload`, `OtlpHttpRequestBody`, `OtlpGrpcRequestBody` +- `OtlpSenderSupport` +- `OtlpTraceFlags` (shared utility) + +**Stays in `dd-trace-core`** (used by collectors, not senders): +- `OtlpProtoBuffer`, `OtlpCommonProto` (used by trace/metrics/logs collectors) +- `OtlpResourceAttributes`, `OtlpResourceJson`, `OtlpResourceProto` +- All per-signal collectors (`OtlpTraceProtoCollector`, `OtlpMetricsProtoCollector`, etc.) + +**The `RemoteApi` problem:** `OtlpSender.send()` returns `RemoteApi.Response`, but `RemoteApi` lives in `dd-trace-core/.../common/writer/`, not `dd-trace-api`. Extracting the senders without addressing this would still require a `dd-trace-core` dependency, defeating the purpose. **Fix:** extract `RemoteApi.Response` (a simple value class with `success()`/`failed()` factory methods and a `boolean success()` getter) to `dd-trace-api` or into the new module as `OtlpResponse`. Update `OtlpSender` to return `OtlpResponse` instead. This is a small, self-contained class with no heavy dependencies. + +Dependencies of new module: `communication/http` (for `OkHttpUtils`, `HttpRetryPolicy`), `dd-trace-api` (for `OtlpConfig`), `utils/logging-utils` (for `RatelimitedLogger`). + +Consumers: `dd-trace-core` (traces, metrics, logs), `profiling-uploader` (profiles). + +**Affected build files:** `settings.gradle.kts` (new module), `dd-trace-core/build.gradle` (depend on new module), `dd-java-agent/agent-profiling/profiling-uploader/build.gradle` (depend on new module). + +### Step 7: Update `ProfilingAgent` wiring + +**File:** `dd-java-agent/agent-profiling/.../ProfilingAgent.java` + +The current wiring in `ProfilingAgent.run()` creates `OtlpProfileUploader` and wraps it in a lambda that calls `retain()` + `otlp.upload()` + `downstream.onNewData()`. This stays the same — the `OtlpProfileUploader` still implements `RecordingDataListener` and the retain/release protocol is unchanged. Only the internals of `OtlpProfileUploader` change (delegating to `OtlpSender` instead of hand-rolled HTTP). + +### Step 8: Migrate config consumers + +Update `OtlpProfileUploader` constructor to read from the new `Config.getOtlpProfiles*()` getters instead of `ConfigProvider.getBoolean(PROFILING_OTLP_*)`. + +Update tests: +- `OtlpProfileUploaderTest` — mock `Config.getOtlpProfiles*()` instead of `ConfigProvider.getBoolean(PROFILING_OTLP_*)` +- Add tests for gRPC protocol path (currently only HTTP is tested) + +### Step 9: Update `settings.gradle.kts` + +Add the new `communication/otlp-exporter` module (if Option 2) and wire it into the `profiling-uploader` and `dd-trace-core` dependency graphs. + +--- + +## Summary of Changes + +| Step | Files | Effort | Risk | +|------|-------|--------|------| +| 1: Config keys | `OtlpConfig`, `Config`, `ConfigDefaults` | Small | Low — additive | +| 2: Sender factory | New `OtlpProfilesSenderFactory` | Small | Low — mirrors existing | +| 3: Rewrite uploader | `OtlpProfileUploader` | Medium | Medium — core rewrite | +| 4: Async dispatch | `OtlpProfileUploader` | Small | Low | +| 5: Telemetry | `OtlpTelemetry` | Small | Low — additive | +| 6: Extract module | New `communication/otlp-exporter`, `dd-trace-core/build.gradle` | Medium | Medium — touches all OTLP signals, extract `RemoteApi.Response` | +| 7: ProfilingAgent | `ProfilingAgent` | Trivial | Low — no change to wiring | +| 8: Migrate config | `OtlpProfileUploader`, tests | Small | Low | +| 9: Settings | `settings.gradle.kts`, `dd-trace-core/build.gradle` | Trivial | Low | + +**Recommended order:** 6 → 1 → 2 → 3 → 4 → 5 → 8 → 7 → 9 + +Step 6 first because it unblocks everything else. Steps 1-2 are additive and safe. Step 3 is the main rewrite. + +## What Gets Eliminated + +- Custom `OkHttpClient` construction in `OtlpProfileUploader` (~40 lines) +- Custom `Request.Builder` / headers / gzip (~30 lines) +- Custom `handleResponse` / `handleFailure` / `uploadSync` / `uploadAsync` (~60 lines) +- Custom `compress()` method (~15 lines) +- Custom connection pool eviction +- Ad-hoc `profiling.otlp.url` / `profiling.otlp.compression.enabled` config keys +- Duplicate retry/error handling logic + +**Net: ~130 lines removed from `OtlpProfileUploader` (~150 removed, ~20 added for factory), replaced by ~10 lines of `OtlpSender` delegation.** + +## What Stays + +- `JfrToOtlpConverter` (the JFR→OTLP conversion — this is profile-specific, not transport) +- `RecordingDataListener` implementation + retain/release protocol +- `ProfilingAgent` wiring (lambda wrapping OTLP + downstream JFR uploader) +- `PROFILING_OTLP_ENABLED` and `PROFILING_OTLP_INCLUDE_ORIGINAL_PAYLOAD` config keys (profile-specific, not transport) diff --git a/dd-java-agent/agent-profiling/profiling-otel/doc/PROFCHECK_INTEGRATION.md b/dd-java-agent/agent-profiling/profiling-otel/doc/PROFCHECK_INTEGRATION.md new file mode 100644 index 00000000000..53a7a6b1016 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/doc/PROFCHECK_INTEGRATION.md @@ -0,0 +1,236 @@ +# Profcheck Integration Analysis + +This document analyzes the feasibility of integrating OpenTelemetry's `profcheck` tool for validating OTLP profiles produced by our JFR-to-OTLP converter. + +## What is Profcheck? + +**Profcheck** is an OpenTelemetry conformance checker for the OTLP Profiles format, maintained in https://github.com/open-telemetry/sig-profiling under `profcheck/`. `docker/Dockerfile.profcheck` builds it from a pinned `main` commit (`SIG_PROFILING_COMMIT`). + +### Key Features + +The tool validates: +- **Dictionary tables**: All tables (mapping, location, function, link, string, attribute, stack) +- **Index validity**: Ensures all indices reference valid entries +- **Reference integrity**: Checks cross-references between data structures +- **Sample consistency**: Validates sample values and timestamps +- **Time range boundaries**: Verifies timestamps are within profile time range +- **Data completeness**: Ensures required fields are present + +### How It Works + +```bash +# Simple CLI tool +profcheck + +# Reads binary protobuf ProfilesData +# Runs comprehensive validation +# Outputs: "conformance checks passed" or detailed errors +``` + +## Integration Feasibility: **HIGH** ✅ + +### Pros + +1. **Simple CLI Interface** + - Single command: `profcheck ` + - Reads standard protobuf files (our converter already produces these) + - Clear pass/fail output with detailed error messages + +2. **No Code Changes Required** + - Written in Go, runs as standalone binary + - Works with our existing protobuf output + - Can be integrated into CI/CD pipeline + +3. **Comprehensive Validation** + - Checks all dictionary tables + - Validates index references + - Ensures spec compliance + - Currently in active development with OTLP community + +4. **Easy to Adopt** + ```bash + # Build profcheck + cd profcheck + go build -o profcheck ./cmd/profcheck + + # Use with our converter + ./gradlew convertJfr --args="input.jfr output.pb" + profcheck output.pb + ``` + +### Cons + +1. **Unstable Upstream** + - Tracks the `v1development` OTLP profiles protos, which still change incompatibly + - The pinned profcheck commit must match the proto version `OtlpProtoFields` targets + - Need to track upstream changes + +2. **Go Dependency** + - Requires Go toolchain to build + - Need to vendor or download pre-built binary + - Cross-platform build considerations + +3. **Limited Scope** + - Only validates structure, not semantics + - Doesn't validate actual profiling data correctness + - Won't catch domain-specific issues (e.g., invalid stack traces) + +## Recommended Integration Approach + +### Phase 1: Docker-Based Testing (✅ IMPLEMENTED) + +Profcheck is now available as a **Docker-based validation tool**: + +```bash +# Convert JFR to OTLP +./gradlew :dd-java-agent:agent-profiling:profiling-otel:convertJfr \ + --args="recording.jfr output.pb" + +# Build profcheck Docker image (one-time) +./gradlew :dd-java-agent:agent-profiling:profiling-otel:buildProfcheck + +# Validate with profcheck +./gradlew :dd-java-agent:agent-profiling:profiling-otel:validateOtlp \ + -PotlpFile=output.pb +``` + +**OR use Docker directly**: + +```bash +# Build once (from project root) +docker build -f docker/Dockerfile.profcheck -t profcheck:latest . + +# Validate +docker run --rm -v $(pwd):/data:ro profcheck:latest /data/output.pb +``` + +**Benefits**: +- ✅ No Go installation required +- ✅ Reproducible environment +- ✅ Works on any platform with Docker +- ✅ Easy to integrate into CI/CD +- ✅ Automatically fetches latest profcheck from PR branch + +### Phase 2: CI/CD Integration + +Once profcheck is merged upstream, integrate into CI: + +```yaml +# .github/workflows/validate-otlp.yml +name: OTLP Validation + +on: [push, pull_request] + +jobs: + validate-otlp: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Go + uses: actions/setup-go@v4 + with: + go-version: '1.25' + + - name: Install profcheck + run: | + git clone https://github.com/open-telemetry/sig-profiling.git + cd sig-profiling/profcheck + go build -o $HOME/bin/profcheck ./cmd/profcheck + echo "$HOME/bin" >> $GITHUB_PATH + + - name: Generate test profile + run: | + ./gradlew :dd-java-agent:agent-profiling:profiling-otel:convertJfr \ + --args="test-data/sample.jfr test-output.pb" + + - name: Validate with profcheck + run: profcheck test-output.pb +``` + +### Phase 3: Test Integration (Long-term) + +Add profcheck validation to existing tests: + +```gradle +// build.gradle.kts +tasks.register("validateOtlpWithProfcheck") { + group = "verification" + description = "Validate OTLP output using profcheck" + + dependsOn("test") + + commandLine("profcheck", "build/test-results/sample-output.pb") +} + +tasks.named("check") { + dependsOn("validateOtlpWithProfcheck") +} +``` + +## Current Implementation Gaps + +Based on profcheck validation, our converter should ensure: + +1. ✅ **String table starts with empty string** (index 0) +2. ✅ **All indices are valid** (within bounds) +3. ✅ **Dictionary zero values** (first entry must be zero/empty) +4. ✅ **Time range consistency** (timestamps within profile bounds) +5. ⚠️ **Attribute indices** (we don't currently use attributes) +6. ⚠️ **Mapping table** (we don't currently populate mappings) + +### Known Gaps to Address + +Our current implementation doesn't populate: +- Mapping table (binary/library information) +- Attribute indices (resource/scope attributes) + +These are optional per spec but profcheck validates them if present. + +## Example Validation Output + +### Valid Profile +``` +$ profcheck output.pb +output.pb: conformance checks passed +``` + +### Invalid Profile +``` +$ profcheck output.pb +output.pb: conformance checks failed: profile 0: sample[5]: + timestamps_unix_nano[0]=1700000000 is outside profile time range + [1700000100, 1700060100] +``` + +## Recommendations + +### Immediate Actions + +1. **Manual Testing**: Use profcheck locally to validate converter output +2. **Document Usage**: Add profcheck instructions to CLI.md +3. **Track Upstream**: Bump `SIG_PROFILING_COMMIT` together with the pinned proto version + +### Next Steps + +1. **CI Integration**: Add profcheck validation to GitHub Actions +2. **Test Data**: Create test JFR files with known-good OTLP output +3. **Regression Testing**: Run profcheck on every converter change + +### Long-term + +1. **Vendoring**: Consider vendoring profcheck or pre-built binaries +2. **Test Suite**: Expand converter tests to cover all profcheck validations +3. **Documentation**: Document profcheck validation in ARCHITECTURE.md + +## Conclusion + +**YES, we can easily use profcheck to validate our OTLP profiles.** + +- ✅ Simple CLI tool with clear interface +- ✅ No code changes required +- ✅ Works with our existing protobuf output +- ✅ Comprehensive validation coverage +- ✅ Can be integrated into CI/CD + +**Recommended**: Start using profcheck manually now, integrate into CI after upstream PR merges. diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/jmh/java/com/datadog/profiling/otel/benchmark/DictionaryTableBenchmark.java b/dd-java-agent/agent-profiling/profiling-otel/src/jmh/java/com/datadog/profiling/otel/benchmark/DictionaryTableBenchmark.java new file mode 100644 index 00000000000..6c11cc9e26b --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/jmh/java/com/datadog/profiling/otel/benchmark/DictionaryTableBenchmark.java @@ -0,0 +1,225 @@ +package com.datadog.profiling.otel.benchmark; + +import static java.util.concurrent.TimeUnit.MICROSECONDS; +import static org.openjdk.jmh.annotations.Mode.Throughput; + +import com.datadog.profiling.otel.proto.dictionary.AttributeTable; +import com.datadog.profiling.otel.proto.dictionary.FunctionTable; +import com.datadog.profiling.otel.proto.dictionary.LinkTable; +import com.datadog.profiling.otel.proto.dictionary.LocationTable; +import com.datadog.profiling.otel.proto.dictionary.StackTable; +import com.datadog.profiling.otel.proto.dictionary.StringTable; +import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** Benchmarks for dictionary table deduplication performance. */ +@State(Scope.Benchmark) +@BenchmarkMode(Throughput) +@OutputTimeUnit(MICROSECONDS) +@Fork(value = 1) +@Warmup(iterations = 3, time = 5) +@Measurement(iterations = 3, time = 5) +public class DictionaryTableBenchmark { + + @Param({"100", "1000", "10000"}) + int uniqueEntries; + + @Param({"0.0", "0.5", "0.95"}) + double hitRate; + + // String table test data + private StringTable stringTable; + private String[] testStrings; + + // Function table test data + private FunctionTable functionTable; + private int[] functionNameIndices; + private int[] functionSystemNameIndices; + private int[] functionFilenameIndices; + private long[] functionStartLines; + + // Location table test data + private LocationTable locationTable; + private int[] locationMappingIndices; + private long[] locationAddresses; + private int[] locationFunctionIndices; + private long[] locationLines; + private long[] locationColumns; + + // Stack table test data + private StackTable stackTable; + private int[][] stackLocationIndices; + + // Link table test data + private LinkTable linkTable; + private byte[][] linkTraceIds; + private byte[][] linkSpanIds; + + // Attribute table test data + private AttributeTable attributeTable; + private int[] attributeKeyIndices; + private long[] attributeValues; + private int[] attributeUnitIndices; + + @Setup(Level.Trial) + public void setup() { + Random rnd = new Random(42); + + // Calculate pool size based on hit rate + // Lower hit rate = larger pool of unique values. Cap the pool — uniqueEntries=10000 with + // hitRate=0.0 would otherwise OOM the trial with a 10M-entry pool. + int poolSize = + hitRate == 0.0 + ? Math.min(uniqueEntries * 1000, 1_000_000) + : (int) (uniqueEntries / (1.0 - hitRate)); + + // Setup StringTable + stringTable = new StringTable(); + testStrings = new String[poolSize]; + for (int i = 0; i < poolSize; i++) { + testStrings[i] = generateClassName(rnd) + "." + generateMethodName(rnd); + } + + // Setup FunctionTable + functionTable = new FunctionTable(); + functionNameIndices = new int[poolSize]; + functionSystemNameIndices = new int[poolSize]; + functionFilenameIndices = new int[poolSize]; + functionStartLines = new long[poolSize]; + for (int i = 0; i < poolSize; i++) { + functionNameIndices[i] = i; + functionSystemNameIndices[i] = i; + functionFilenameIndices[i] = i % 100; // Reuse filenames + functionStartLines[i] = rnd.nextInt(1000); + } + + // Setup LocationTable + locationTable = new LocationTable(); + locationMappingIndices = new int[poolSize]; + locationAddresses = new long[poolSize]; + locationFunctionIndices = new int[poolSize]; + locationLines = new long[poolSize]; + locationColumns = new long[poolSize]; + for (int i = 0; i < poolSize; i++) { + locationMappingIndices[i] = 0; + locationAddresses[i] = rnd.nextLong(); + locationFunctionIndices[i] = i; + locationLines[i] = rnd.nextInt(1000); + locationColumns[i] = rnd.nextInt(100); + } + + // Setup StackTable + stackTable = new StackTable(); + stackLocationIndices = new int[poolSize][]; + for (int i = 0; i < poolSize; i++) { + int depth = 5 + rnd.nextInt(20); // 5-25 frames + stackLocationIndices[i] = new int[depth]; + for (int j = 0; j < depth; j++) { + stackLocationIndices[i][j] = rnd.nextInt(poolSize); + } + } + + // Setup LinkTable + linkTable = new LinkTable(); + linkTraceIds = new byte[poolSize][]; + linkSpanIds = new byte[poolSize][]; + for (int i = 0; i < poolSize; i++) { + linkTraceIds[i] = new byte[16]; + linkSpanIds[i] = new byte[8]; + rnd.nextBytes(linkTraceIds[i]); + rnd.nextBytes(linkSpanIds[i]); + } + + // Setup AttributeTable + attributeTable = new AttributeTable(); + attributeKeyIndices = new int[poolSize]; + attributeValues = new long[poolSize]; + attributeUnitIndices = new int[poolSize]; + for (int i = 0; i < poolSize; i++) { + attributeKeyIndices[i] = i % 10; // Reuse keys + attributeValues[i] = rnd.nextLong(); + attributeUnitIndices[i] = i % 5; // Reuse units + } + } + + @Benchmark + public void internString(Blackhole bh) { + int idx = ThreadLocalRandom.current().nextInt(testStrings.length); + int result = stringTable.intern(testStrings[idx]); + bh.consume(result); + } + + @Benchmark + public void internFunction(Blackhole bh) { + int idx = ThreadLocalRandom.current().nextInt(functionNameIndices.length); + int result = + functionTable.intern( + functionNameIndices[idx], + functionSystemNameIndices[idx], + functionFilenameIndices[idx], + functionStartLines[idx]); + bh.consume(result); + } + + @Benchmark + public void internLocation(Blackhole bh) { + int idx = ThreadLocalRandom.current().nextInt(locationMappingIndices.length); + int result = + locationTable.intern( + locationMappingIndices[idx], + locationAddresses[idx], + locationFunctionIndices[idx], + locationLines[idx], + locationColumns[idx]); + bh.consume(result); + } + + @Benchmark + public void internStack(Blackhole bh) { + int idx = ThreadLocalRandom.current().nextInt(stackLocationIndices.length); + int result = stackTable.intern(stackLocationIndices[idx]); + bh.consume(result); + } + + @Benchmark + public void internLink(Blackhole bh) { + int idx = ThreadLocalRandom.current().nextInt(linkTraceIds.length); + int result = linkTable.intern(linkTraceIds[idx], linkSpanIds[idx]); + bh.consume(result); + } + + @Benchmark + public void internAttribute(Blackhole bh) { + int idx = ThreadLocalRandom.current().nextInt(attributeKeyIndices.length); + int result = + attributeTable.internInt( + attributeKeyIndices[idx], attributeValues[idx], attributeUnitIndices[idx]); + bh.consume(result); + } + + private String generateClassName(Random rnd) { + String[] packages = {"com.example", "org.apache", "io.netty", "datadog.trace"}; + String[] classes = {"Handler", "Service", "Controller", "Manager", "Factory"}; + return packages[rnd.nextInt(packages.length)] + + "." + + classes[rnd.nextInt(classes.length)] + + rnd.nextInt(100); + } + + private String generateMethodName(Random rnd) { + String[] methods = {"process", "handle", "execute", "invoke", "run", "doWork"}; + return methods[rnd.nextInt(methods.length)] + rnd.nextInt(100); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/jmh/java/com/datadog/profiling/otel/benchmark/JfrToOtlpConverterBenchmark.java b/dd-java-agent/agent-profiling/profiling-otel/src/jmh/java/com/datadog/profiling/otel/benchmark/JfrToOtlpConverterBenchmark.java new file mode 100644 index 00000000000..9dd91c1bf27 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/jmh/java/com/datadog/profiling/otel/benchmark/JfrToOtlpConverterBenchmark.java @@ -0,0 +1,152 @@ +package com.datadog.profiling.otel.benchmark; + +import static com.datadog.profiling.otel.JfrTools.epochNanos; +import static com.datadog.profiling.otel.JfrTools.putStackTrace; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.openjdk.jmh.annotations.Mode.Throughput; + +import com.datadog.profiling.otel.JfrToOtlpConverter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.Random; +import org.openjdk.jmc.flightrecorder.writer.api.Recording; +import org.openjdk.jmc.flightrecorder.writer.api.Recordings; +import org.openjdk.jmc.flightrecorder.writer.api.Type; +import org.openjdk.jmc.flightrecorder.writer.api.Types; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * End-to-end benchmarks for JFR-to-OTLP profile conversion. Run + * + *

Tests full conversion pipeline including: + * + *

+ */ +@State(Scope.Benchmark) +@BenchmarkMode(Throughput) +@OutputTimeUnit(SECONDS) +@Fork(value = 1) +@Warmup(iterations = 3, time = 5) +@Measurement(iterations = 5, time = 10) +public class JfrToOtlpConverterBenchmark { + + @Param({"50", "500", "5000"}) + int eventCount; + + @Param({"10", "50", "100"}) + int stackDepth; + + @Param({"100", "1000"}) + int uniqueContexts; + + /** + * Percentage of events that reuse existing stack traces. 0 = all unique stacks (worst case for + * cache), 90 = 90% of events reuse stacks from first 10% (best case for cache, realistic for + * production workloads). + */ + @Param({"0", "70", "90"}) + int stackDuplicationPercent; + + private Path jfrFile; + private JfrToOtlpConverter converter; + private Instant start; + private Instant end; + + @Setup(Level.Trial) + public void setup() throws IOException { + jfrFile = Files.createTempFile("jfr-otlp-benchmark-", ".jfr"); + converter = new JfrToOtlpConverter(); + + // Create JFR recording with synthetic events + try (Recording recording = Recordings.newRecording(jfrFile)) { + Types types = recording.getTypes(); + + Type executionSampleType = + recording.registerEventType( + "datadog.ExecutionSample", + type -> { + type.addField("spanId", Types.Builtin.LONG); + type.addField("localRootSpanId", Types.Builtin.LONG); + }); + + Random random = new Random(42); + + // Pre-generate unique stack traces that will be reused + int uniqueStackCount = Math.max(1, (eventCount * (100 - stackDuplicationPercent)) / 100); + StackTraceElement[][] uniqueStacks = new StackTraceElement[uniqueStackCount][]; + + for (int stackIdx = 0; stackIdx < uniqueStackCount; stackIdx++) { + StackTraceElement[] stackTrace = new StackTraceElement[stackDepth]; + for (int frameIdx = 0; frameIdx < stackDepth; frameIdx++) { + int classId = random.nextInt(200); + int methodId = random.nextInt(50); + int lineNumber = 10 + random.nextInt(990); + + stackTrace[frameIdx] = + new StackTraceElement( + "com.example.Class" + classId, + "method" + methodId, + "Class" + classId + ".java", + lineNumber); + } + uniqueStacks[stackIdx] = stackTrace; + } + + // Generate events, reusing stacks according to duplication percentage + for (int i = 0; i < eventCount; i++) { + // Select stack trace (first uniqueStackCount events get unique stacks, rest reuse) + int stackIndex = i < uniqueStackCount ? i : random.nextInt(uniqueStackCount); + final StackTraceElement[] stackTrace = uniqueStacks[stackIndex]; + + long contextId = random.nextInt(uniqueContexts); + final long spanId = 50000L + contextId; + final long rootSpanId = 60000L + contextId; + + recording.writeEvent( + executionSampleType.asValue( + valueBuilder -> { + valueBuilder.putField("startTime", epochNanos()); + valueBuilder.putField("spanId", spanId); + valueBuilder.putField("localRootSpanId", rootSpanId); + valueBuilder.putField( + "stackTrace", + stackTraceBuilder -> putStackTrace(types, stackTraceBuilder, stackTrace)); + })); + } + } + + start = Instant.now().minusSeconds(60); + end = Instant.now(); + } + + @TearDown(Level.Trial) + public void tearDown() throws IOException { + Files.deleteIfExists(jfrFile); + } + + @Benchmark + public void convertJfrToOtlp(Blackhole bh) throws IOException { + byte[] result = converter.addFile(jfrFile, start, end).convert(); + bh.consume(result); + converter.reset(); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/jmh/java/com/datadog/profiling/otel/benchmark/ProtobufEncoderBenchmark.java b/dd-java-agent/agent-profiling/profiling-otel/src/jmh/java/com/datadog/profiling/otel/benchmark/ProtobufEncoderBenchmark.java new file mode 100644 index 00000000000..6ba5bf521d9 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/jmh/java/com/datadog/profiling/otel/benchmark/ProtobufEncoderBenchmark.java @@ -0,0 +1,200 @@ +package com.datadog.profiling.otel.benchmark; + +import static java.util.concurrent.TimeUnit.MICROSECONDS; +import static org.openjdk.jmh.annotations.Mode.Throughput; + +import com.datadog.profiling.otel.proto.ProtobufEncoder; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Benchmarks for protobuf encoding primitives. + * + *

Tests the performance of various protobuf encoding operations including varint encoding, + * fixed-size fields, strings, and nested messages. + */ +@State(Scope.Benchmark) +@BenchmarkMode(Throughput) +@OutputTimeUnit(MICROSECONDS) +@Fork(value = 1) +@Warmup(iterations = 3, time = 5) +@Measurement(iterations = 3, time = 5) +public class ProtobufEncoderBenchmark { + + private ProtobufEncoder encoder; + + // Test data + private static final String SHORT_STRING = "process"; + private static final String MEDIUM_STRING = "com.example.MyClass.myMethod"; + private static final String LONG_STRING = + "com.example.very.deep.package.structure.MyVeryLongClassName.myVeryLongMethodNameWithLotsOfParameters"; + + private static final byte[] SHORT_BYTES = new byte[] {1, 2, 3, 4, 5}; + private static final byte[] MEDIUM_BYTES = new byte[64]; + private static final byte[] LONG_BYTES = new byte[1024]; + + static { + for (int i = 0; i < MEDIUM_BYTES.length; i++) { + MEDIUM_BYTES[i] = (byte) i; + } + for (int i = 0; i < LONG_BYTES.length; i++) { + LONG_BYTES[i] = (byte) i; + } + } + + @Setup(Level.Invocation) + public void setup() { + encoder = new ProtobufEncoder(4096); + } + + // Varint encoding benchmarks + @Benchmark + public void writeVarintSmall(Blackhole bh) { + encoder.writeVarintField(1, 42); // < 128, single byte + bh.consume(encoder); + } + + @Benchmark + public void writeVarintMedium(Blackhole bh) { + encoder.writeVarintField(1, 5000); // 2 bytes + bh.consume(encoder); + } + + @Benchmark + public void writeVarintLarge(Blackhole bh) { + encoder.writeVarintField(1, 1_000_000); // 3+ bytes + bh.consume(encoder); + } + + @Benchmark + public void writeVarintVeryLarge(Blackhole bh) { + encoder.writeVarintField(1, Long.MAX_VALUE); // max bytes + bh.consume(encoder); + } + + // Fixed64 encoding benchmarks + @Benchmark + public void writeFixed64(Blackhole bh) { + encoder.writeFixed64Field(1, 123456789012345L); + bh.consume(encoder); + } + + // String encoding benchmarks + @Benchmark + public void writeStringShort(Blackhole bh) { + encoder.writeStringField(1, SHORT_STRING); + bh.consume(encoder); + } + + @Benchmark + public void writeStringMedium(Blackhole bh) { + encoder.writeStringField(1, MEDIUM_STRING); + bh.consume(encoder); + } + + @Benchmark + public void writeStringLong(Blackhole bh) { + encoder.writeStringField(1, LONG_STRING); + bh.consume(encoder); + } + + // Bytes encoding benchmarks + @Benchmark + public void writeBytesShort(Blackhole bh) { + encoder.writeBytesField(1, SHORT_BYTES); + bh.consume(encoder); + } + + @Benchmark + public void writeBytesMedium(Blackhole bh) { + encoder.writeBytesField(1, MEDIUM_BYTES); + bh.consume(encoder); + } + + @Benchmark + public void writeBytesLong(Blackhole bh) { + encoder.writeBytesField(1, LONG_BYTES); + bh.consume(encoder); + } + + // Nested message encoding benchmarks + @Benchmark + public void writeNestedMessageSimple(Blackhole bh) { + encoder.writeNestedMessage( + 1, + enc -> { + enc.writeVarintField(1, 42); + enc.writeStringField(2, "test"); + }); + bh.consume(encoder); + } + + @Benchmark + public void writeNestedMessageComplex(Blackhole bh) { + encoder.writeNestedMessage( + 1, + enc -> { + enc.writeVarintField(1, 42); + enc.writeStringField(2, MEDIUM_STRING); + enc.writeFixed64Field(3, 123456789L); + enc.writeNestedMessage( + 4, + enc2 -> { + enc2.writeVarintField(1, 1); + enc2.writeVarintField(2, 2); + }); + }); + bh.consume(encoder); + } + + // Combined operations (realistic usage) + @Benchmark + public void writeTypicalSample(Blackhole bh) { + encoder.writeVarintField(1, 123); // stack_index + encoder.writeVarintField(3, 456); // link_index + encoder.writeVarintField(4, 1); // value + encoder.writeFixed64Field(5, 1234567890123456L); // timestamp + bh.consume(encoder); + } + + @Benchmark + public void writeTypicalLocation(Blackhole bh) { + encoder.writeVarintField(1, 0); // mapping_index + encoder.writeVarintField(2, 0x1234567890ABCDEFL); // address + encoder.writeNestedMessage( + 3, + enc -> { + enc.writeVarintField(1, 100); // function_index + enc.writeVarintField(2, 42); // line + }); + bh.consume(encoder); + } + + @Benchmark + public void writeTypicalFunction(Blackhole bh) { + encoder.writeVarintField(1, 10); // name_strindex + encoder.writeVarintField(2, 10); // system_name_strindex + encoder.writeVarintField(3, 5); // filename_strindex + encoder.writeVarintField(4, 100); // start_line + bh.consume(encoder); + } + + // Conversion to byte array (measures final serialization overhead) + @Benchmark + public void toByteArray(Blackhole bh) { + encoder.writeVarintField(1, 42); + encoder.writeStringField(2, MEDIUM_STRING); + encoder.writeFixed64Field(3, 123456789L); + byte[] result = encoder.toByteArray(); + bh.consume(result); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/jmh/java/com/datadog/profiling/otel/benchmark/RealFileConversionBenchmark.java b/dd-java-agent/agent-profiling/profiling-otel/src/jmh/java/com/datadog/profiling/otel/benchmark/RealFileConversionBenchmark.java new file mode 100644 index 00000000000..eb590d09d68 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/jmh/java/com/datadog/profiling/otel/benchmark/RealFileConversionBenchmark.java @@ -0,0 +1,55 @@ +package com.datadog.profiling.otel.benchmark; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.openjdk.jmh.annotations.Mode.Throughput; + +import com.datadog.profiling.otel.JfrToOtlpConverter; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** Benchmarks JFR-to-OTLP conversion on real production JFR recordings. */ +@State(Scope.Benchmark) +@BenchmarkMode(Throughput) +@OutputTimeUnit(SECONDS) +@Fork(value = 1) +@Warmup(iterations = 3, time = 5) +@Measurement(iterations = 5, time = 10) +public class RealFileConversionBenchmark { + + @Param({"/tmp/inventory-cache.jfr", "/tmp/sbom.jfr", "/tmp/otelp.jfr"}) + String jfrFilePath; + + private Path jfrFile; + private JfrToOtlpConverter converter; + private Instant start; + private Instant end; + + @Setup(Level.Trial) + public void setup() throws IOException { + jfrFile = Paths.get(jfrFilePath); + converter = new JfrToOtlpConverter(); + start = Instant.EPOCH; + end = Instant.now(); + } + + @Benchmark + public void convertJfrToOtlp(Blackhole bh) throws IOException { + byte[] result = converter.addFile(jfrFile, start, end).convert(); + bh.consume(result); + converter.reset(); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/jmh/java/com/datadog/profiling/otel/benchmark/StackTraceConversionBenchmark.java b/dd-java-agent/agent-profiling/profiling-otel/src/jmh/java/com/datadog/profiling/otel/benchmark/StackTraceConversionBenchmark.java new file mode 100644 index 00000000000..7e486f87121 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/jmh/java/com/datadog/profiling/otel/benchmark/StackTraceConversionBenchmark.java @@ -0,0 +1,118 @@ +package com.datadog.profiling.otel.benchmark; + +import static com.datadog.profiling.otel.JfrTools.putStackTrace; +import static com.datadog.profiling.otel.JfrTools.writeEvent; +import static java.util.concurrent.TimeUnit.MICROSECONDS; +import static org.openjdk.jmh.annotations.Mode.Throughput; + +import com.datadog.profiling.otel.JfrToOtlpConverter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.Random; +import org.openjdk.jmc.flightrecorder.writer.api.Recording; +import org.openjdk.jmc.flightrecorder.writer.api.Recordings; +import org.openjdk.jmc.flightrecorder.writer.api.Type; +import org.openjdk.jmc.flightrecorder.writer.api.Types; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Benchmarks for stack trace conversion with varying depths and deduplication ratios. + * + *

Uses a fixed small event count so that stack trace processing dominates over JFR I/O. + */ +@State(Scope.Benchmark) +@BenchmarkMode(Throughput) +@OutputTimeUnit(MICROSECONDS) +@Fork(value = 1) +@Warmup(iterations = 3, time = 5) +@Measurement(iterations = 3, time = 5) +public class StackTraceConversionBenchmark { + + @Param({"5", "15", "30", "50"}) + int stackDepth; + + @Param({"1", "10", "100"}) + int uniqueStacks; + + private static final int EVENTS_PER_STACK = 5; + + private Path jfrFile; + private JfrToOtlpConverter converter; + private Instant start; + private Instant end; + + @Setup(Level.Trial) + public void setup() throws IOException { + jfrFile = Files.createTempFile("jfr-stacktrace-bench-", ".jfr"); + converter = new JfrToOtlpConverter(); + Random rnd = new Random(42); + + try (Recording recording = Recordings.newRecording(jfrFile)) { + Types types = recording.getTypes(); + Type executionSampleType = + recording.registerEventType( + "datadog.ExecutionSample", + type -> { + type.addField("spanId", Types.Builtin.LONG); + type.addField("localRootSpanId", Types.Builtin.LONG); + }); + + StackTraceElement[][] stacks = new StackTraceElement[uniqueStacks][]; + for (int s = 0; s < uniqueStacks; s++) { + stacks[s] = new StackTraceElement[stackDepth]; + for (int f = 0; f < stackDepth; f++) { + stacks[s][f] = + new StackTraceElement( + "com.example.Class" + (s * 31 + f) % 200, + "method" + (s * 17 + f) % 50, + "Class.java", + 10 + f); + } + } + + for (int s = 0; s < uniqueStacks; s++) { + final StackTraceElement[] stack = stacks[s]; + final long spanId = 1000L + s; + for (int e = 0; e < EVENTS_PER_STACK; e++) { + writeEvent( + recording, + executionSampleType, + vb -> { + vb.putField("spanId", spanId); + vb.putField("localRootSpanId", spanId); + vb.putField("stackTrace", stb -> putStackTrace(types, stb, stack)); + }); + } + } + } + + start = Instant.now().minusSeconds(60); + end = Instant.now(); + } + + @TearDown(Level.Trial) + public void tearDown() throws IOException { + Files.deleteIfExists(jfrFile); + } + + @Benchmark + public void convertStackTraces(Blackhole bh) throws IOException { + byte[] result = converter.addFile(jfrFile, start, end).convert(); + bh.consume(result); + converter.reset(); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/JfrToOtlpConverter.java b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/JfrToOtlpConverter.java new file mode 100644 index 00000000000..c9466e2e2a0 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/JfrToOtlpConverter.java @@ -0,0 +1,1473 @@ +package com.datadog.profiling.otel; + +import com.datadog.profiling.otel.jfr.ExecutionSample; +import com.datadog.profiling.otel.jfr.JavaMonitorEnter; +import com.datadog.profiling.otel.jfr.JavaMonitorWait; +import com.datadog.profiling.otel.jfr.JdkCPUTimeSample; +import com.datadog.profiling.otel.jfr.JdkExecutionSample; +import com.datadog.profiling.otel.jfr.JdkNativeMethodSample; +import com.datadog.profiling.otel.jfr.JfrClass; +import com.datadog.profiling.otel.jfr.JfrMethod; +import com.datadog.profiling.otel.jfr.JfrStackFrame; +import com.datadog.profiling.otel.jfr.JfrStackTrace; +import com.datadog.profiling.otel.jfr.MethodSample; +import com.datadog.profiling.otel.jfr.ObjectSample; +import com.datadog.profiling.otel.proto.OtlpProtoFields; +import com.datadog.profiling.otel.proto.OtlpResourceAttributes; +import com.datadog.profiling.otel.proto.ProtobufEncoder; +import com.datadog.profiling.otel.proto.dictionary.AttributeTable; +import com.datadog.profiling.otel.proto.dictionary.FunctionTable; +import com.datadog.profiling.otel.proto.dictionary.LinkTable; +import com.datadog.profiling.otel.proto.dictionary.LocationTable; +import com.datadog.profiling.otel.proto.dictionary.StackTable; +import com.datadog.profiling.otel.proto.dictionary.StringTable; +import datadog.json.JsonWriter; +import datadog.trace.api.profiling.RecordingData; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import io.jafar.parser.api.Control; +import io.jafar.parser.api.ParsingContext; +import io.jafar.parser.api.TypedJafarParser; +import io.jafar.parser.impl.ParsingContextImpl; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ThreadLocalRandom; + +public final class JfrToOtlpConverter { + + public enum Kind { + PROTO, + JSON, + JSON_PRETTY + } + + private static final class PathEntry { + final Path path; + final boolean ephemeral; + + PathEntry(Path path, boolean ephemeral) { + this.path = path; + this.ephemeral = ephemeral; + } + + // ephemeral is part of identity so a caller file and a stream copy of the same path + // stay distinct entries — otherwise reset() could delete a caller-owned file + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + PathEntry pathEntry = (PathEntry) o; + return ephemeral == pathEntry.ephemeral && Objects.equals(path, pathEntry.path); + } + + @Override + public int hashCode() { + return Objects.hash(path, ephemeral); + } + } + + private static final String PROFILE_TYPE_CPU = "cpu"; + private static final String PROFILE_TYPE_WALL = "wall"; + private static final String PROFILE_TYPE_ALLOC = "alloc-samples"; + private static final String PROFILE_TYPE_LOCK = "lock-contention"; + + private static final String UNIT_SAMPLES = "samples"; + private static final String UNIT_BYTES = "bytes"; + private static final String UNIT_NANOSECONDS = "nanoseconds"; + + // Dictionary tables (shared across all samples) + private final StringTable stringTable = new StringTable(); + private final FunctionTable functionTable = new FunctionTable(); + private final LocationTable locationTable = new LocationTable(); + private final StackTable stackTable = new StackTable(); + private final LinkTable linkTable = new LinkTable(); + private final AttributeTable attributeTable = new AttributeTable(); + + // Stack trace cache: maps (stackTraceId + chunkId) → stack index + private final LongKeyIntMap stackTraceCache = new LongKeyIntMap(1, 1024); + + // Frame cache: maps (methodId + chunkId + lineNumber) → locationIndex + private final LongKeyIntMap frameCache = new LongKeyIntMap(1, 1024); + + // Per-type attribute index arrays, lazily initialized once per conversion session and shared + // across all samples of the same type to avoid per-sample allocation. + private int[] cpuAttrIndices; + private int[] wallAttrIndices; + private int[] lockAttrIndices; + // sample-type-only attribute array for alloc samples without a class name; a dedicated field + // instead of a "" sentinel key in allocAttrCache so an empty-string class name can never + // alias with the no-class-name case + private int[] allocNoClassAttrIndices; + + // Alloc sample field availability: probed per chunk (field layout can differ between chunks), + // so allocProbeChunkOrdinal tracks which chunk the cached result belongs to. + private int allocProbeChunkOrdinal = -1; + private boolean allocHasSizeWeight; + + // Cached attribute indices per alloc class name, avoiding per-sample int[] allocation. + private final java.util.Map allocAttrCache = new java.util.HashMap<>(); + + // Cached chunk ordinal to avoid re-assigning the chunk discriminator on every event. + // A monotonic ordinal is used because identity hashes are not guaranteed unique — a collision + // would silently attribute samples to the wrong chunk's cached stacks/frames. + private Object lastChunk; + private int chunkOrdinal; + + // RecordingData instances retained by addRecording(RecordingData) so their backing files + // stay alive until convert() consumes them; released in reset() + private final List retainedRecordings = new ArrayList<>(); + + // Sample collectors by profile type + private final List cpuSamples = new ArrayList<>(); + private final List wallSamples = new ArrayList<>(); + private final List allocSamples = new ArrayList<>(); + private final List lockSamples = new ArrayList<>(); + + // insertion-ordered so the concatenated original_payload and sample order are + // deterministic for identical inputs + private final Set pathEntries = new LinkedHashSet<>(); + private final ProtobufEncoder protoEncoder = new ProtobufEncoder(64 * 1024); + private ParsingContext parsingContext = new ParsingContextImpl(); + + // Profile metadata + private long startTimeNanos; + private long endTimeNanos; + private boolean timeRangeInitialized; + + // when set, the profile window is derived from the observed event timestamps instead of the + // caller-supplied recording window (CLI conversions pass a wide synthetic window) + private boolean deriveTimeBoundsFromEvents = false; + private long minEventTimestampNanos = Long.MAX_VALUE; + private long maxEventTimestampNanos = Long.MIN_VALUE; + + // Original payload support + private boolean includeOriginalPayload = false; + // per-conversion flag: the original payload is embedded only once per ProfilesData, in the + // first non-empty profile — re-embedding the same recording in every sample kind would + // multiply the payload size by the number of emitted profiles + private boolean originalPayloadWritten = false; + + // Resource attributes written into the Resource message of ResourceProfiles + private Map resourceAttributes = Collections.emptyMap(); + + private static final class SampleData { + final int stackIndex; + final int linkIndex; + final long value; + final long timestampNanos; + final int[] attributeIndices; + + SampleData( + int stackIndex, int linkIndex, long value, long timestampNanos, int[] attributeIndices) { + this.stackIndex = stackIndex; + this.linkIndex = linkIndex; + this.value = value; + this.timestampNanos = timestampNanos; + this.attributeIndices = attributeIndices != null ? attributeIndices : new int[0]; + } + } + + // attaches the raw JFR files as the original_payload blob; multiple files are + // concatenated into a single blob carrying one file header per input — consumable only by + // parsers accepting repeated headers (jafar does); standard single-header JFR readers may not + public JfrToOtlpConverter setIncludeOriginalPayload(boolean include) { + this.includeOriginalPayload = include; + return this; + } + + // empty by default - no resource message is emitted + // setIncludeOriginalPayload, setDeriveTimeBoundsFromEvents and setResourceAttributes are + // sticky — they survive reset() and apply to all subsequent conversions + public JfrToOtlpConverter setResourceAttributes(Map attributes) { + this.resourceAttributes = attributes != null ? attributes : Collections.emptyMap(); + return this; + } + + public JfrToOtlpConverter setDeriveTimeBoundsFromEvents(boolean derive) { + this.deriveTimeBoundsFromEvents = derive; + return this; + } + + /** + * Registers a recording for conversion. The converter retains it and releases it in {@link + * #reset()}, so callers may release their own reference right away. + * + * @throws IllegalStateException if the recording has already been fully released + */ + public JfrToOtlpConverter addRecording(RecordingData recordingData) throws IOException { + // keep the recording (and its backing file) alive until convert() consumes it — the caller + // may release its own reference in the meantime + recordingData.retain(); + retainedRecordings.add(recordingData); + Path file = recordingData.getPath(); + if (file != null) { + try { + return addFile(file, recordingData.getStart(), recordingData.getEnd()); + } catch (RuntimeException | Error e) { + rollbackRecording(recordingData); + throw e; + } + } + try (InputStream stream = recordingData.getStream()) { + return addStream(stream, recordingData.getStart(), recordingData.getEnd()); + } catch (IOException | RuntimeException | Error e) { + rollbackRecording(recordingData); + throw e; + } + } + + // a failed registration must not leave the recording retained forever — drop the retained + // reference so the caller's own release can retire the backing resource + private void rollbackRecording(RecordingData recordingData) { + retainedRecordings.remove(recordingData); + recordingData.release(); + } + + public JfrToOtlpConverter addFile(Path jfrFile, Instant start, Instant end) { + return addPathEntry(new PathEntry(jfrFile, false), start, end); + } + + // the parser requires file access, so the stream is copied to a temporary file; + // the copy is reclaimed by reset() in convert()'s finally. deleteOnExit is a safety net for + // the window where the converter is abandoned before convert() runs. + public JfrToOtlpConverter addStream(InputStream jfrStream, Instant start, Instant end) + throws IOException { + Path tempFile = Files.createTempFile("jfr-convert-", ".jfr"); + tempFile.toFile().deleteOnExit(); + try { + Files.copy(jfrStream, tempFile, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException | RuntimeException e) { + // the copy failed before the entry was registered — reclaim the temp file here; + // deleteOnExit only covers abandonment after successful registration + try { + Files.deleteIfExists(tempFile); + } catch (IOException ignored) { + // best effort; deleteOnExit still reclaims the file at JVM exit + } + throw e; + } + return addPathEntry(new PathEntry(tempFile, true), start, end); + } + + private JfrToOtlpConverter addPathEntry(PathEntry pathEntry, Instant start, Instant end) { + updateTimeRange(start, end); + pathEntries.add(pathEntry); + return this; + } + + // merges all added recordings into a single ProfilesData with shared dictionary tables; + // resets the converter afterwards so it is ready for reuse + public byte[] convert(Kind kind) throws IOException { + try { + for (PathEntry pathEntry : pathEntries) { + parseJfrEvents(pathEntry.path); + } + + // override the caller-supplied window with the actual event range — the profile bounds + // are metadata only (samples are never filtered by them) + if (deriveTimeBoundsFromEvents && minEventTimestampNanos <= maxEventTimestampNanos) { + startTimeNanos = minEventTimestampNanos; + endTimeNanos = maxEventTimestampNanos; + } + + switch (kind) { + case JSON: + return encodeProfilesDataAsJson(false); + case JSON_PRETTY: + return encodeProfilesDataAsJson(true); + case PROTO: + default: + return encodeProfilesData(); + } + } finally { + reset(); + } + } + + public byte[] convert() throws IOException { + return convert(Kind.PROTO); + } + + public void reset() { + // release the recordings retained by addRecording(RecordingData) + for (RecordingData recording : retainedRecordings) { + recording.release(); + } + retainedRecordings.clear(); + // remove any ephemeral files even in case of exception + pathEntries.stream() + .filter(e -> e.ephemeral) + .forEach( + e -> { + try { + Files.deleteIfExists(e.path); + } catch (IOException ignored) { + } + }); + pathEntries.clear(); + stringTable.reset(); + functionTable.reset(); + locationTable.reset(); + stackTable.reset(); + linkTable.reset(); + attributeTable.reset(); + stackTraceCache.clear(); + frameCache.clear(); + cpuAttrIndices = null; + wallAttrIndices = null; + lockAttrIndices = null; + allocNoClassAttrIndices = null; + allocProbeChunkOrdinal = -1; + allocAttrCache.clear(); + lastChunk = null; + chunkOrdinal = 0; + parsingContext = new ParsingContextImpl(); + minEventTimestampNanos = Long.MAX_VALUE; + maxEventTimestampNanos = Long.MIN_VALUE; + originalPayloadWritten = false; + cpuSamples.clear(); + wallSamples.clear(); + allocSamples.clear(); + lockSamples.clear(); + startTimeNanos = 0; + endTimeNanos = 0; + timeRangeInitialized = false; + } + + // chains the added JFR files into a single stream; JFR parsers process concatenated + // recordings sequentially + private InputStream createJfrPayloadStream() throws IOException { + if (pathEntries.isEmpty()) { + return null; + } + + if (pathEntries.size() == 1) { + return Files.newInputStream(pathEntries.iterator().next().path); + } + + java.util.Enumeration streams = + new java.util.Enumeration() { + private final java.util.Iterator iterator = pathEntries.iterator(); + + @Override + public boolean hasMoreElements() { + return iterator.hasNext(); + } + + @Override + public InputStream nextElement() { + try { + return Files.newInputStream(iterator.next().path); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }; + + return new java.io.SequenceInputStream(streams); + } + + private void updateTimeRange(Instant start, Instant end) { + long startNanos = start.getEpochSecond() * 1_000_000_000L + start.getNano(); + long endNanos = end.getEpochSecond() * 1_000_000_000L + end.getNano(); + + // 0 is a legitimate timestamp (Instant.EPOCH), so track the unset state explicitly for both + // bounds — an end at/before the epoch would otherwise leave the end bound at 0 + if (!timeRangeInitialized || startNanos < startTimeNanos) { + startTimeNanos = startNanos; + timeRangeInitialized = true; + } + if (!timeRangeInitialized || endNanos > endTimeNanos) { + endTimeNanos = endNanos; + } + } + + private void parseJfrEvents(Path jfrFile) throws IOException { + try (TypedJafarParser parser = TypedJafarParser.open(jfrFile, parsingContext)) { + parser.handle(ExecutionSample.class, this::handleExecutionSample); + parser.handle(JdkExecutionSample.class, this::handleJdkExecutionSample); + parser.handle(JdkCPUTimeSample.class, this::handleJdkCPUTimeSample); + parser.handle(JdkNativeMethodSample.class, this::handleJdkNativeMethodSample); + parser.handle(MethodSample.class, this::handleMethodSample); + parser.handle(ObjectSample.class, this::handleObjectSample); + parser.handle(JavaMonitorEnter.class, this::handleMonitorEnter); + parser.handle(JavaMonitorWait.class, this::handleMonitorWait); + + parser.run(); + } catch (Exception e) { + throw new IOException(e); + } + } + + private void handleExecutionSample(ExecutionSample event, Control ctl) { + if (event == null) { + return; + } + int stackIndex = convertStackTrace(event::stackTrace, event.stackTraceId(), ctl); + int linkIndex = extractLinkIndex(event.spanId(), event.localRootSpanId()); + long timestamp = convertTimestamp(event.startTime(), ctl); + + if (cpuAttrIndices == null) cpuAttrIndices = new int[] {getSampleTypeAttributeIndex("cpu")}; + cpuSamples.add(new SampleData(stackIndex, linkIndex, 1, timestamp, cpuAttrIndices)); + } + + // Standard OpenJDK sampling events carry no span correlation fields, so the link index stays 0 + private void handleJdkExecutionSample(JdkExecutionSample event, Control ctl) { + if (event == null) { + return; + } + int stackIndex = convertStackTrace(event::stackTrace, event.stackTraceId(), ctl); + long timestamp = convertTimestamp(event.startTime(), ctl); + + if (cpuAttrIndices == null) cpuAttrIndices = new int[] {getSampleTypeAttributeIndex("cpu")}; + cpuSamples.add(new SampleData(stackIndex, 0, 1, timestamp, cpuAttrIndices)); + } + + private void handleJdkCPUTimeSample(JdkCPUTimeSample event, Control ctl) { + if (event == null) { + return; + } + int stackIndex = convertStackTrace(event::stackTrace, event.stackTraceId(), ctl); + long timestamp = convertTimestamp(event.startTime(), ctl); + + if (cpuAttrIndices == null) cpuAttrIndices = new int[] {getSampleTypeAttributeIndex("cpu")}; + cpuSamples.add(new SampleData(stackIndex, 0, 1, timestamp, cpuAttrIndices)); + } + + // native-state samples count as CPU samples: they are the primary sample source in recordings + // produced by the OpenJDK profiler without ddprof + private void handleJdkNativeMethodSample(JdkNativeMethodSample event, Control ctl) { + if (event == null) { + return; + } + int stackIndex = convertStackTrace(event::stackTrace, event.stackTraceId(), ctl); + long timestamp = convertTimestamp(event.startTime(), ctl); + + if (cpuAttrIndices == null) cpuAttrIndices = new int[] {getSampleTypeAttributeIndex("cpu")}; + cpuSamples.add(new SampleData(stackIndex, 0, 1, timestamp, cpuAttrIndices)); + } + + private void handleMethodSample(MethodSample event, Control ctl) { + if (event == null) { + return; + } + int stackIndex = convertStackTrace(event::stackTrace, event.stackTraceId(), ctl); + int linkIndex = extractLinkIndex(event.spanId(), event.localRootSpanId()); + long timestamp = convertTimestamp(event.startTime(), ctl); + + if (wallAttrIndices == null) wallAttrIndices = new int[] {getSampleTypeAttributeIndex("wall")}; + wallSamples.add(new SampleData(stackIndex, linkIndex, 1, timestamp, wallAttrIndices)); + } + + private void handleObjectSample(ObjectSample event, Control ctl) { + if (event == null) { + return; + } + int stackIndex = convertStackTrace(event::stackTrace, event.stackTraceId(), ctl); + int linkIndex = extractLinkIndex(event.spanId(), event.localRootSpanId()); + long timestamp = convertTimestamp(event.startTime(), ctl); + int chunkOrdinal = getChunkOrdinal(ctl); + + // Probe size/weight field availability per chunk (field layout can differ between chunks), + // reusing the result within the chunk to avoid per-sample exception overhead + long size; + float weight; + if (chunkOrdinal != allocProbeChunkOrdinal) { + allocProbeChunkOrdinal = chunkOrdinal; + try { + size = event.size(); + weight = event.weight(); + if (size == 0 && weight == 0) { + size = event.allocationSize(); + weight = 1; + } + allocHasSizeWeight = true; + } catch (Exception e) { + size = event.allocationSize(); + weight = 1; + allocHasSizeWeight = false; + } + } else if (allocHasSizeWeight) { + size = event.size(); + weight = event.weight(); + if (size == 0 && weight == 0) { + size = event.allocationSize(); + weight = 1; + } + } else { + size = event.allocationSize(); + weight = 1; + } + + long upscaledSize = Math.round(size * weight); + + // Build attributes: sample.type + alloc.class (if available) + int sampleTypeIndex = getSampleTypeAttributeIndex("alloc"); + String className = null; + try { + className = event.objectClass().name(); + } catch (Exception ignored) { + // objectClass field doesn't exist in this JFR event - skip it + } + + int[] attributeIndices; + if (className != null && !className.isEmpty()) { + // Cache the attribute indices array per class name to avoid per-sample allocation. + attributeIndices = allocAttrCache.get(className); + if (attributeIndices == null) { + int keyIndex = stringTable.intern("alloc.class"); + int classAttrIndex = attributeTable.internString(keyIndex, className, 0); + attributeIndices = new int[] {sampleTypeIndex, classAttrIndex}; + allocAttrCache.put(className, attributeIndices); + } + } else { + // No class name: use the dedicated sample-type-only array, never a string-keyed cache + // entry, so an empty-string class name cannot alias with this case + if (allocNoClassAttrIndices == null) { + allocNoClassAttrIndices = new int[] {sampleTypeIndex}; + } + attributeIndices = allocNoClassAttrIndices; + } + + allocSamples.add( + new SampleData(stackIndex, linkIndex, upscaledSize, timestamp, attributeIndices)); + } + + private void handleMonitorEnter(JavaMonitorEnter event, Control ctl) { + if (event == null) { + return; + } + int stackIndex = convertStackTrace(event::stackTrace, event.stackTraceId(), ctl); + long timestamp = convertTimestamp(event.startTime(), ctl); + long durationNanos = ctl.chunkInfo().asDuration(event.duration()).toNanos(); + + // linkIndex stays 0 (no link): the jdk.JavaMonitorEnter/Wait JFR event types carry no + // spanId/localRootSpanId fields, so lock samples cannot be correlated with traces + if (lockAttrIndices == null) + lockAttrIndices = new int[] {getSampleTypeAttributeIndex("lock-contention")}; + lockSamples.add(new SampleData(stackIndex, 0, durationNanos, timestamp, lockAttrIndices)); + } + + private void handleMonitorWait(JavaMonitorWait event, Control ctl) { + if (event == null) { + return; + } + int stackIndex = convertStackTrace(event::stackTrace, event.stackTraceId(), ctl); + long timestamp = convertTimestamp(event.startTime(), ctl); + long durationNanos = ctl.chunkInfo().asDuration(event.duration()).toNanos(); + + // linkIndex stays 0 (no link): the jdk.JavaMonitorEnter/Wait JFR event types carry no + // spanId/localRootSpanId fields, so lock samples cannot be correlated with traces + if (lockAttrIndices == null) + lockAttrIndices = new int[] {getSampleTypeAttributeIndex("lock-contention")}; + lockSamples.add(new SampleData(stackIndex, 0, durationNanos, timestamp, lockAttrIndices)); + } + + private int getChunkOrdinal(Control ctl) { + // jafar TypedJafarParser invokes the handle* callbacks on a single thread per parser run, + // so this check-then-act update needs no synchronization + Object chunk = ctl.chunkInfo(); + if (chunk != lastChunk) { + lastChunk = chunk; + chunkOrdinal++; + } + return chunkOrdinal; + } + + @SuppressFBWarnings("DCN_NULLPOINTER_EXCEPTION") + private JfrStackTrace safeGetStackTrace(java.util.function.Supplier supplier) { + try { + return supplier.get(); + } catch (NullPointerException e) { + return null; + } + } + + private int convertStackTrace( + java.util.function.Supplier stackTraceSupplier, + long stackTraceId, + Control ctl) { + // tag the cache key with the chunk ordinal so per-chunk ids don't collide across chunks + int chunkOrdinal = getChunkOrdinal(ctl); + long cacheKey = mixKey(stackTraceId, chunkOrdinal); + + int cachedIndex = stackTraceCache.get1(cacheKey); + if (cachedIndex != -1) { + return cachedIndex; + } + + JfrStackTrace stackTrace = safeGetStackTrace(stackTraceSupplier); + if (stackTrace == null) { + stackTraceCache.put1(cacheKey, 0); + return 0; + } + + JfrStackFrame[] frames = stackTrace.frames(); + if (frames == null || frames.length == 0) { + stackTraceCache.put1(cacheKey, 0); + return 0; + } + + int[] locationIndices = new int[frames.length]; + for (int i = 0; i < frames.length; i++) { + locationIndices[i] = convertFrame(frames[i], ctl); + } + + int stackIndex = stackTable.intern(locationIndices); + stackTraceCache.put1(cacheKey, stackIndex); + return stackIndex; + } + + private int convertFrame(JfrStackFrame frame, Control ctl) { + if (frame == null) { + return 0; + } + + JfrMethod method = frame.method(); + if (method == null) { + return 0; + } + + int lineNumber = frame.lineNumber(); + long methodId = frame.methodId(); + + // Cache key mirrors the stackTraceCache pattern: tag methodId with the chunk ordinal + // so per-chunk CP indices don't collide across chunks; line numbers fit well below 2^32 + int chunkOrdinal = getChunkOrdinal(ctl); + long cacheKey = mixKey(methodId, ((long) chunkOrdinal << 32) ^ lineNumber); + int cached = frameCache.get1(cacheKey); + if (cached != -1) { + return cached; + } + + String methodName = method.name(); + JfrClass type = method.type(); + String className = type != null ? type.name() : null; + + long line = Math.max(lineNumber, 0); + + String fullName; + if (className != null && !className.isEmpty()) { + fullName = className + "." + (methodName != null ? methodName : ""); + } else { + fullName = methodName != null ? methodName : ""; + } + + int nameIndex = stringTable.intern(fullName); + int classNameIndex = stringTable.intern(className); + int methodNameIndex = stringTable.intern(methodName); + int functionIndex = functionTable.intern(nameIndex, methodNameIndex, classNameIndex, 0); + int locationIndex = locationTable.intern(0, 0, functionIndex, line, 0); + + frameCache.put1(cacheKey, locationIndex); + return locationIndex; + } + + private static long mixKey(long primary, long secondary) { + long h = primary * 0x9E3779B97F4A7C15L; + h ^= (h >>> 31) ^ secondary; + h *= 0xff51afd7ed558ccdL; + h ^= h >>> 33; + return h; + } + + private int extractLinkIndex(long spanId, long localRootSpanId) { + if (spanId == 0) { + return 0; + } + // the datadog.ExecutionSample JFR events only carry spanId/localRootSpanId, so the link's + // trace_id is synthesized from localRootSpanId (high 8 bytes zero) — backends resolving + // links by (trace_id, span_id) may fail to correlate these samples with the trace + return linkTable.intern(localRootSpanId, spanId); + } + + private int getSampleTypeAttributeIndex(String sampleType) { + int keyIndex = stringTable.intern("sample.type"); + int unitIndex = 0; // No unit for string labels + return attributeTable.internString(keyIndex, sampleType, unitIndex); + } + + private long convertTimestamp(long startTimeTicks, Control ctl) { + if (startTimeTicks == 0) { + return 0; + } + Instant instant = ctl.chunkInfo().asInstant(startTimeTicks); + long nanos = instant.getEpochSecond() * 1_000_000_000L + instant.getNano(); + if (deriveTimeBoundsFromEvents) { + if (nanos < minEventTimestampNanos) { + minEventTimestampNanos = nanos; + } + if (nanos > maxEventTimestampNanos) { + maxEventTimestampNanos = nanos; + } + } + return nanos; + } + + private byte[] encodeProfilesData() throws IOException { + ProtobufEncoder encoder = protoEncoder; + encoder.reset(); + originalPayloadWritten = false; + + // ProfilesData message + // Field 1: resource_profiles (repeated) + encoder.writeNestedMessage( + OtlpProtoFields.ProfilesData.RESOURCE_PROFILES, + enc -> { + try { + encodeResourceProfiles(enc); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + // Field 2: dictionary + encoder.writeNestedMessage(OtlpProtoFields.ProfilesData.DICTIONARY, this::encodeDictionary); + + return encoder.toByteArray(); + } + + private void encodeResourceProfiles(ProtobufEncoder encoder) throws IOException { + // ResourceProfiles message + // Field 1: resource (skipped when no attributes are set, e.g. for offline CLI conversion) + if (!resourceAttributes.isEmpty()) { + OtlpResourceAttributes.writeResource(encoder, resourceAttributes); + } + // Field 2: scope_profiles (repeated) + encoder.writeNestedMessage( + OtlpProtoFields.ResourceProfiles.SCOPE_PROFILES, + enc -> { + try { + encodeScopeProfiles(enc); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + private void encodeScopeProfiles(ProtobufEncoder encoder) throws IOException { + // ScopeProfiles message + // Field 2: profiles (repeated) + // Encode each profile type that has samples + + if (!cpuSamples.isEmpty()) { + encoder.writeNestedMessage( + OtlpProtoFields.ScopeProfiles.PROFILES, + enc -> { + try { + encodeProfile(enc, PROFILE_TYPE_CPU, UNIT_SAMPLES, cpuSamples, true); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + if (!wallSamples.isEmpty()) { + encoder.writeNestedMessage( + OtlpProtoFields.ScopeProfiles.PROFILES, + enc -> { + try { + encodeProfile(enc, PROFILE_TYPE_WALL, UNIT_SAMPLES, wallSamples, true); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + if (!allocSamples.isEmpty()) { + encoder.writeNestedMessage( + OtlpProtoFields.ScopeProfiles.PROFILES, + enc -> { + try { + // event-driven sample type — no period/period_type + encodeProfile(enc, PROFILE_TYPE_ALLOC, UNIT_BYTES, allocSamples, false); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + if (!lockSamples.isEmpty()) { + encoder.writeNestedMessage( + OtlpProtoFields.ScopeProfiles.PROFILES, + enc -> { + try { + // event-driven sample type — no period/period_type + encodeProfile(enc, PROFILE_TYPE_LOCK, UNIT_NANOSECONDS, lockSamples, false); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + } + + private void encodeProfile( + ProtobufEncoder encoder, + String profileType, + String unit, + List samples, + boolean periodic) + throws IOException { + // Profile message + + // Field 1: sample_type + int typeIndex = stringTable.intern(profileType); + int unitIndex = stringTable.intern(unit); + encoder.writeNestedMessage( + OtlpProtoFields.Profile.SAMPLE_TYPE, enc -> encodeValueType(enc, typeIndex, unitIndex)); + + // Field 2: samples (repeated) + for (SampleData sample : samples) { + encoder.writeNestedMessage(OtlpProtoFields.Profile.SAMPLES, enc -> encodeSample(enc, sample)); + } + + // Field 3: time_unix_nano + encoder.writeFixed64Field(OtlpProtoFields.Profile.TIME_UNIX_NANO, startTimeNanos); + + // Field 4: duration_nano — clamped so a reversed window cannot underflow + encoder.writeVarintField( + OtlpProtoFields.Profile.DURATION_NANO, Math.max(0, endTimeNanos - startTimeNanos)); + + // Field 5: period_type (same as sample_type); omitted for event-driven profiles where a + // period=1 would assert a false 1-byte/1-nanosecond sampling period + if (periodic) { + encoder.writeNestedMessage( + OtlpProtoFields.Profile.PERIOD_TYPE, enc -> encodeValueType(enc, typeIndex, unitIndex)); + + // Field 6: period (1 for count-based) + encoder.writeVarintField(OtlpProtoFields.Profile.PERIOD, 1); + } + + // Field 7: profile_id (16 bytes UUID) + byte[] profileId = generateProfileId(); + encoder.writeBytesField(OtlpProtoFields.Profile.PROFILE_ID, profileId); + + // Fields 9 & 10: original_payload_format and original_payload (if enabled) + // embedded only once per ProfilesData — the recording is shared by every emitted profile, + // and repeating it per sample kind would multiply the payload by the profile count + if (includeOriginalPayload && !pathEntries.isEmpty() && !originalPayloadWritten) { + originalPayloadWritten = true; + encoder.writeStringField(OtlpProtoFields.Profile.ORIGINAL_PAYLOAD_FORMAT, "jfr"); + + long totalSize = 0; + for (PathEntry entry : pathEntries) { + totalSize += Files.size(entry.path); + } + + encoder.writeBytesField( + OtlpProtoFields.Profile.ORIGINAL_PAYLOAD, createJfrPayloadStream(), totalSize); + } + } + + private void encodeValueType(ProtobufEncoder encoder, int typeIndex, int unitIndex) { + encoder.writeVarintField(OtlpProtoFields.ValueType.TYPE_STRINDEX, typeIndex); + encoder.writeVarintField(OtlpProtoFields.ValueType.UNIT_STRINDEX, unitIndex); + } + + private void encodeSample(ProtobufEncoder encoder, SampleData sample) { + // Field 1: stack_index + encoder.writeVarintField(OtlpProtoFields.Sample.STACK_INDEX, sample.stackIndex); + + // Field 2: attribute_indices (packed repeated int32 - proto3 default) + if (sample.attributeIndices.length > 0) { + encoder.writePackedVarintField( + OtlpProtoFields.Sample.ATTRIBUTE_INDICES, sample.attributeIndices); + } + + // Field 3: link_index + encoder.writeVarintField(OtlpProtoFields.Sample.LINK_INDEX, sample.linkIndex); + + // Field 4: values (packed) + encoder.writePackedVarintField(OtlpProtoFields.Sample.VALUES, sample.value); + + // Field 5: timestamps_unix_nano (packed) + if (sample.timestampNanos > 0) { + encoder.writePackedFixed64Field( + OtlpProtoFields.Sample.TIMESTAMPS_UNIX_NANO, sample.timestampNanos); + } + } + + private void encodeDictionary(ProtobufEncoder encoder) { + // ProfilesDictionary message + // every table must carry at least its index-0 (null/unset sentinel) entry per the OTLP spec + + // Field 1: mapping_table — locations never reference a mapping, so only the zero-value + // entry at index 0 is written + encoder.writeTag( + OtlpProtoFields.ProfilesDictionary.MAPPING_TABLE, + ProtobufEncoder.WIRETYPE_LENGTH_DELIMITED); + encoder.writeVarint(0); + + // Field 2: location_table + for (int i = 0; i < locationTable.size(); i++) { + final int idx = i; + encoder.writeNestedMessage( + OtlpProtoFields.ProfilesDictionary.LOCATION_TABLE, enc -> encodeLocation(enc, idx)); + } + + // Field 3: function_table + for (int i = 0; i < functionTable.size(); i++) { + final int idx = i; + encoder.writeNestedMessage( + OtlpProtoFields.ProfilesDictionary.FUNCTION_TABLE, enc -> encodeFunction(enc, idx)); + } + + // Field 4: link_table + for (int i = 0; i < linkTable.size(); i++) { + final int idx = i; + encoder.writeNestedMessage( + OtlpProtoFields.ProfilesDictionary.LINK_TABLE, enc -> encodeLink(enc, idx)); + } + + // Field 5: string_table (repeated strings) + // every entry must be written, including the empty index-0 sentinel — writeStringField + // skips empty strings, which would shift all subsequent indices + for (String s : stringTable.getStrings()) { + encoder.writeTag( + OtlpProtoFields.ProfilesDictionary.STRING_TABLE, + ProtobufEncoder.WIRETYPE_LENGTH_DELIMITED); + encoder.writeString(s); + } + + // Field 6: attribute_table + for (int i = 0; i < attributeTable.size(); i++) { + final int idx = i; + encoder.writeNestedMessage( + OtlpProtoFields.ProfilesDictionary.ATTRIBUTE_TABLE, enc -> encodeAttribute(enc, idx)); + } + + // Field 7: stack_table + for (int i = 0; i < stackTable.size(); i++) { + final int idx = i; + encoder.writeNestedMessage( + OtlpProtoFields.ProfilesDictionary.STACK_TABLE, enc -> encodeStack(enc, idx)); + } + } + + private void encodeLocation(ProtobufEncoder encoder, int index) { + LocationTable.LocationEntry entry = locationTable.get(index); + + // Field 1: mapping_index (zero values are skipped by writeVarintField; the index 0 + // sentinel is an empty message, which is valid per proto3) + encoder.writeVarintField(OtlpProtoFields.Location.MAPPING_INDEX, entry.mappingIndex); + + // Field 2: address + encoder.writeVarintField(OtlpProtoFields.Location.ADDRESS, entry.address); + + // Field 3: lines (repeated) + for (LocationTable.LineEntry line : entry.lines) { + encoder.writeNestedMessage(OtlpProtoFields.Location.LINES, enc -> encodeLine(enc, line)); + } + } + + private void encodeLine(ProtobufEncoder encoder, LocationTable.LineEntry line) { + encoder.writeVarintField(OtlpProtoFields.Line.FUNCTION_INDEX, line.functionIndex); + encoder.writeVarintField(OtlpProtoFields.Line.LINE, line.line); + encoder.writeVarintField(OtlpProtoFields.Line.COLUMN, line.column); + } + + private void encodeFunction(ProtobufEncoder encoder, int index) { + FunctionTable.FunctionEntry entry = functionTable.get(index); + + encoder.writeVarintField(OtlpProtoFields.Function.NAME_STRINDEX, entry.nameIndex); + encoder.writeVarintField(OtlpProtoFields.Function.SYSTEM_NAME_STRINDEX, entry.systemNameIndex); + encoder.writeVarintField(OtlpProtoFields.Function.FILENAME_STRINDEX, entry.filenameIndex); + encoder.writeVarintField(OtlpProtoFields.Function.START_LINE, entry.startLine); + } + + private void encodeLink(ProtobufEncoder encoder, int index) { + if (index == 0) { + return; // the index-0 sentinel must be the zero value (empty trace and span ids) + } + LinkTable.LinkEntry entry = linkTable.get(index); + + encoder.writeBytesField(OtlpProtoFields.Link.TRACE_ID, entry.traceId); + encoder.writeBytesField(OtlpProtoFields.Link.SPAN_ID, entry.spanId); + } + + private void encodeStack(ProtobufEncoder encoder, int index) { + StackTable.StackEntry entry = stackTable.get(index); + + // For index 0 (null sentinel), location_indices is empty + // writePackedVarintField handles empty arrays by writing nothing, but writeNestedMessage + // now always writes the message envelope (tag + length=0) even if the content is empty + encoder.writePackedVarintField(OtlpProtoFields.Stack.LOCATION_INDICES, entry.locationIndices); + } + + private void encodeAttribute(ProtobufEncoder encoder, int index) { + if (index == 0) { + return; // the index-0 sentinel must be the zero value (no key, no value) + } + AttributeTable.AttributeEntry entry = attributeTable.get(index); + + // Field 1: key_strindex + encoder.writeVarintField(OtlpProtoFields.KeyValueAndUnit.KEY_STRINDEX, entry.keyIndex); + + // Field 2: value (AnyValue oneof) + encoder.writeNestedMessage( + OtlpProtoFields.KeyValueAndUnit.VALUE, + enc -> { + switch (entry.valueType) { + case STRING: + // write the oneof field even for an empty string — writeStringField would skip + // it entirely, leaving the AnyValue with no field set (proto3 reads that as + // an unset value rather than string_value="") + String stringValue = (String) entry.value; + if (stringValue == null || stringValue.isEmpty()) { + enc.writeTag( + OtlpProtoFields.AnyValue.STRING_VALUE, + ProtobufEncoder.WIRETYPE_LENGTH_DELIMITED); + enc.writeVarint(0); + } else { + enc.writeStringField(OtlpProtoFields.AnyValue.STRING_VALUE, stringValue); + } + break; + case BOOL: + enc.writeBoolField(OtlpProtoFields.AnyValue.BOOL_VALUE, (Boolean) entry.value); + break; + case INT: + // int_value is a plain int64 — standard varint, not zigzag + enc.writeVarintField(OtlpProtoFields.AnyValue.INT_VALUE, (Long) entry.value); + break; + case DOUBLE: + // Note: protobuf doubles are fixed64, not varint + long doubleBits = Double.doubleToRawLongBits((Double) entry.value); + enc.writeFixed64Field(OtlpProtoFields.AnyValue.DOUBLE_VALUE, doubleBits); + break; + } + }); + + // Field 3: unit_strindex + encoder.writeVarintField(OtlpProtoFields.KeyValueAndUnit.UNIT_STRINDEX, entry.unitIndex); + } + + private byte[] generateProfileId() { + // Use ThreadLocalRandom instead of UUID.randomUUID (which uses SecureRandom and can block + // on entropy). Profile IDs are not security-sensitive. + long msb = ThreadLocalRandom.current().nextLong(); + long lsb = ThreadLocalRandom.current().nextLong(); + byte[] bytes = new byte[16]; + for (int i = 0; i < 8; i++) { + bytes[i] = (byte) ((msb >> (56 - i * 8)) & 0xFF); + bytes[i + 8] = (byte) ((lsb >> (56 - i * 8)) & 0xFF); + } + return bytes; + } + + // JSON encoding methods + + // JSON output is a debug format: unlike PROTO it does not carry + // original_payload / original_payload_format, even when includeOriginalPayload is set + private byte[] encodeProfilesDataAsJson(boolean prettyPrint) { + JsonWriter json = new JsonWriter(); + json.beginObject(); + + // resource_profiles array + json.name("resource_profiles").beginArray(); + encodeResourceProfilesJson(json); + json.endArray(); + + // dictionary + json.name("dictionary"); + encodeDictionaryJson(json); + + json.endObject(); + byte[] compactJson = json.toByteArray(); + + return prettyPrint ? prettyPrintJson(compactJson) : compactJson; + } + + private byte[] prettyPrintJson(byte[] compactJson) { + String compact = new String(compactJson, java.nio.charset.StandardCharsets.UTF_8); + StringBuilder pretty = new StringBuilder(compact.length() + compact.length() / 4); + int indent = 0; + boolean inString = false; + boolean escape = false; + + for (int i = 0; i < compact.length(); i++) { + char c = compact.charAt(i); + + if (escape) { + pretty.append(c); + escape = false; + continue; + } + + if (c == '\\') { + pretty.append(c); + escape = true; + continue; + } + + if (c == '"') { + pretty.append(c); + inString = !inString; + continue; + } + + if (inString) { + pretty.append(c); + continue; + } + + switch (c) { + case '{': + case '[': + pretty.append(c).append('\n'); + indent++; + appendIndent(pretty, indent); + break; + case '}': + case ']': + pretty.append('\n'); + indent--; + appendIndent(pretty, indent); + pretty.append(c); + break; + case ',': + pretty.append(c).append('\n'); + appendIndent(pretty, indent); + break; + case ':': + pretty.append(c).append(' '); + break; + default: + if (!Character.isWhitespace(c)) { + pretty.append(c); + } + break; + } + } + + return pretty.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8); + } + + private void appendIndent(StringBuilder sb, int indent) { + for (int i = 0; i < indent * 2; i++) { + sb.append(' '); + } + } + + private void encodeResourceProfilesJson(JsonWriter json) { + json.beginObject(); + + // resource with standard KeyValue attributes + if (!resourceAttributes.isEmpty()) { + json.name("resource").beginObject(); + json.name("attributes").beginArray(); + for (Map.Entry entry : resourceAttributes.entrySet()) { + String value = entry.getValue(); + if (value == null || value.isEmpty()) { + continue; + } + json.beginObject(); + json.name("key").value(entry.getKey()); + json.name("value").beginObject(); + json.name("stringValue").value(value); + json.endObject(); + json.endObject(); + } + json.endArray(); + json.endObject(); + } + + // scope_profiles array + json.name("scope_profiles").beginArray(); + encodeScopeProfilesJson(json); + json.endArray(); + + json.endObject(); + } + + private void encodeScopeProfilesJson(JsonWriter json) { + json.beginObject(); + + // profiles array + json.name("profiles").beginArray(); + + if (!cpuSamples.isEmpty()) { + encodeProfileJson(json, PROFILE_TYPE_CPU, UNIT_SAMPLES, cpuSamples, true); + } + + if (!wallSamples.isEmpty()) { + encodeProfileJson(json, PROFILE_TYPE_WALL, UNIT_SAMPLES, wallSamples, true); + } + + if (!allocSamples.isEmpty()) { + encodeProfileJson(json, PROFILE_TYPE_ALLOC, UNIT_BYTES, allocSamples, false); + } + + if (!lockSamples.isEmpty()) { + encodeProfileJson(json, PROFILE_TYPE_LOCK, UNIT_NANOSECONDS, lockSamples, false); + } + + json.endArray(); + json.endObject(); + } + + private void encodeProfileJson( + JsonWriter json, + String profileType, + String unit, + List samples, + boolean periodic) { + json.beginObject(); + + // sample_type + int typeIndex = stringTable.intern(profileType); + int unitIndex = stringTable.intern(unit); + json.name("sample_type"); + encodeValueTypeJson(json, typeIndex, unitIndex); + + // samples array + json.name("samples").beginArray(); + for (SampleData sample : samples) { + encodeSampleJson(json, sample); + } + json.endArray(); + + // time_unix_nano + json.name("time_unix_nano").value(startTimeNanos); + + // duration_nano — clamped to match the PROTO path + json.name("duration_nano").value(Math.max(0, endTimeNanos - startTimeNanos)); + + // period_type / period: only for periodic profiles, mirroring the PROTO encoding + if (periodic) { + json.name("period_type"); + encodeValueTypeJson(json, typeIndex, unitIndex); + + json.name("period").value(1); + } + + // profile_id (as hex string for readability) + byte[] profileId = generateProfileId(); + StringBuilder hexId = new StringBuilder(32); + for (byte b : profileId) { + hexId.append(String.format("%02x", b)); + } + json.name("profile_id").value(hexId.toString()); + + json.endObject(); + } + + private void encodeValueTypeJson(JsonWriter json, int typeIndex, int unitIndex) { + json.beginObject(); + json.name("type_strindex").value(typeIndex); + json.name("unit_strindex").value(unitIndex); + json.endObject(); + } + + private void encodeSampleJson(JsonWriter json, SampleData sample) { + json.beginObject(); + + // stack_index + json.name("stack_index").value(sample.stackIndex); + + // link_index + if (sample.linkIndex > 0) { + json.name("link_index").value(sample.linkIndex); + } + + // values array + json.name("values").beginArray().value(sample.value).endArray(); + + // attribute_indices array (mirrors the packed proto field; carries sample.type and alloc.class) + if (sample.attributeIndices.length > 0) { + json.name("attribute_indices").beginArray(); + for (int attributeIndex : sample.attributeIndices) { + json.value(attributeIndex); + } + json.endArray(); + } + + // timestamps_unix_nano array + if (sample.timestampNanos > 0) { + json.name("timestamps_unix_nano").beginArray().value(sample.timestampNanos).endArray(); + } + + json.endObject(); + } + + private void encodeDictionaryJson(JsonWriter json) { + json.beginObject(); + + // every table must carry at least its index-0 (null/unset sentinel) entry per the OTLP spec + + // mapping_table array (zero-value entry only, see encodeDictionary) + json.name("mapping_table").beginArray(); + json.beginObject(); + json.endObject(); + json.endArray(); + + // location_table array + json.name("location_table").beginArray(); + for (int i = 0; i < locationTable.size(); i++) { + encodeLocationJson(json, i); + } + json.endArray(); + + // function_table array + json.name("function_table").beginArray(); + for (int i = 0; i < functionTable.size(); i++) { + encodeFunctionJson(json, i); + } + json.endArray(); + + // link_table array + json.name("link_table").beginArray(); + for (int i = 0; i < linkTable.size(); i++) { + encodeLinkJson(json, i); + } + json.endArray(); + + // string_table array + json.name("string_table").beginArray(); + for (String s : stringTable.getStrings()) { + json.value(s); + } + json.endArray(); + + // attribute_table array + json.name("attribute_table").beginArray(); + for (int i = 0; i < attributeTable.size(); i++) { + encodeAttributeJson(json, i); + } + json.endArray(); + + // stack_table array + json.name("stack_table").beginArray(); + for (int i = 0; i < stackTable.size(); i++) { + encodeStackJson(json, i); + } + json.endArray(); + + json.endObject(); + } + + private void encodeLocationJson(JsonWriter json, int index) { + LocationTable.LocationEntry entry = locationTable.get(index); + json.beginObject(); + + // mapping_index + json.name("mapping_index").value(entry.mappingIndex); + + // address + json.name("address").value(entry.address); + + // lines array + json.name("lines").beginArray(); + for (LocationTable.LineEntry line : entry.lines) { + encodeLineJson(json, line); + } + json.endArray(); + + json.endObject(); + } + + private void encodeLineJson(JsonWriter json, LocationTable.LineEntry line) { + json.beginObject(); + json.name("function_index").value(line.functionIndex); + json.name("line").value(line.line); + if (line.column > 0) { + json.name("column").value(line.column); + } + json.endObject(); + } + + private void encodeFunctionJson(JsonWriter json, int index) { + FunctionTable.FunctionEntry entry = functionTable.get(index); + json.beginObject(); + + json.name("name_strindex").value(entry.nameIndex); + json.name("system_name_strindex").value(entry.systemNameIndex); + json.name("filename_strindex").value(entry.filenameIndex); + if (entry.startLine > 0) { + json.name("start_line").value(entry.startLine); + } + + json.endObject(); + } + + private void encodeLinkJson(JsonWriter json, int index) { + json.beginObject(); + if (index == 0) { + json.endObject(); + return; + } + LinkTable.LinkEntry entry = linkTable.get(index); + + // Encode trace_id and span_id as hex strings for readability + StringBuilder traceIdHex = new StringBuilder(32); + for (byte b : entry.traceId) { + traceIdHex.append(String.format("%02x", b)); + } + json.name("trace_id").value(traceIdHex.toString()); + + StringBuilder spanIdHex = new StringBuilder(16); + for (byte b : entry.spanId) { + spanIdHex.append(String.format("%02x", b)); + } + json.name("span_id").value(spanIdHex.toString()); + + json.endObject(); + } + + private void encodeAttributeJson(JsonWriter json, int index) { + json.beginObject(); + if (index == 0) { + json.endObject(); + return; + } + AttributeTable.AttributeEntry entry = attributeTable.get(index); + + // key_strindex + json.name("key_strindex").value(entry.keyIndex); + + // value object (AnyValue) + json.name("value").beginObject(); + switch (entry.valueType) { + case STRING: + json.name("string_value").value((String) entry.value); + break; + case BOOL: + json.name("bool_value").value((Boolean) entry.value); + break; + case INT: + json.name("int_value").value((Long) entry.value); + break; + case DOUBLE: + json.name("double_value").value((Double) entry.value); + break; + } + json.endObject(); + + // unit_strindex + json.name("unit_strindex").value(entry.unitIndex); + + json.endObject(); + } + + private void encodeStackJson(JsonWriter json, int index) { + StackTable.StackEntry entry = stackTable.get(index); + json.beginObject(); + + // location_indices array + json.name("location_indices").beginArray(); + for (int locationIndex : entry.locationIndices) { + json.value(locationIndex); + } + json.endArray(); + + json.endObject(); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/JfrToOtlpConverterCLI.java b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/JfrToOtlpConverterCLI.java new file mode 100644 index 00000000000..3b2f5dfd22e --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/JfrToOtlpConverterCLI.java @@ -0,0 +1,196 @@ +package com.datadog.profiling.otel; + +import de.thetaphi.forbiddenapis.SuppressForbidden; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; + +/** + * Command-line interface for converting JFR recordings to OTLP profiles format. + * + *

Usage: + * + *

+ * # Convert single JFR file to protobuf (default)
+ * java -cp ... com.datadog.profiling.otel.JfrToOtlpConverterCLI input.jfr output.pb
+ *
+ * # Convert to JSON format (compact)
+ * java -cp ... com.datadog.profiling.otel.JfrToOtlpConverterCLI --json input.jfr output.json
+ *
+ * # Convert to pretty-printed JSON
+ * java -cp ... com.datadog.profiling.otel.JfrToOtlpConverterCLI --json --pretty input.jfr output.json
+ *
+ * # Include original JFR payload
+ * java -cp ... com.datadog.profiling.otel.JfrToOtlpConverterCLI --include-payload input.jfr output.pb
+ *
+ * # Convert multiple JFR files into single output
+ * java -cp ... com.datadog.profiling.otel.JfrToOtlpConverterCLI file1.jfr file2.jfr output.pb
+ * 
+ */ +@SuppressForbidden // command-line tool: prints to stdout/stderr +public class JfrToOtlpConverterCLI { + + private static final int EXIT_USAGE_ERROR = 2; + + public static void main(String[] args) { + if (args.length < 2) { + printUsage(); + System.exit(EXIT_USAGE_ERROR); + } + + try { + new JfrToOtlpConverterCLI().run(args); + } catch (IllegalArgumentException e) { + System.err.println("Usage error: " + e.getMessage()); + printUsage(); + System.exit(EXIT_USAGE_ERROR); + } catch (Exception e) { + System.err.println("Error: " + e.getMessage()); + e.printStackTrace(); + throw new RuntimeException("Conversion failed", e); + } + } + + private void run(String[] args) throws IOException { + // flags are recognized anywhere in the argument list — a trailing --help must not be + // mistaken for an input file name + for (String arg : args) { + if (arg.equals("--help") || arg.equals("-h")) { + printUsage(); + return; + } + } + + JfrToOtlpConverter.Kind outputKind = JfrToOtlpConverter.Kind.PROTO; + boolean includePayload = false; + boolean prettyPrint = false; + int firstInputIndex = 0; + + while (firstInputIndex < args.length && args[firstInputIndex].startsWith("--")) { + String flag = args[firstInputIndex]; + switch (flag) { + case "--json": + outputKind = JfrToOtlpConverter.Kind.JSON; + firstInputIndex++; + break; + case "--pretty": + prettyPrint = true; + firstInputIndex++; + break; + case "--include-payload": + includePayload = true; + firstInputIndex++; + break; + case "--help": + printUsage(); + return; + default: + throw new IllegalArgumentException("Unknown flag: " + flag); + } + } + + // Apply pretty-printing to JSON output; --pretty implies --json + if (prettyPrint && outputKind == JfrToOtlpConverter.Kind.PROTO) { + outputKind = JfrToOtlpConverter.Kind.JSON; + } + if (prettyPrint && outputKind == JfrToOtlpConverter.Kind.JSON) { + outputKind = JfrToOtlpConverter.Kind.JSON_PRETTY; + } + + // Remaining args: input1.jfr [input2.jfr ...] output.pb/json + if (args.length - firstInputIndex < 2) { + throw new IllegalArgumentException("At least one input file and one output file required"); + } + + Path outputPath = Paths.get(args[args.length - 1]); + + Path[] inputPaths = new Path[args.length - firstInputIndex - 1]; + for (int i = 0; i < inputPaths.length; i++) { + inputPaths[i] = Paths.get(args[firstInputIndex + i]); + if (!Files.exists(inputPaths[i])) { + throw new IOException("Input file not found: " + inputPaths[i]); + } + } + + System.out.println("Converting " + inputPaths.length + " JFR file(s) to OTLP format..."); + long startTime = System.currentTimeMillis(); + + JfrToOtlpConverter converter = new JfrToOtlpConverter(); + converter.setIncludeOriginalPayload(includePayload); + // the recording's own time range is only known after parsing, so the profile window is + // derived from the observed event timestamps instead of a synthetic caller-supplied window + converter.setDeriveTimeBoundsFromEvents(true); + + Instant end = Instant.now(); + Instant start = Instant.EPOCH; + + for (Path input : inputPaths) { + System.out.println(" Adding: " + input); + converter.addFile(input, start, end); + } + + byte[] result = converter.convert(outputKind); + Files.write(outputPath, result); + + long elapsed = System.currentTimeMillis() - startTime; + + System.out.println("Conversion complete!"); + System.out.println(" Output: " + outputPath); + System.out.println(" Format: " + outputKind); + System.out.println(" Size: " + formatBytes(result.length)); + System.out.println(" Time: " + elapsed + " ms"); + + if (includePayload) { + long totalInputSize = 0; + for (Path input : inputPaths) { + totalInputSize += Files.size(input); + } + System.out.println(" Input size: " + formatBytes(totalInputSize)); + if (totalInputSize > 0) { + System.out.println( + " Compression: " + String.format("%.1f%%", 100.0 * result.length / totalInputSize)); + } + } + } + + private static String formatBytes(long bytes) { + if (bytes < 1024) { + return bytes + " B"; + } else if (bytes < 1024 * 1024) { + return String.format("%.1f KB", bytes / 1024.0); + } else { + return String.format("%.1f MB", bytes / (1024.0 * 1024.0)); + } + } + + private static void printUsage() { + System.out.println("JFR to OTLP Converter"); + System.out.println(); + System.out.println("Usage: JfrToOtlpConverterCLI [options] input.jfr [input2.jfr ...] output"); + System.out.println(); + System.out.println("Options:"); + System.out.println(" --json Output JSON format instead of protobuf"); + System.out.println(" --pretty Pretty-print JSON output (implies --json)"); + System.out.println( + " --include-payload Include original JFR payload in output (increases size)"); + System.out.println(" --help Show this help message"); + System.out.println(); + System.out.println("Examples:"); + System.out.println(" # Convert to protobuf (default)"); + System.out.println(" JfrToOtlpConverterCLI recording.jfr output.pb"); + System.out.println(); + System.out.println(" # Convert to compact JSON"); + System.out.println(" JfrToOtlpConverterCLI --json recording.jfr output.json"); + System.out.println(); + System.out.println(" # Convert to pretty-printed JSON"); + System.out.println(" JfrToOtlpConverterCLI --json --pretty recording.jfr output.json"); + System.out.println(); + System.out.println(" # Merge multiple recordings"); + System.out.println(" JfrToOtlpConverterCLI file1.jfr file2.jfr file3.jfr merged.pb"); + System.out.println(); + System.out.println(" # Include original payload"); + System.out.println(" JfrToOtlpConverterCLI --include-payload recording.jfr output.pb"); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/LongKeyIntMap.java b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/LongKeyIntMap.java new file mode 100644 index 00000000000..6df26f37a74 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/LongKeyIntMap.java @@ -0,0 +1,141 @@ +package com.datadog.profiling.otel; + +import java.util.Arrays; + +/** + * Open-addressing hash map from a fixed number of long key components to an int value, avoiding + * boxing in the profile conversion hot path. Any long value is a legal key component (occupancy is + * tracked in a separate array, so there are no sentinel keys). + */ +public final class LongKeyIntMap { + private final int arity; + // arity longs per slot, laid out consecutively: slot i uses keys[i * arity .. i * arity + arity) + private long[] keys; + private int[] values; + private byte[] used; + private int mask; + private int size; + + public LongKeyIntMap(int arity, int initialCapacity) { + if (arity < 1) { + throw new IllegalArgumentException("arity must be positive: " + arity); + } + int cap = Integer.highestOneBit(Math.max(initialCapacity * 2, 16) - 1) << 1; + this.arity = arity; + keys = new long[cap * arity]; + values = new int[cap]; + used = new byte[cap]; + mask = cap - 1; + } + + /** Lookup with a single key component. Returns -1 when absent. */ + public int get1(long k1) { + int slot = (int) (mix(k1) & mask); + while (used[slot] != 0) { + if (keys[slot] == k1) { + return values[slot]; + } + slot = (slot + 1) & mask; + } + return -1; + } + + /** Insert or replace with a single key component. */ + public void put1(long k1, int value) { + if (size * 2 > mask) { + resize(); + } + int slot = (int) (mix(k1) & mask); + while (used[slot] != 0) { + if (keys[slot] == k1) { + values[slot] = value; + return; + } + slot = (slot + 1) & mask; + } + used[slot] = 1; + keys[slot] = k1; + values[slot] = value; + size++; + } + + /** Lookup with three key components. Returns -1 when absent. */ + public int get3(long k1, long k2, long k3) { + int slot = (int) (mix3(k1, k2, k3) & mask); + int base = slot * arity; + while (used[slot] != 0) { + if (keys[base] == k1 && keys[base + 1] == k2 && keys[base + 2] == k3) { + return values[slot]; + } + slot = (slot + 1) & mask; + base = slot * arity; + } + return -1; + } + + /** Insert or replace with three key components. */ + public void put3(long k1, long k2, long k3, int value) { + if (size * 2 > mask) { + resize(); + } + int slot = (int) (mix3(k1, k2, k3) & mask); + int base = slot * arity; + while (used[slot] != 0) { + if (keys[base] == k1 && keys[base + 1] == k2 && keys[base + 2] == k3) { + values[slot] = value; + return; + } + slot = (slot + 1) & mask; + base = slot * arity; + } + used[slot] = 1; + keys[base] = k1; + keys[base + 1] = k2; + keys[base + 2] = k3; + values[slot] = value; + size++; + } + + public void clear() { + Arrays.fill(used, (byte) 0); + size = 0; + } + + public int size() { + return size; + } + + private void resize() { + long[] oldKeys = keys; + int[] oldValues = values; + byte[] oldUsed = used; + int oldCap = mask + 1; + int newCap = oldCap * 2; + keys = new long[newCap * arity]; + values = new int[newCap]; + used = new byte[newCap]; + mask = newCap - 1; + size = 0; + for (int i = 0; i < oldCap; i++) { + if (oldUsed[i] != 0) { + int base = i * arity; + if (arity == 1) { + put1(oldKeys[base], oldValues[i]); + } else { + put3(oldKeys[base], oldKeys[base + 1], oldKeys[base + 2], oldValues[i]); + } + } + } + } + + private static long mix(long key) { + key ^= key >>> 33; + key *= 0xff51afd7ed558ccdL; + key ^= key >>> 33; + return key; + } + + private static long mix3(long k1, long k2, long k3) { + return mix(mix(k1) ^ k2) ^ mix(k3); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/ExecutionSample.java b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/ExecutionSample.java new file mode 100644 index 00000000000..7651538f5ce --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/ExecutionSample.java @@ -0,0 +1,19 @@ +package com.datadog.profiling.otel.jfr; + +import io.jafar.parser.api.JfrField; +import io.jafar.parser.api.JfrType; + +@JfrType("datadog.ExecutionSample") +public interface ExecutionSample { + long startTime(); + + @JfrField("stackTrace") + JfrStackTrace stackTrace(); + + @JfrField(value = "stackTrace", raw = true) + long stackTraceId(); + + long spanId(); + + long localRootSpanId(); +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JavaMonitorEnter.java b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JavaMonitorEnter.java new file mode 100644 index 00000000000..fc6f28d07fb --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JavaMonitorEnter.java @@ -0,0 +1,17 @@ +package com.datadog.profiling.otel.jfr; + +import io.jafar.parser.api.JfrField; +import io.jafar.parser.api.JfrType; + +@JfrType("jdk.JavaMonitorEnter") +public interface JavaMonitorEnter { + long startTime(); + + long duration(); + + @JfrField("stackTrace") + JfrStackTrace stackTrace(); + + @JfrField(value = "stackTrace", raw = true) + long stackTraceId(); +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JavaMonitorWait.java b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JavaMonitorWait.java new file mode 100644 index 00000000000..c586bedc9c8 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JavaMonitorWait.java @@ -0,0 +1,17 @@ +package com.datadog.profiling.otel.jfr; + +import io.jafar.parser.api.JfrField; +import io.jafar.parser.api.JfrType; + +@JfrType("jdk.JavaMonitorWait") +public interface JavaMonitorWait { + long startTime(); + + long duration(); + + @JfrField("stackTrace") + JfrStackTrace stackTrace(); + + @JfrField(value = "stackTrace", raw = true) + long stackTraceId(); +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JdkCPUTimeSample.java b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JdkCPUTimeSample.java new file mode 100644 index 00000000000..942a948d119 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JdkCPUTimeSample.java @@ -0,0 +1,20 @@ +package com.datadog.profiling.otel.jfr; + +import io.jafar.parser.api.JfrField; +import io.jafar.parser.api.JfrType; + +/** + * Standard OpenJDK CPU-time sampling event. The OpenJDK profiler records it instead of {@link + * JdkExecutionSample} on JDK 25+ on Linux; like {@link JdkExecutionSample} it carries no span + * correlation fields. + */ +@JfrType("jdk.CPUTimeSample") +public interface JdkCPUTimeSample { + long startTime(); + + @JfrField("stackTrace") + JfrStackTrace stackTrace(); + + @JfrField(value = "stackTrace", raw = true) + long stackTraceId(); +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JdkExecutionSample.java b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JdkExecutionSample.java new file mode 100644 index 00000000000..397fa3a4219 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JdkExecutionSample.java @@ -0,0 +1,20 @@ +package com.datadog.profiling.otel.jfr; + +import io.jafar.parser.api.JfrField; +import io.jafar.parser.api.JfrType; + +/** + * Standard OpenJDK CPU sampling event. It appears in recordings produced without ddprof — e.g. when + * the OpenJDK profiler is the only controller-supplied sample source — and carries no span + * correlation fields. + */ +@JfrType("jdk.ExecutionSample") +public interface JdkExecutionSample { + long startTime(); + + @JfrField("stackTrace") + JfrStackTrace stackTrace(); + + @JfrField(value = "stackTrace", raw = true) + long stackTraceId(); +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JdkNativeMethodSample.java b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JdkNativeMethodSample.java new file mode 100644 index 00000000000..a2cc843246f --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JdkNativeMethodSample.java @@ -0,0 +1,20 @@ +package com.datadog.profiling.otel.jfr; + +import io.jafar.parser.api.JfrField; +import io.jafar.parser.api.JfrType; + +/** + * Standard OpenJDK native-method sampling event. It appears in recordings produced without ddprof + * and captures threads sampled in native state; like {@link JdkExecutionSample} it carries no span + * correlation fields. + */ +@JfrType("jdk.NativeMethodSample") +public interface JdkNativeMethodSample { + long startTime(); + + @JfrField("stackTrace") + JfrStackTrace stackTrace(); + + @JfrField(value = "stackTrace", raw = true) + long stackTraceId(); +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JfrClass.java b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JfrClass.java new file mode 100644 index 00000000000..1e1cbe9f84d --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JfrClass.java @@ -0,0 +1,8 @@ +package com.datadog.profiling.otel.jfr; + +import io.jafar.parser.api.JfrType; + +@JfrType("java.lang.Class") +public interface JfrClass { + String name(); +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JfrMethod.java b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JfrMethod.java new file mode 100644 index 00000000000..c4c9f0de333 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JfrMethod.java @@ -0,0 +1,10 @@ +package com.datadog.profiling.otel.jfr; + +import io.jafar.parser.api.JfrType; + +@JfrType("jdk.types.Method") +public interface JfrMethod { + JfrClass type(); + + String name(); +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JfrStackFrame.java b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JfrStackFrame.java new file mode 100644 index 00000000000..3da464a8042 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JfrStackFrame.java @@ -0,0 +1,14 @@ +package com.datadog.profiling.otel.jfr; + +import io.jafar.parser.api.JfrField; +import io.jafar.parser.api.JfrType; + +@JfrType("jdk.types.StackFrame") +public interface JfrStackFrame { + JfrMethod method(); + + @JfrField(value = "method", raw = true) + long methodId(); + + int lineNumber(); +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JfrStackTrace.java b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JfrStackTrace.java new file mode 100644 index 00000000000..6a998a4e42d --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/JfrStackTrace.java @@ -0,0 +1,8 @@ +package com.datadog.profiling.otel.jfr; + +import io.jafar.parser.api.JfrType; + +@JfrType("jdk.types.StackTrace") +public interface JfrStackTrace { + JfrStackFrame[] frames(); +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/MethodSample.java b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/MethodSample.java new file mode 100644 index 00000000000..aebbf432f3c --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/MethodSample.java @@ -0,0 +1,19 @@ +package com.datadog.profiling.otel.jfr; + +import io.jafar.parser.api.JfrField; +import io.jafar.parser.api.JfrType; + +@JfrType("datadog.MethodSample") +public interface MethodSample { + long startTime(); + + @JfrField("stackTrace") + JfrStackTrace stackTrace(); + + @JfrField(value = "stackTrace", raw = true) + long stackTraceId(); + + long spanId(); + + long localRootSpanId(); +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/ObjectSample.java b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/ObjectSample.java new file mode 100644 index 00000000000..d2572b311f9 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/ObjectSample.java @@ -0,0 +1,30 @@ +package com.datadog.profiling.otel.jfr; + +import io.jafar.parser.api.JfrField; +import io.jafar.parser.api.JfrType; + +@JfrType("datadog.ObjectSample") +public interface ObjectSample { + long startTime(); + + @JfrField("stackTrace") + JfrStackTrace stackTrace(); + + @JfrField(value = "stackTrace", raw = true) + long stackTraceId(); + + long spanId(); + + long localRootSpanId(); + + long allocationSize(); + + @JfrField("objectClass") + JfrClass objectClass(); + + @JfrField("size") + long size(); + + @JfrField("weight") + float weight(); +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/package-info.java b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/package-info.java new file mode 100644 index 00000000000..f3a5202215b --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/jfr/package-info.java @@ -0,0 +1,5 @@ +/** + * JFR event type definitions for the typed JafarParser API. Only fields actually used by the + * converter are declared - this allows the parser to skip extraction of unused fields. + */ +package com.datadog.profiling.otel.jfr; diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/OtlpProtoFields.java b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/OtlpProtoFields.java new file mode 100644 index 00000000000..e4157d0c771 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/OtlpProtoFields.java @@ -0,0 +1,186 @@ +package com.datadog.profiling.otel.proto; + +/** + * OTLP Profiles protobuf field numbers. Based on + * https://github.com/open-telemetry/opentelemetry-proto/blob/v1.10.0/opentelemetry/proto/profiles/v1development/profiles.proto + * (keep in sync with {@code OTLP_PROTO_COMMIT} in {@code docker/Dockerfile.profcheck}). + */ +public final class OtlpProtoFields { + + private OtlpProtoFields() {} + + // ProfilesData fields + public static final class ProfilesData { + public static final int RESOURCE_PROFILES = 1; + public static final int DICTIONARY = 2; + + private ProfilesData() {} + } + + // ProfilesDictionary fields + public static final class ProfilesDictionary { + public static final int MAPPING_TABLE = 1; + public static final int LOCATION_TABLE = 2; + public static final int FUNCTION_TABLE = 3; + public static final int LINK_TABLE = 4; + public static final int STRING_TABLE = 5; + public static final int ATTRIBUTE_TABLE = 6; + public static final int STACK_TABLE = 7; + + private ProfilesDictionary() {} + } + + // ResourceProfiles fields + public static final class ResourceProfiles { + public static final int RESOURCE = 1; + public static final int SCOPE_PROFILES = 2; + public static final int SCHEMA_URL = 3; + + private ResourceProfiles() {} + } + + // ScopeProfiles fields + public static final class ScopeProfiles { + public static final int SCOPE = 1; + public static final int PROFILES = 2; + public static final int SCHEMA_URL = 3; + + private ScopeProfiles() {} + } + + // Profile fields + public static final class Profile { + public static final int SAMPLE_TYPE = 1; + public static final int SAMPLES = 2; + public static final int TIME_UNIX_NANO = 3; + public static final int DURATION_NANO = 4; + public static final int PERIOD_TYPE = 5; + public static final int PERIOD = 6; + public static final int PROFILE_ID = 7; + public static final int DROPPED_ATTRIBUTES_COUNT = 8; + public static final int ORIGINAL_PAYLOAD_FORMAT = 9; + public static final int ORIGINAL_PAYLOAD = 10; + public static final int ATTRIBUTE_INDICES = 11; + + private Profile() {} + } + + // Sample fields + public static final class Sample { + public static final int STACK_INDEX = 1; + public static final int ATTRIBUTE_INDICES = 2; + public static final int LINK_INDEX = 3; + public static final int VALUES = 4; + public static final int TIMESTAMPS_UNIX_NANO = 5; + + private Sample() {} + } + + // ValueType fields + public static final class ValueType { + public static final int TYPE_STRINDEX = 1; + public static final int UNIT_STRINDEX = 2; + + private ValueType() {} + } + + // Mapping fields + public static final class Mapping { + public static final int MEMORY_START = 1; + public static final int MEMORY_LIMIT = 2; + public static final int FILE_OFFSET = 3; + public static final int FILENAME_STRINDEX = 4; + public static final int ATTRIBUTE_INDICES = 5; + + private Mapping() {} + } + + // Location fields + public static final class Location { + public static final int MAPPING_INDEX = 1; + public static final int ADDRESS = 2; + public static final int LINES = 3; + public static final int ATTRIBUTE_INDICES = 4; + + private Location() {} + } + + // Line fields + public static final class Line { + public static final int FUNCTION_INDEX = 1; + public static final int LINE = 2; + public static final int COLUMN = 3; + + private Line() {} + } + + // Function fields + public static final class Function { + public static final int NAME_STRINDEX = 1; + public static final int SYSTEM_NAME_STRINDEX = 2; + public static final int FILENAME_STRINDEX = 3; + public static final int START_LINE = 4; + + private Function() {} + } + + // Stack fields + public static final class Stack { + public static final int LOCATION_INDICES = 1; + + private Stack() {} + } + + // Link fields + public static final class Link { + public static final int TRACE_ID = 1; + public static final int SPAN_ID = 2; + + private Link() {} + } + + // KeyValueAndUnit fields + public static final class KeyValueAndUnit { + public static final int KEY_STRINDEX = 1; + public static final int VALUE = 2; + public static final int UNIT_STRINDEX = 3; + + private KeyValueAndUnit() {} + } + + // AnyValue fields (from common.proto) + public static final class AnyValue { + public static final int STRING_VALUE = 1; + public static final int BOOL_VALUE = 2; + public static final int INT_VALUE = 3; + public static final int DOUBLE_VALUE = 4; + + private AnyValue() {} + } + + // Resource fields (from resource.proto) + public static final class Resource { + public static final int ATTRIBUTES = 1; + public static final int DROPPED_ATTRIBUTES_COUNT = 2; + + private Resource() {} + } + + // KeyValue fields (from common.proto) + public static final class KeyValue { + public static final int KEY = 1; + public static final int VALUE = 2; + + private KeyValue() {} + } + + // InstrumentationScope fields (from common.proto) + public static final class InstrumentationScope { + public static final int NAME = 1; + public static final int VERSION = 2; + public static final int ATTRIBUTES = 3; + public static final int DROPPED_ATTRIBUTES_COUNT = 4; + + private InstrumentationScope() {} + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/OtlpResourceAttributes.java b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/OtlpResourceAttributes.java new file mode 100644 index 00000000000..175c576fa4d --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/OtlpResourceAttributes.java @@ -0,0 +1,36 @@ +package com.datadog.profiling.otel.proto; + +import java.util.Map; + +/** + * Writes the {@code resource} message of {@code ResourceProfiles} with standard {@code KeyValue} + * attributes; entries with a null or empty value are skipped. + */ +public final class OtlpResourceAttributes { + + private OtlpResourceAttributes() {} + + public static void writeResource(ProtobufEncoder encoder, Map attributes) { + encoder.writeNestedMessage( + OtlpProtoFields.ResourceProfiles.RESOURCE, + resourceEncoder -> { + for (Map.Entry entry : attributes.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + if (value == null || value.isEmpty()) { + continue; + } + resourceEncoder.writeNestedMessage( + OtlpProtoFields.Resource.ATTRIBUTES, + attributeEncoder -> { + attributeEncoder.writeStringField(OtlpProtoFields.KeyValue.KEY, key); + attributeEncoder.writeNestedMessage( + OtlpProtoFields.KeyValue.VALUE, + valueEncoder -> + valueEncoder.writeStringField( + OtlpProtoFields.AnyValue.STRING_VALUE, value)); + }); + } + }); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/ProtobufEncoder.java b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/ProtobufEncoder.java new file mode 100644 index 00000000000..8a4c548f4d9 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/ProtobufEncoder.java @@ -0,0 +1,370 @@ +package com.datadog.profiling.otel.proto; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; + +/** Low-level protobuf wire-format encoder with no external dependencies. */ +public final class ProtobufEncoder { + + // Wire types + public static final int WIRETYPE_VARINT = 0; + public static final int WIRETYPE_FIXED64 = 1; + public static final int WIRETYPE_LENGTH_DELIMITED = 2; + public static final int WIRETYPE_FIXED32 = 5; + + private byte[] buf; + private int size; + // Lazily initialized child encoder, reused across writeNestedMessage / writePackedVarintField + // calls to avoid per-call allocation. Nesting is always sequential, so one child per depth level + // (forming a chain) is safe. + private ProtobufEncoder child; + + public ProtobufEncoder() { + this.buf = new byte[4096]; + } + + public ProtobufEncoder(int initialCapacity) { + this.buf = new byte[initialCapacity]; + } + + /** Resets the encoder for reuse. */ + public void reset() { + size = 0; + } + + private void resize(int needed) { + buf = Arrays.copyOf(buf, Math.max(buf.length * 2, size + needed)); + } + + public void writeTag(int fieldNumber, int wireType) { + writeVarint((fieldNumber << 3) | wireType); + } + + public void writeVarint(long value) { + if (size + 10 > buf.length) resize(10); + while ((value & ~0x7FL) != 0) { + buf[size++] = (byte) ((value & 0x7F) | 0x80); + value >>>= 7; + } + buf[size++] = (byte) value; + } + + public void writeSignedVarint(long value) { + writeVarint((value << 1) ^ (value >> 63)); + } + + public void writeFixed64(long value) { + if (size + 8 > buf.length) resize(8); + buf[size] = (byte) (value & 0xFF); + buf[size + 1] = (byte) ((value >> 8) & 0xFF); + buf[size + 2] = (byte) ((value >> 16) & 0xFF); + buf[size + 3] = (byte) ((value >> 24) & 0xFF); + buf[size + 4] = (byte) ((value >> 32) & 0xFF); + buf[size + 5] = (byte) ((value >> 40) & 0xFF); + buf[size + 6] = (byte) ((value >> 48) & 0xFF); + buf[size + 7] = (byte) ((value >> 56) & 0xFF); + size += 8; + } + + public void writeFixed32(int value) { + if (size + 4 > buf.length) resize(4); + buf[size] = (byte) (value & 0xFF); + buf[size + 1] = (byte) ((value >> 8) & 0xFF); + buf[size + 2] = (byte) ((value >> 16) & 0xFF); + buf[size + 3] = (byte) ((value >> 24) & 0xFF); + size += 4; + } + + public void writeBytes(byte[] bytes) { + writeVarint(bytes.length); + if (size + bytes.length > buf.length) resize(bytes.length); + System.arraycopy(bytes, 0, buf, size, bytes.length); + size += bytes.length; + } + + public void writeString(String value) { + if (value == null || value.isEmpty()) { + writeVarint(0); + return; + } + // Encode UTF-8 directly into the buffer, avoiding the intermediate byte[] allocation + // from String.getBytes(StandardCharsets.UTF_8). + int length = utf8Length(value); + writeVarint(length); + if (size + length > buf.length) resize(length); + encodeUtf8(value); + } + + private static int utf8Length(String s) { + int len = 0; + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (c < 0x80) { + len++; + } else if (c < 0x800) { + len += 2; + } else if (Character.isHighSurrogate(c) + && i + 1 < s.length() + && Character.isLowSurrogate(s.charAt(i + 1))) { + len += 4; + i++; // skip the low surrogate + } else { + // Lone surrogate (replaced with U+FFFD = 3 bytes) or BMP char (3 bytes) + len += 3; + } + } + return len; + } + + private void encodeUtf8(String s) { + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (c < 0x80) { + buf[size++] = (byte) c; + } else if (c < 0x800) { + buf[size++] = (byte) (0xC0 | (c >> 6)); + buf[size++] = (byte) (0x80 | (c & 0x3F)); + } else if (Character.isSurrogate(c)) { + // Surrogate pair: combine with next char to form a 4-byte UTF-8 sequence + if (Character.isHighSurrogate(c) && i + 1 < s.length()) { + char c2 = s.charAt(++i); + if (Character.isLowSurrogate(c2)) { + int cp = Character.toCodePoint(c, c2); + buf[size++] = (byte) (0xF0 | (cp >> 18)); + buf[size++] = (byte) (0x80 | ((cp >> 12) & 0x3F)); + buf[size++] = (byte) (0x80 | ((cp >> 6) & 0x3F)); + buf[size++] = (byte) (0x80 | (cp & 0x3F)); + } else { + // Malformed: lone high surrogate, replace with U+FFFD + buf[size++] = (byte) 0xEF; + buf[size++] = (byte) 0xBF; + buf[size++] = (byte) 0xBD; + i--; // re-process c2 + } + } else { + // Lone low surrogate, replace with U+FFFD + buf[size++] = (byte) 0xEF; + buf[size++] = (byte) 0xBF; + buf[size++] = (byte) 0xBD; + } + } else { + buf[size++] = (byte) (0xE0 | (c >> 12)); + buf[size++] = (byte) (0x80 | ((c >> 6) & 0x3F)); + buf[size++] = (byte) (0x80 | (c & 0x3F)); + } + } + } + + /** + * Writes a nested message as a length-delimited field. The child encoder is reused to avoid + * allocation; content is copied directly into this encoder's buffer without an intermediate + * {@code toByteArray()} copy. + * + *

Empty messages are always written (tag + zero length): required for OTLP dictionary tables + * where index 0 is the null/unset sentinel. + */ + public void writeNestedMessage(int fieldNumber, MessageWriter writer) { + if (child == null) { + child = new ProtobufEncoder(); + } else { + child.reset(); + } + writer.write(child); + + writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); + writeVarint(child.size); + if (child.size > 0) { + if (size + child.size > buf.length) resize(child.size); + System.arraycopy(child.buf, 0, buf, size, child.size); + size += child.size; + } + } + + public void writeVarintField(int fieldNumber, long value) { + if (value != 0) { + writeTag(fieldNumber, WIRETYPE_VARINT); + writeVarint(value); + } + } + + public void writeSignedVarintField(int fieldNumber, long value) { + if (value != 0) { + writeTag(fieldNumber, WIRETYPE_VARINT); + writeSignedVarint(value); + } + } + + public void writeFixed64Field(int fieldNumber, long value) { + if (value != 0) { + writeTag(fieldNumber, WIRETYPE_FIXED64); + writeFixed64(value); + } + } + + public void writeFixed32Field(int fieldNumber, int value) { + if (value != 0) { + writeTag(fieldNumber, WIRETYPE_FIXED32); + writeFixed32(value); + } + } + + public void writeStringField(int fieldNumber, String value) { + if (value != null && !value.isEmpty()) { + writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); + writeString(value); + } + } + + public void writeBytesField(int fieldNumber, byte[] value) { + if (value != null && value.length > 0) { + writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); + writeBytes(value); + } + } + + public void writeBytesField(int fieldNumber, InputStream inputStream, long length) + throws IOException { + if (inputStream == null || length == 0) { + // the stream is caller-supplied; close it here so the early-return path cannot leak it + if (inputStream != null) { + inputStream.close(); + } + return; + } + + int savedSize = size; + try { + writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); + writeVarint(length); + + byte[] chunk = new byte[8192]; + long remaining = length; + while (remaining > 0) { + int toRead = (int) Math.min(chunk.length, remaining); + int read = inputStream.read(chunk, 0, toRead); + if (read < 0) { + throw new IOException("Unexpected end of stream"); + } + if (size + read > buf.length) resize(read); + System.arraycopy(chunk, 0, buf, size, read); + size += read; + remaining -= read; + } + } catch (IOException | RuntimeException e) { + // roll the encoder back to its pre-write state so a failed field write cannot leave a + // committed tag + length prefix followed by a truncated payload + size = savedSize; + throw e; + } finally { + inputStream.close(); + } + } + + public void writeBoolField(int fieldNumber, boolean value) { + if (value) { + writeTag(fieldNumber, WIRETYPE_VARINT); + writeVarint(1); + } + } + + public void writePackedVarintField(int fieldNumber, int[] values) { + if (values == null || values.length == 0) { + return; // Empty packed arrays are omitted per protobuf3 spec + } + if (child == null) { + child = new ProtobufEncoder(); + } else { + child.reset(); + } + for (int value : values) { + child.writeVarint(value); + } + writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); + writeVarint(child.size); + if (size + child.size > buf.length) resize(child.size); + System.arraycopy(child.buf, 0, buf, size, child.size); + size += child.size; + } + + public void writePackedVarintField(int fieldNumber, long[] values) { + if (values == null || values.length == 0) { + return; + } + if (child == null) { + child = new ProtobufEncoder(); + } else { + child.reset(); + } + for (long value : values) { + child.writeVarint(value); + } + writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); + writeVarint(child.size); + if (size + child.size > buf.length) resize(child.size); + System.arraycopy(child.buf, 0, buf, size, child.size); + size += child.size; + } + + /** Single-value variant — avoids {@code new long[]{value}} allocation at call site. */ + public void writePackedVarintField(int fieldNumber, long value) { + if (child == null) { + child = new ProtobufEncoder(); + } else { + child.reset(); + } + child.writeVarint(value); + writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); + writeVarint(child.size); + if (size + child.size > buf.length) resize(child.size); + System.arraycopy(child.buf, 0, buf, size, child.size); + size += child.size; + } + + public void writePackedFixed64Field(int fieldNumber, long[] values) { + if (values == null || values.length == 0) { + return; + } + writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); + writeVarint(values.length * 8L); + for (long value : values) { + writeFixed64(value); + } + } + + /** Single-value variant — avoids {@code new long[]{value}} allocation at call site. */ + public void writePackedFixed64Field(int fieldNumber, long value) { + writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); + writeVarint(8); + writeFixed64(value); + } + + public byte[] toByteArray() { + return Arrays.copyOf(buf, size); + } + + /** + * Read-only heap view over the internal buffer without the {@link #toByteArray()} copy. The view + * is only valid while the encoder is not reset or written to afterwards; the caller must treat + * the content as immutable. + */ + public ByteBuffer toByteBuffer() { + return ByteBuffer.wrap(buf, 0, size); + } + + public void writeTo(OutputStream out) throws IOException { + out.write(buf, 0, size); + } + + public int size() { + return size; + } + + /** Functional interface for writing nested messages. */ + @FunctionalInterface + public interface MessageWriter { + void write(ProtobufEncoder encoder); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/dictionary/AttributeTable.java b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/dictionary/AttributeTable.java new file mode 100644 index 00000000000..b9a3a4d4301 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/dictionary/AttributeTable.java @@ -0,0 +1,155 @@ +package com.datadog.profiling.otel.proto.dictionary; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Attribute deduplication table for OTLP profiles. Index 0 is reserved for the null/unset + * attribute. Attributes are key-value pairs with optional unit. + */ +public final class AttributeTable { + + /** Attribute value types. */ + public enum ValueType { + STRING, + BOOL, + INT, + DOUBLE + } + + /** Immutable key for attribute lookup. */ + private static final class AttributeKey { + final int keyIndex; + final ValueType valueType; + final Object value; + final int unitIndex; + + AttributeKey(int keyIndex, ValueType valueType, Object value, int unitIndex) { + this.keyIndex = keyIndex; + this.valueType = valueType; + this.value = value; + this.unitIndex = unitIndex; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AttributeKey that = (AttributeKey) o; + return keyIndex == that.keyIndex + && unitIndex == that.unitIndex + && valueType == that.valueType + && Objects.equals(value, that.value); + } + + @Override + public int hashCode() { + return Objects.hash(keyIndex, valueType, value, unitIndex); + } + } + + /** Attribute entry stored in the table. */ + public static final class AttributeEntry { + public final int keyIndex; + public final ValueType valueType; + public final Object value; + public final int unitIndex; + + AttributeEntry(int keyIndex, ValueType valueType, Object value, int unitIndex) { + this.keyIndex = keyIndex; + this.valueType = valueType; + this.value = value; + this.unitIndex = unitIndex; + } + + public String getStringValue() { + return valueType == ValueType.STRING ? (String) value : null; + } + + @edu.umd.cs.findbugs.annotations.SuppressFBWarnings("NP_BOOLEAN_RETURN_NULL") + public Boolean getBoolValue() { + return valueType == ValueType.BOOL ? (Boolean) value : null; + } + + public Long getIntValue() { + return valueType == ValueType.INT ? (Long) value : null; + } + + public Double getDoubleValue() { + return valueType == ValueType.DOUBLE ? (Double) value : null; + } + } + + private final List attributes; + private final Map attributeToIndex; + + public AttributeTable() { + attributes = new ArrayList<>(); + attributeToIndex = new HashMap<>(); + // Index 0 is reserved for null/unset attribute + attributes.add(new AttributeEntry(0, ValueType.STRING, "", 0)); + } + + public int internString(int keyIndex, String value, int unitIndex) { + if (keyIndex == 0) { + return 0; + } + return intern(keyIndex, ValueType.STRING, value, unitIndex); + } + + public int internBool(int keyIndex, boolean value, int unitIndex) { + if (keyIndex == 0) { + return 0; + } + return intern(keyIndex, ValueType.BOOL, value, unitIndex); + } + + public int internInt(int keyIndex, long value, int unitIndex) { + if (keyIndex == 0) { + return 0; + } + return intern(keyIndex, ValueType.INT, value, unitIndex); + } + + public int internDouble(int keyIndex, double value, int unitIndex) { + if (keyIndex == 0) { + return 0; + } + return intern(keyIndex, ValueType.DOUBLE, value, unitIndex); + } + + private int intern(int keyIndex, ValueType valueType, Object value, int unitIndex) { + AttributeKey key = new AttributeKey(keyIndex, valueType, value, unitIndex); + Integer existing = attributeToIndex.get(key); + if (existing != null) { + return existing; + } + + int index = attributes.size(); + attributes.add(new AttributeEntry(keyIndex, valueType, value, unitIndex)); + attributeToIndex.put(key, index); + return index; + } + + public AttributeEntry get(int index) { + return attributes.get(index); + } + + public int size() { + return attributes.size(); + } + + public List getAttributes() { + return attributes; + } + + /** Resets the table to its initial state with only the null attribute at index 0. */ + public void reset() { + attributes.clear(); + attributeToIndex.clear(); + attributes.add(new AttributeEntry(0, ValueType.STRING, "", 0)); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/dictionary/FunctionTable.java b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/dictionary/FunctionTable.java new file mode 100644 index 00000000000..b56a27bdec6 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/dictionary/FunctionTable.java @@ -0,0 +1,112 @@ +package com.datadog.profiling.otel.proto.dictionary; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Function deduplication table for OTLP profiles. Index 0 is reserved for the null/unset function. + * Functions are identified by their (nameIndex, systemNameIndex, filenameIndex, startLine) tuple. + */ +public final class FunctionTable { + + /** Immutable key for function lookup. */ + private static final class FunctionKey { + final int nameIndex; + final int systemNameIndex; + final int filenameIndex; + final long startLine; + private final int hash; + + FunctionKey(int nameIndex, int systemNameIndex, int filenameIndex, long startLine) { + this.nameIndex = nameIndex; + this.systemNameIndex = systemNameIndex; + this.filenameIndex = filenameIndex; + this.startLine = startLine; + int h = nameIndex; + h = 31 * h + systemNameIndex; + h = 31 * h + filenameIndex; + h = 31 * h + Long.hashCode(startLine); + this.hash = h; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + FunctionKey that = (FunctionKey) o; + return nameIndex == that.nameIndex + && systemNameIndex == that.systemNameIndex + && filenameIndex == that.filenameIndex + && startLine == that.startLine; + } + + @Override + public int hashCode() { + return hash; + } + } + + /** Function entry stored in the table. */ + public static final class FunctionEntry { + public final int nameIndex; + public final int systemNameIndex; + public final int filenameIndex; + public final long startLine; + + FunctionEntry(int nameIndex, int systemNameIndex, int filenameIndex, long startLine) { + this.nameIndex = nameIndex; + this.systemNameIndex = systemNameIndex; + this.filenameIndex = filenameIndex; + this.startLine = startLine; + } + } + + private final List functions; + private final Map functionToIndex; + + public FunctionTable() { + functions = new ArrayList<>(); + functionToIndex = new HashMap<>(); + // Index 0 is reserved for null/unset function + functions.add(new FunctionEntry(0, 0, 0, 0)); + } + + public int intern(int nameIndex, int systemNameIndex, int filenameIndex, long startLine) { + // All zeros means null function + if (nameIndex == 0 && systemNameIndex == 0 && filenameIndex == 0 && startLine == 0) { + return 0; + } + + FunctionKey key = new FunctionKey(nameIndex, systemNameIndex, filenameIndex, startLine); + Integer existing = functionToIndex.get(key); + if (existing != null) { + return existing; + } + + int index = functions.size(); + functions.add(new FunctionEntry(nameIndex, systemNameIndex, filenameIndex, startLine)); + functionToIndex.put(key, index); + return index; + } + + public FunctionEntry get(int index) { + return functions.get(index); + } + + public int size() { + return functions.size(); + } + + public List getFunctions() { + return functions; + } + + /** Resets the table to its initial state with only the null function at index 0. */ + public void reset() { + functions.clear(); + functionToIndex.clear(); + functions.add(new FunctionEntry(0, 0, 0, 0)); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/dictionary/LinkTable.java b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/dictionary/LinkTable.java new file mode 100644 index 00000000000..4f86964299c --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/dictionary/LinkTable.java @@ -0,0 +1,134 @@ +package com.datadog.profiling.otel.proto.dictionary; + +import com.datadog.profiling.otel.LongKeyIntMap; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Link deduplication table for OTLP profiles. Index 0 is reserved for the null/unset link. A link + * connects a profile sample to a trace span for correlation. + */ +public final class LinkTable { + + /** Link entry stored in the table. */ + public static final class LinkEntry { + public final byte[] traceId; + public final byte[] spanId; + + LinkEntry(byte[] traceId, byte[] spanId) { + this.traceId = traceId; + this.spanId = spanId; + } + } + + private static final byte[] EMPTY_TRACE_ID = new byte[16]; + private static final byte[] EMPTY_SPAN_ID = new byte[8]; + + private final List links; + private final LongKeyIntMap linkToIndex; + + public LinkTable() { + links = new ArrayList<>(); + linkToIndex = new LongKeyIntMap(3, 16); + // Index 0 is reserved for null/unset link + links.add(new LinkEntry(EMPTY_TRACE_ID, EMPTY_SPAN_ID)); + } + + /** + * Interns a link keyed on the full 128-bit trace id plus the span id. No production call site yet + * — the converter only carries spanId/localRootSpanId longs; this overload exists for when the + * profiling JFR events gain a real 128-bit trace id. + * + *

The key spaces of the two overloads are kept disjoint (the long overload always keys + * traceIdHigh as 0; this overload tags the high bit) so equivalent links cannot alias across + * overloads. + */ + public int intern(byte[] traceId, byte[] spanId) { + if (traceId == null || spanId == null) { + return 0; + } + long traceIdHigh = bytesToLong(traceId, 0); + long traceIdLow = bytesToLong(traceId, 8); + long spanIdLong = bytesToLong(spanId, 0); + if (traceIdHigh == 0 && traceIdLow == 0 && spanIdLong == 0) { + return 0; + } + + // tag the high bit so byte[]-overload keys never collide with long-overload keys (which + // always carry traceIdHigh == 0) + long keyHigh = traceIdHigh | Long.MIN_VALUE; + int cached = linkToIndex.get3(keyHigh, traceIdLow, spanIdLong); + if (cached != -1) return cached; + + int index = links.size(); + byte[] traceIdCopy = Arrays.copyOf(traceId, traceId.length); + byte[] spanIdCopy = Arrays.copyOf(spanId, spanId.length); + links.add(new LinkEntry(traceIdCopy, spanIdCopy)); + linkToIndex.put3(keyHigh, traceIdLow, spanIdLong, index); + return index; + } + + /** + * Interns a link keyed on the low 64 bits of the trace id plus the span id — the only overload + * with a production call site (see JfrToOtlpConverter.extractLinkIndex). + */ + public int intern(long traceIdLow, long spanId) { + if (traceIdLow == 0 && spanId == 0) { + return 0; + } + + // the long overload only carries the low 64 bits of the trace id; the high bits key as 0 + // (byte[]-overload keys tag the high bit, keeping the two key spaces disjoint) + int cached = linkToIndex.get3(0, traceIdLow, spanId); + if (cached != -1) return cached; + + // Cache miss — allocate byte arrays only once per unique link + byte[] traceIdBytes = new byte[16]; + long tmp = traceIdLow; + for (int i = 15; i >= 8; i--) { + traceIdBytes[i] = (byte) (tmp & 0xFF); + tmp >>>= 8; + } + + byte[] spanIdBytes = new byte[8]; + tmp = spanId; + for (int i = 7; i >= 0; i--) { + spanIdBytes[i] = (byte) (tmp & 0xFF); + tmp >>>= 8; + } + + int index = links.size(); + links.add(new LinkEntry(traceIdBytes, spanIdBytes)); + linkToIndex.put3(0, traceIdLow, spanId, index); + return index; + } + + private static long bytesToLong(byte[] bytes, int offset) { + if (bytes.length < offset + 8) return 0; + long v = 0; + for (int i = offset; i < offset + 8; i++) { + v = (v << 8) | (bytes[i] & 0xFF); + } + return v; + } + + public LinkEntry get(int index) { + return links.get(index); + } + + public int size() { + return links.size(); + } + + public List getLinks() { + return links; + } + + /** Resets the table to its initial state with only the null link at index 0. */ + public void reset() { + links.clear(); + linkToIndex.clear(); + links.add(new LinkEntry(EMPTY_TRACE_ID, EMPTY_SPAN_ID)); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/dictionary/LocationTable.java b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/dictionary/LocationTable.java new file mode 100644 index 00000000000..387e1e9c373 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/dictionary/LocationTable.java @@ -0,0 +1,131 @@ +package com.datadog.profiling.otel.proto.dictionary; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Location deduplication table for OTLP profiles. Index 0 is reserved for the null/unset location. + * A location represents a stack frame with mapping, address, and line information. + */ +public final class LocationTable { + + /** Immutable key for location lookup. */ + private static final class LocationKey { + final int mappingIndex; + final long address; + final int functionIndex; + final long line; + final long column; + private final int hash; + + LocationKey(int mappingIndex, long address, int functionIndex, long line, long column) { + this.mappingIndex = mappingIndex; + this.address = address; + this.functionIndex = functionIndex; + this.line = line; + this.column = column; + int h = mappingIndex; + h = 31 * h + Long.hashCode(address); + h = 31 * h + functionIndex; + h = 31 * h + Long.hashCode(line); + h = 31 * h + Long.hashCode(column); + this.hash = h; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + LocationKey that = (LocationKey) o; + return mappingIndex == that.mappingIndex + && address == that.address + && functionIndex == that.functionIndex + && line == that.line + && column == that.column; + } + + @Override + public int hashCode() { + return hash; + } + } + + /** Line information within a location (for inlined functions). */ + public static final class LineEntry { + public final int functionIndex; + public final long line; + public final long column; + + public LineEntry(int functionIndex, long line, long column) { + this.functionIndex = functionIndex; + this.line = line; + this.column = column; + } + } + + /** Location entry stored in the table. */ + public static final class LocationEntry { + public final int mappingIndex; + public final long address; + public final List lines; + + LocationEntry(int mappingIndex, long address, List lines) { + this.mappingIndex = mappingIndex; + this.address = address; + this.lines = lines; + } + } + + private final List locations; + private final Map locationToIndex; + + public LocationTable() { + locations = new ArrayList<>(); + locationToIndex = new HashMap<>(); + // Index 0 is reserved for null/unset location + locations.add(new LocationEntry(0, 0, new ArrayList<>())); + } + + public int intern(int mappingIndex, long address, int functionIndex, long line, long column) { + // All zeros means null location + if (mappingIndex == 0 && address == 0 && functionIndex == 0 && line == 0 && column == 0) { + return 0; + } + + LocationKey key = new LocationKey(mappingIndex, address, functionIndex, line, column); + Integer existing = locationToIndex.get(key); + if (existing != null) { + return existing; + } + + int index = locations.size(); + List lines = new ArrayList<>(); + if (functionIndex != 0 || line != 0 || column != 0) { + lines.add(new LineEntry(functionIndex, line, column)); + } + locations.add(new LocationEntry(mappingIndex, address, lines)); + locationToIndex.put(key, index); + return index; + } + + public LocationEntry get(int index) { + return locations.get(index); + } + + public int size() { + return locations.size(); + } + + public List getLocations() { + return locations; + } + + /** Resets the table to its initial state with only the null location at index 0. */ + public void reset() { + locations.clear(); + locationToIndex.clear(); + locations.add(new LocationEntry(0, 0, new ArrayList<>())); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/dictionary/StackTable.java b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/dictionary/StackTable.java new file mode 100644 index 00000000000..5110d2bc5ea --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/dictionary/StackTable.java @@ -0,0 +1,96 @@ +package com.datadog.profiling.otel.proto.dictionary; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Stack (call stack) deduplication table for OTLP profiles. Index 0 is reserved for the null/unset + * stack. A stack is an ordered sequence of location indices, where the first entry is the leaf + * frame. + */ +public final class StackTable { + + /** Wrapper for int[] to use as HashMap key with proper equals/hashCode. */ + private static final class StackKey { + final int[] locationIndices; + private final int hashCode; + + StackKey(int[] locationIndices) { + this.locationIndices = locationIndices; + this.hashCode = Arrays.hashCode(locationIndices); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + StackKey that = (StackKey) o; + return Arrays.equals(locationIndices, that.locationIndices); + } + + @Override + public int hashCode() { + return hashCode; + } + } + + /** Stack entry stored in the table. */ + public static final class StackEntry { + public final int[] locationIndices; + + StackEntry(int[] locationIndices) { + this.locationIndices = locationIndices; + } + } + + private final List stacks; + private final Map stackToIndex; + + public StackTable() { + stacks = new ArrayList<>(); + stackToIndex = new HashMap<>(); + // Index 0 is reserved for null/unset stack (empty) + stacks.add(new StackEntry(new int[0])); + } + + public int intern(int[] locationIndices) { + if (locationIndices == null || locationIndices.length == 0) { + return 0; + } + + StackKey key = new StackKey(locationIndices); + Integer existing = stackToIndex.get(key); + if (existing != null) { + return existing; + } + + int index = stacks.size(); + // Make a defensive copy + int[] copy = Arrays.copyOf(locationIndices, locationIndices.length); + stacks.add(new StackEntry(copy)); + stackToIndex.put(new StackKey(copy), index); + return index; + } + + public StackEntry get(int index) { + return stacks.get(index); + } + + public int size() { + return stacks.size(); + } + + public List getStacks() { + return stacks; + } + + /** Resets the table to its initial state with only the null stack at index 0. */ + public void reset() { + stacks.clear(); + stackToIndex.clear(); + stacks.add(new StackEntry(new int[0])); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/dictionary/StringTable.java b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/dictionary/StringTable.java new file mode 100644 index 00000000000..915b1084907 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/main/java/com/datadog/profiling/otel/proto/dictionary/StringTable.java @@ -0,0 +1,57 @@ +package com.datadog.profiling.otel.proto.dictionary; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * String interning table for OTLP profiles. Index 0 is reserved for the empty string (null/unset + * sentinel). + */ +public final class StringTable { + private final List strings; + private final Map stringToIndex; + + public StringTable() { + strings = new ArrayList<>(); + stringToIndex = new HashMap<>(); + // Index 0 is reserved for empty string (null/unset sentinel) + strings.add(""); + stringToIndex.put("", 0); + } + + public int intern(String s) { + if (s == null || s.isEmpty()) { + return 0; + } + Integer existing = stringToIndex.get(s); + if (existing != null) { + return existing; + } + int index = strings.size(); + strings.add(s); + stringToIndex.put(s, index); + return index; + } + + public String get(int index) { + return strings.get(index); + } + + public int size() { + return strings.size(); + } + + public List getStrings() { + return strings; + } + + /** Resets the table to its initial state with only the empty string at index 0. */ + public void reset() { + strings.clear(); + stringToIndex.clear(); + strings.add(""); + stringToIndex.put("", 0); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/JfrToOtlpConverterDeduplicationTest.java b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/JfrToOtlpConverterDeduplicationTest.java new file mode 100644 index 00000000000..798c02983d2 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/JfrToOtlpConverterDeduplicationTest.java @@ -0,0 +1,221 @@ +package com.datadog.profiling.otel; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +import com.datadog.profiling.otel.proto.dictionary.FunctionTable; +import com.datadog.profiling.otel.proto.dictionary.LinkTable; +import com.datadog.profiling.otel.proto.dictionary.LocationTable; +import com.datadog.profiling.otel.proto.dictionary.StackTable; +import com.datadog.profiling.otel.proto.dictionary.StringTable; +import java.lang.reflect.Field; +import org.junit.jupiter.api.Test; + +/** + * Tests for verifying deduplication behavior in JFR to OTLP conversion. + * + *

These tests use reflection to access internal dictionary tables and verify that: + * + *

    + *
  • Identical stacktraces are deduplicated correctly + *
  • Dictionary tables (String, Function, Location, Stack) work as expected + *
  • Large-scale conversions handle deduplication efficiently + *
+ */ +class JfrToOtlpConverterDeduplicationTest { + + @Test + void verifyStacktraceDeduplication() throws Exception { + JfrToOtlpConverter converter = new JfrToOtlpConverter(); + + // Access internal dictionary tables via reflection + StringTable strings = getDictionaryTable(converter, "stringTable", StringTable.class); + FunctionTable functions = getDictionaryTable(converter, "functionTable", FunctionTable.class); + LocationTable locations = getDictionaryTable(converter, "locationTable", LocationTable.class); + StackTable stacks = getDictionaryTable(converter, "stackTable", StackTable.class); + + // Simulate multiple samples with identical stacks + // Stack A: method1 -> method2 -> method3 + int className1 = strings.intern("com.example.Class1"); + int methodName1 = strings.intern("method1"); + int func1 = functions.intern(className1, methodName1, 0, 10); + int loc1 = locations.intern(0, 0x1000, func1, 10, 0); + + int className2 = strings.intern("com.example.Class2"); + int methodName2 = strings.intern("method2"); + int func2 = functions.intern(className2, methodName2, 0, 20); + int loc2 = locations.intern(0, 0x2000, func2, 20, 0); + + int className3 = strings.intern("com.example.Class3"); + int methodName3 = strings.intern("method3"); + int func3 = functions.intern(className3, methodName3, 0, 30); + int loc3 = locations.intern(0, 0x3000, func3, 30, 0); + + int[] stackA = new int[] {loc1, loc2, loc3}; + + // Intern stack A multiple times - should get same index + int stackIndex1 = stacks.intern(stackA); + int stackIndex2 = stacks.intern(stackA); + int stackIndex3 = stacks.intern(new int[] {loc1, loc2, loc3}); // New array, same content + + // All should reference the same stack index + assertEquals(stackIndex1, stackIndex2, "Identical stacks should deduplicate"); + assertEquals(stackIndex1, stackIndex3, "Stacks with same content should deduplicate"); + + // Create different stack B: method4 -> method5 + int className4 = strings.intern("com.example.Class4"); + int methodName4 = strings.intern("method4"); + int func4 = functions.intern(className4, methodName4, 0, 40); + int loc4 = locations.intern(0, 0x4000, func4, 40, 0); + + int className5 = strings.intern("com.example.Class5"); + int methodName5 = strings.intern("method5"); + int func5 = functions.intern(className5, methodName5, 0, 50); + int loc5 = locations.intern(0, 0x5000, func5, 50, 0); + + int[] stackB = new int[] {loc4, loc5}; + int stackIndexB = stacks.intern(stackB); + + // Stack B should have different index + assertNotEquals(stackIndex1, stackIndexB, "Different stacks should have different indices"); + + // Verify stack table size - should have 3 entries: index 0 (null), Stack A, Stack B + assertEquals(3, stacks.size(), "Should have 3 stacks: null, A, B"); + + // Verify string deduplication - repeated interns return same index + int className1Again = strings.intern("com.example.Class1"); + assertEquals(className1, className1Again, "Duplicate strings should deduplicate"); + } + + @Test + void verifyDictionaryTableDeduplication() throws Exception { + JfrToOtlpConverter converter = new JfrToOtlpConverter(); + + // Access internal dictionary tables via reflection + StringTable strings = getDictionaryTable(converter, "stringTable", StringTable.class); + FunctionTable functions = getDictionaryTable(converter, "functionTable", FunctionTable.class); + LocationTable locations = getDictionaryTable(converter, "locationTable", LocationTable.class); + + // Verify string deduplication + int str1 = strings.intern("test.Class"); + int str2 = strings.intern("test.Class"); // Duplicate + int str3 = strings.intern("other.Class"); // Different + + assertEquals(str1, str2, "Duplicate strings should return same index"); + assertNotEquals(str1, str3, "Different strings should return different index"); + + // Initial size: 1 (index 0 for null/empty) + 2 unique strings = 3 + assertEquals(3, strings.size(), "StringTable should have 3 entries"); + + // Verify function deduplication + int method1 = strings.intern("method"); + int func1 = functions.intern(str1, method1, 0, 100); + int func2 = functions.intern(str1, method1, 0, 100); // Duplicate + int func3 = functions.intern(str3, method1, 0, 200); // Different class + + assertEquals(func1, func2, "Duplicate functions should return same index"); + assertNotEquals(func1, func3, "Different functions should return different index"); + + // Function table size: 1 (index 0) + 2 unique functions = 3 + assertEquals(3, functions.size(), "FunctionTable should have 3 entries"); + + // Verify location deduplication + int loc1 = locations.intern(0, 0x1000, func1, 10, 0); + int loc2 = locations.intern(0, 0x1000, func1, 10, 0); // Duplicate + int loc3 = locations.intern(0, 0x2000, func1, 20, 0); // Different address + + assertEquals(loc1, loc2, "Duplicate locations should return same index"); + assertNotEquals(loc1, loc3, "Different locations should return different index"); + + // Location table size: 1 (index 0) + 2 unique locations = 3 + assertEquals(3, locations.size(), "LocationTable should have 3 entries"); + } + + @Test + void verifyLargeScaleDeduplication() throws Exception { + JfrToOtlpConverter converter = new JfrToOtlpConverter(); + + // Access internal dictionary tables via reflection + StringTable strings = getDictionaryTable(converter, "stringTable", StringTable.class); + FunctionTable functions = getDictionaryTable(converter, "functionTable", FunctionTable.class); + LocationTable locations = getDictionaryTable(converter, "locationTable", LocationTable.class); + StackTable stacks = getDictionaryTable(converter, "stackTable", StackTable.class); + + // Create 10 unique stacks + int[][] uniqueStacks = new int[10][]; + for (int i = 0; i < 10; i++) { + int className = strings.intern("com.example.Class" + i); + int methodName = strings.intern("method" + i); + int func = functions.intern(className, methodName, 0, i * 10); + int loc = locations.intern(0, 0x1000 + (i * 0x100), func, i * 10, 0); + uniqueStacks[i] = new int[] {loc}; + } + + // Intern each unique stack 100 times + int[] stackIndices = new int[10]; + for (int i = 0; i < 10; i++) { + int firstIndex = stacks.intern(uniqueStacks[i]); + stackIndices[i] = firstIndex; + + // Intern same stack 99 more times + for (int repeat = 0; repeat < 99; repeat++) { + int repeatIndex = stacks.intern(uniqueStacks[i]); + assertEquals( + firstIndex, repeatIndex, "Stack " + i + " repeat " + repeat + " should deduplicate"); + } + } + + // Verify all 10 stacks have unique indices + for (int i = 0; i < 10; i++) { + for (int j = i + 1; j < 10; j++) { + assertNotEquals( + stackIndices[i], + stackIndices[j], + "Stack " + i + " and " + j + " should have different indices"); + } + } + + // Stack table should have 11 entries: index 0 (null) + 10 unique stacks + assertEquals(11, stacks.size(), "StackTable should have 11 entries after 1000 interns"); + + // Verify string table has expected number of entries + // Initial: 1 (index 0) + 10 class names + 10 method names = 21 + assertEquals(21, strings.size(), "StringTable should have 21 entries"); + + // Verify function table has expected number of entries + // Initial: 1 (index 0) + 10 unique functions = 11 + assertEquals(11, functions.size(), "FunctionTable should have 11 entries"); + + // Verify location table has expected number of entries + // Initial: 1 (index 0) + 10 unique locations = 11 + assertEquals(11, locations.size(), "LocationTable should have 11 entries"); + } + + @Test + void verifyLinkTableDeduplication() throws Exception { + JfrToOtlpConverter converter = new JfrToOtlpConverter(); + + // Access link table via reflection + LinkTable links = getDictionaryTable(converter, "linkTable", LinkTable.class); + + // Create trace links + int link1 = links.intern(123L, 456L); + int link2 = links.intern(123L, 456L); // Duplicate + int link3 = links.intern(789L, 101112L); // Different + + assertEquals(link1, link2, "Duplicate links should return same index"); + assertNotEquals(link1, link3, "Different links should return different index"); + + // Link table size: 1 (index 0) + 2 unique links = 3 + assertEquals(3, links.size(), "LinkTable should have 3 entries"); + } + + // Helper method to access private dictionary table fields using reflection + @SuppressWarnings("unchecked") + private T getDictionaryTable(JfrToOtlpConverter converter, String fieldName, Class type) + throws Exception { + Field field = JfrToOtlpConverter.class.getDeclaredField(fieldName); + field.setAccessible(true); + return (T) field.get(converter); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/JfrToOtlpConverterResourceAttributesTest.java b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/JfrToOtlpConverterResourceAttributesTest.java new file mode 100644 index 00000000000..82e641bda5b --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/JfrToOtlpConverterResourceAttributesTest.java @@ -0,0 +1,81 @@ +package com.datadog.profiling.otel; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.openjdk.jmc.flightrecorder.writer.api.Recording; +import org.openjdk.jmc.flightrecorder.writer.api.Recordings; + +/** Verifies resource attribute emission for both protobuf and JSON conversion output. */ +class JfrToOtlpConverterResourceAttributesTest { + @TempDir Path tempDir; + + private JfrToOtlpConverter converter; + private Map attributes; + + @BeforeEach + void setUp() { + converter = new JfrToOtlpConverter(); + attributes = new LinkedHashMap<>(); + attributes.put("service.name", "test-service"); + attributes.put("deployment.environment.name", "test"); + attributes.put("telemetry.sdk.name", "datadog"); + } + + @Test + void protobufOutputCarriesResourceAttributes() throws IOException { + Path jfrFile = tempDir.resolve("empty.jfr"); + try (Recording recording = Recordings.newRecording(jfrFile)) {} + + byte[] result = + converter + .setResourceAttributes(attributes) + .addFile(jfrFile, Instant.now().minusSeconds(10), Instant.now()) + .convert(JfrToOtlpConverter.Kind.PROTO); + + String payload = new String(result, StandardCharsets.ISO_8859_1); + assertTrue(payload.contains("service.name")); + assertTrue(payload.contains("test-service")); + assertTrue(payload.contains("telemetry.sdk.name")); + } + + @Test + void jsonOutputCarriesResourceAttributes() throws IOException { + Path jfrFile = tempDir.resolve("empty-json.jfr"); + try (Recording recording = Recordings.newRecording(jfrFile)) {} + + byte[] result = + converter + .setResourceAttributes(attributes) + .addFile(jfrFile, Instant.now().minusSeconds(10), Instant.now()) + .convert(JfrToOtlpConverter.Kind.JSON); + + String payload = new String(result, StandardCharsets.UTF_8); + assertTrue(payload.contains("\"resource\"")); + assertTrue(payload.contains("service.name")); + assertTrue(payload.contains("test-service")); + } + + @Test + void emptyAttributesOmitResourceMessage() throws IOException { + Path jfrFile = tempDir.resolve("empty-none.jfr"); + try (Recording recording = Recordings.newRecording(jfrFile)) {} + + byte[] result = + converter + .addFile(jfrFile, Instant.now().minusSeconds(10), Instant.now()) + .convert(JfrToOtlpConverter.Kind.PROTO); + + String payload = new String(result, StandardCharsets.ISO_8859_1); + assertFalse(payload.contains("service.name")); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/JfrToOtlpConverterSmokeTest.java b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/JfrToOtlpConverterSmokeTest.java new file mode 100644 index 00000000000..e81e846ffd8 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/JfrToOtlpConverterSmokeTest.java @@ -0,0 +1,972 @@ +package com.datadog.profiling.otel; + +import static com.datadog.profiling.otel.JfrTools.epochNanos; +import static com.datadog.profiling.otel.JfrTools.putStackTrace; +import static com.datadog.profiling.otel.JfrTools.writeEvent; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.time.Instant; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.openjdk.jmc.flightrecorder.writer.api.Recording; +import org.openjdk.jmc.flightrecorder.writer.api.Recordings; +import org.openjdk.jmc.flightrecorder.writer.api.Type; +import org.openjdk.jmc.flightrecorder.writer.api.Types; + +/** Smoke tests for JfrToOtlpConverter using JMC JFR writer to generate test recordings. */ +class JfrToOtlpConverterSmokeTest { + @TempDir Path tempDir; + + private JfrToOtlpConverter converter; + + @BeforeEach + void setUp() { + converter = new JfrToOtlpConverter(); + } + + @Test + void convertEmptyRecording() throws IOException { + Path jfrFile = tempDir.resolve("empty.jfr"); + + // Create empty JFR file with minimal setup + try (Recording recording = Recordings.newRecording(jfrFile)) { + // Just create an empty recording - no events + } + + Instant start = Instant.now().minusSeconds(10); + Instant end = Instant.now(); + + byte[] result = converter.addFile(jfrFile, start, end).convert(); + + // Empty recordings produce a valid but minimal protobuf structure + assertNotNull(result); + } + + @Test + void convertRecordingWithExecutionSample() throws IOException { + Path jfrFile = tempDir.resolve("cpu.jfr"); + + try (Recording recording = Recordings.newRecording(jfrFile)) { + // Register custom datadog.ExecutionSample event type with minimal fields + Type executionSampleType = + recording.registerEventType( + "datadog.ExecutionSample", + type -> { + type.addField("spanId", Types.Builtin.LONG); + type.addField("localRootSpanId", Types.Builtin.LONG); + }); + + // Write execution sample event without stack trace for simplicity + writeEvent( + recording, + executionSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", 12345L); + valueBuilder.putField("localRootSpanId", 67890L); + }); + } + + Instant start = Instant.now().minusSeconds(10); + Instant end = Instant.now(); + + byte[] result = converter.addFile(jfrFile, start, end).convert(); + + assertNotNull(result); + assertTrue(result.length > 0); + } + + @Test + void convertRecordingWithJdkExecutionSample() throws IOException { + Path jfrFile = tempDir.resolve("jdk-cpu.jfr"); + + try (Recording recording = Recordings.newRecording(jfrFile)) { + // Standard OpenJDK sampling event — the only CPU sample source in recordings produced + // without ddprof + Type executionSampleType = recording.registerEventType("jdk.ExecutionSample", type -> {}); + + writeEvent(recording, executionSampleType, valueBuilder -> {}); + } + + Instant start = Instant.now().minusSeconds(10); + Instant end = Instant.now(); + + // JSON output is inspected because the samples carried by the profile are only visible + // there without a protobuf decoder — a jdk.ExecutionSample event must surface as a cpu + // sample, not be silently dropped + String json = + new String( + converter.addFile(jfrFile, start, end).convert(JfrToOtlpConverter.Kind.JSON), + StandardCharsets.UTF_8); + + assertTrue( + json.contains("\"values\""), "jdk.ExecutionSample must produce a sample, got: " + json); + } + + @Test + void convertRecordingWithJdkCPUTimeSample() throws IOException { + Path jfrFile = tempDir.resolve("jdk-cputime.jfr"); + + try (Recording recording = Recordings.newRecording(jfrFile)) { + // replaces jdk.ExecutionSample as the OpenJDK profiler's CPU sample source on JDK 25+ (Linux) + Type cpuTimeSampleType = recording.registerEventType("jdk.CPUTimeSample", type -> {}); + + writeEvent(recording, cpuTimeSampleType, valueBuilder -> {}); + } + + Instant start = Instant.now().minusSeconds(10); + Instant end = Instant.now(); + + String json = + new String( + converter.addFile(jfrFile, start, end).convert(JfrToOtlpConverter.Kind.JSON), + StandardCharsets.UTF_8); + + assertTrue( + json.contains("\"values\""), "jdk.CPUTimeSample must produce a sample, got: " + json); + } + + @Test + void convertRecordingWithJdkNativeMethodSample() throws IOException { + Path jfrFile = tempDir.resolve("jdk-native.jfr"); + + try (Recording recording = Recordings.newRecording(jfrFile)) { + Type nativeMethodSampleType = + recording.registerEventType("jdk.NativeMethodSample", type -> {}); + + writeEvent(recording, nativeMethodSampleType, valueBuilder -> {}); + } + + Instant start = Instant.now().minusSeconds(10); + Instant end = Instant.now(); + + String json = + new String( + converter.addFile(jfrFile, start, end).convert(JfrToOtlpConverter.Kind.JSON), + StandardCharsets.UTF_8); + + assertTrue( + json.contains("\"values\""), "jdk.NativeMethodSample must produce a sample, got: " + json); + } + + @Test + void derivedTimeBoundsOverrideSyntheticWindow() throws IOException { + Path jfrFile = tempDir.resolve("bounds.jfr"); + + try (Recording recording = Recordings.newRecording(jfrFile)) { + Type executionSampleType = recording.registerEventType("jdk.ExecutionSample", type -> {}); + + writeEvent(recording, executionSampleType, valueBuilder -> {}); + } + + // the CLI passes a wide synthetic window; with derived bounds the profile window must + // come from the observed event timestamps instead — visible as a non-epoch start bound + converter.setDeriveTimeBoundsFromEvents(true); + String json = + new String( + converter + .addFile(jfrFile, Instant.EPOCH, Instant.now()) + .convert(JfrToOtlpConverter.Kind.JSON), + StandardCharsets.UTF_8); + + java.util.regex.Matcher matcher = + java.util.regex.Pattern.compile("\"time_unix_nano\":(\\d+)").matcher(json); + assertTrue(matcher.find(), "profile window must be present, got: " + json); + assertTrue( + Long.parseLong(matcher.group(1)) > 0, + "derived start bound must not stay at the synthetic epoch value, got: " + json); + } + + @Test + void convertRecordingWithMethodSample() throws IOException { + Path jfrFile = tempDir.resolve("wall.jfr"); + + try (Recording recording = Recordings.newRecording(jfrFile)) { + // Register custom datadog.MethodSample event type + Type methodSampleType = + recording.registerEventType( + "datadog.MethodSample", + type -> { + type.addField("spanId", Types.Builtin.LONG); + type.addField("localRootSpanId", Types.Builtin.LONG); + }); + + // Write method sample event + writeEvent( + recording, + methodSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", 11111L); + valueBuilder.putField("localRootSpanId", 22222L); + }); + } + + Instant start = Instant.now().minusSeconds(10); + Instant end = Instant.now(); + + byte[] result = converter.addFile(jfrFile, start, end).convert(); + + assertNotNull(result); + assertTrue(result.length > 0); + } + + @Test + void convertRecordingWithObjectSample() throws IOException { + Path jfrFile = tempDir.resolve("alloc.jfr"); + + try (Recording recording = Recordings.newRecording(jfrFile)) { + // Register custom datadog.ObjectSample event type + Type objectSampleType = + recording.registerEventType( + "datadog.ObjectSample", + type -> { + type.addField("spanId", Types.Builtin.LONG); + type.addField("localRootSpanId", Types.Builtin.LONG); + type.addField("size", Types.Builtin.LONG); + type.addField("weight", Types.Builtin.FLOAT); + }); + + // Write object sample event + writeEvent( + recording, + objectSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", 33333L); + valueBuilder.putField("localRootSpanId", 44444L); + valueBuilder.putField("size", 1024L); + valueBuilder.putField("weight", 3.5f); + }); + } + + Instant start = Instant.now().minusSeconds(10); + Instant end = Instant.now(); + + byte[] result = converter.addFile(jfrFile, start, end).convert(); + + assertNotNull(result); + assertTrue(result.length > 0); + } + + @Test + void convertRecordingWithJavaMonitorEnter() throws IOException { + Path jfrFile = tempDir.resolve("lock.jfr"); + + try (Recording recording = Recordings.newRecording(jfrFile)) { + // Register jdk.JavaMonitorEnter event type + Type monitorEnterType = + recording.registerEventType( + "jdk.JavaMonitorEnter", + type -> { + type.addField("duration", Types.Builtin.LONG); + }); + + // Write monitor enter event + writeEvent( + recording, + monitorEnterType, + valueBuilder -> { + valueBuilder.putField("duration", 5000000L); // 5ms in nanos + }); + } + + Instant start = Instant.now().minusSeconds(10); + Instant end = Instant.now(); + + byte[] result = converter.addFile(jfrFile, start, end).convert(); + + assertNotNull(result); + assertTrue(result.length > 0); + } + + @Test + void convertRecordingWithMultipleExecutionSamples() throws IOException { + Path jfrFile = tempDir.resolve("multiple-cpu.jfr"); + + try (Recording recording = Recordings.newRecording(jfrFile)) { + // Register custom datadog.ExecutionSample event type + Type executionSampleType = + recording.registerEventType( + "datadog.ExecutionSample", + type -> { + type.addField("spanId", Types.Builtin.LONG); + type.addField("localRootSpanId", Types.Builtin.LONG); + }); + + // Write multiple execution sample events with different trace contexts + writeEvent( + recording, + executionSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", 100L); + valueBuilder.putField("localRootSpanId", 200L); + }); + + writeEvent( + recording, + executionSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", 300L); + valueBuilder.putField("localRootSpanId", 400L); + }); + + writeEvent( + recording, + executionSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", 100L); // Same as first sample + valueBuilder.putField("localRootSpanId", 200L); + }); + } + + Instant start = Instant.now().minusSeconds(10); + Instant end = Instant.now(); + + byte[] result = converter.addFile(jfrFile, start, end).convert(); + + assertNotNull(result); + assertTrue(result.length > 0); + } + + @Test + void convertRecordingWithMultipleMethodSamples() throws IOException { + Path jfrFile = tempDir.resolve("multiple-wall.jfr"); + + try (Recording recording = Recordings.newRecording(jfrFile)) { + // Register custom datadog.MethodSample event type + Type methodSampleType = + recording.registerEventType( + "datadog.MethodSample", + type -> { + type.addField("spanId", Types.Builtin.LONG); + type.addField("localRootSpanId", Types.Builtin.LONG); + }); + + // Write multiple method sample events + for (int i = 0; i < 5; i++) { + final long spanId = i * 100L; + final long rootSpanId = i * 100L + 50; + writeEvent( + recording, + methodSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", spanId); + valueBuilder.putField("localRootSpanId", rootSpanId); + }); + } + } + + Instant start = Instant.now().minusSeconds(10); + Instant end = Instant.now(); + + byte[] result = converter.addFile(jfrFile, start, end).convert(); + + assertNotNull(result); + assertTrue(result.length > 0); + } + + @Test + void convertRecordingWithMultipleObjectSamples() throws IOException { + Path jfrFile = tempDir.resolve("multiple-alloc.jfr"); + + try (Recording recording = Recordings.newRecording(jfrFile)) { + // Register custom datadog.ObjectSample event type + Type objectSampleType = + recording.registerEventType( + "datadog.ObjectSample", + type -> { + type.addField("spanId", Types.Builtin.LONG); + type.addField("localRootSpanId", Types.Builtin.LONG); + type.addField("size", Types.Builtin.LONG); + type.addField("weight", Types.Builtin.FLOAT); + }); + + // Write multiple object sample events with varying allocation sizes + writeEvent( + recording, + objectSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", 1000L); + valueBuilder.putField("localRootSpanId", 2000L); + valueBuilder.putField("size", 1024L); + valueBuilder.putField("weight", 3.5f); + }); + + writeEvent( + recording, + objectSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", 3000L); + valueBuilder.putField("localRootSpanId", 4000L); + valueBuilder.putField("size", 2048L); + valueBuilder.putField("weight", 1.5f); + }); + + writeEvent( + recording, + objectSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", 1000L); // Same trace as first + valueBuilder.putField("localRootSpanId", 2000L); + valueBuilder.putField("size", 4096L); + valueBuilder.putField("weight", 0.8f); + }); + } + + Instant start = Instant.now().minusSeconds(10); + Instant end = Instant.now(); + + byte[] result = converter.addFile(jfrFile, start, end).convert(); + + assertNotNull(result); + assertTrue(result.length > 0); + } + + @Test + void convertRecordingWithMultipleMonitorSamples() throws IOException { + Path jfrFile = tempDir.resolve("multiple-lock.jfr"); + + try (Recording recording = Recordings.newRecording(jfrFile)) { + // Register jdk.JavaMonitorEnter event type + Type monitorEnterType = + recording.registerEventType( + "jdk.JavaMonitorEnter", + type -> { + type.addField("duration", Types.Builtin.LONG); + }); + + // Write multiple monitor enter events with varying durations + writeEvent( + recording, + monitorEnterType, + valueBuilder -> { + valueBuilder.putField("duration", 1000000L); // 1ms + }); + + writeEvent( + recording, + monitorEnterType, + valueBuilder -> { + valueBuilder.putField("duration", 5000000L); // 5ms + }); + + writeEvent( + recording, + monitorEnterType, + valueBuilder -> { + valueBuilder.putField("duration", 10000000L); // 10ms + }); + } + + Instant start = Instant.now().minusSeconds(10); + Instant end = Instant.now(); + + byte[] result = converter.addFile(jfrFile, start, end).convert(); + + assertNotNull(result); + assertTrue(result.length > 0); + } + + @Test + void convertRecordingWithMixedEventTypes() throws IOException { + Path jfrFile = tempDir.resolve("mixed-events.jfr"); + + try (Recording recording = Recordings.newRecording(jfrFile)) { + // Register multiple event types + Type executionSampleType = + recording.registerEventType( + "datadog.ExecutionSample", + type -> { + type.addField("spanId", Types.Builtin.LONG); + type.addField("localRootSpanId", Types.Builtin.LONG); + }); + + Type methodSampleType = + recording.registerEventType( + "datadog.MethodSample", + type -> { + type.addField("spanId", Types.Builtin.LONG); + type.addField("localRootSpanId", Types.Builtin.LONG); + }); + + Type objectSampleType = + recording.registerEventType( + "datadog.ObjectSample", + type -> { + type.addField("spanId", Types.Builtin.LONG); + type.addField("localRootSpanId", Types.Builtin.LONG); + type.addField("size", Types.Builtin.LONG); + type.addField("weight", Types.Builtin.FLOAT); + }); + + // Write events of different types with same trace context + long sharedSpanId = 9999L; + long sharedRootSpanId = 8888L; + + writeEvent( + recording, + executionSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", sharedSpanId); + valueBuilder.putField("localRootSpanId", sharedRootSpanId); + }); + + writeEvent( + recording, + methodSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", sharedSpanId); + valueBuilder.putField("localRootSpanId", sharedRootSpanId); + }); + + writeEvent( + recording, + objectSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", sharedSpanId); + valueBuilder.putField("localRootSpanId", sharedRootSpanId); + valueBuilder.putField("size", 4096L); + valueBuilder.putField("weight", 1.1f); + }); + + // Add more ExecutionSamples + writeEvent( + recording, + executionSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", sharedSpanId); + valueBuilder.putField("localRootSpanId", sharedRootSpanId); + }); + } + + Instant start = Instant.now().minusSeconds(10); + Instant end = Instant.now(); + + byte[] result = converter.addFile(jfrFile, start, end).convert(); + + assertNotNull(result); + assertTrue(result.length > 0); + } + + @Test + void convertMultipleRecordings() throws IOException { + Path jfrFile1 = tempDir.resolve("recording1.jfr"); + Path jfrFile2 = tempDir.resolve("recording2.jfr"); + + // Create first recording with execution sample + try (Recording recording = Recordings.newRecording(jfrFile1)) { + Type executionSampleType = + recording.registerEventType( + "datadog.ExecutionSample", + type -> { + type.addField("spanId", Types.Builtin.LONG); + type.addField("localRootSpanId", Types.Builtin.LONG); + }); + + writeEvent( + recording, + executionSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", 1L); + valueBuilder.putField("localRootSpanId", 2L); + }); + } + + // Create second recording with method sample + try (Recording recording = Recordings.newRecording(jfrFile2)) { + Type methodSampleType = + recording.registerEventType( + "datadog.MethodSample", + type -> { + type.addField("spanId", Types.Builtin.LONG); + type.addField("localRootSpanId", Types.Builtin.LONG); + }); + + writeEvent( + recording, + methodSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", 3L); + valueBuilder.putField("localRootSpanId", 4L); + }); + } + + Instant start = Instant.now().minusSeconds(20); + Instant middle = Instant.now().minusSeconds(10); + Instant end = Instant.now(); + + // Convert both recordings together + byte[] result = + converter.addFile(jfrFile1, start, middle).addFile(jfrFile2, middle, end).convert(); + + assertNotNull(result); + assertTrue(result.length > 0); + } + + @Test + void converterCanBeReused() throws IOException { + Path jfrFile = tempDir.resolve("reuse.jfr"); + + // Create a recording with a matching event type + try (Recording recording = Recordings.newRecording(jfrFile)) { + Type executionSampleType = + recording.registerEventType( + "datadog.ExecutionSample", + type -> { + type.addField("spanId", Types.Builtin.LONG); + type.addField("localRootSpanId", Types.Builtin.LONG); + }); + + writeEvent( + recording, + executionSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", 42L); + valueBuilder.putField("localRootSpanId", 42L); + }); + } + + Instant start = Instant.now().minusSeconds(10); + Instant end = Instant.now(); + + // First conversion + byte[] result1 = converter.addFile(jfrFile, start, end).convert(); + assertNotNull(result1); + assertTrue(result1.length > 0); + + // Second conversion (reusing the same converter) + byte[] result2 = converter.addFile(jfrFile, start, end).convert(); + assertNotNull(result2); + assertTrue(result2.length > 0); + } + + @Test + void convertRecordingWithThousandsOfSamples() throws IOException { + Path jfrFile = tempDir.resolve("thousands-of-samples.jfr"); + + // Create recording with 10,000 ExecutionSample events + // Using 100 unique trace contexts, each repeated 100 times + try (Recording recording = Recordings.newRecording(jfrFile)) { + Type executionSampleType = + recording.registerEventType( + "datadog.ExecutionSample", + type -> { + type.addField("spanId", Types.Builtin.LONG); + type.addField("localRootSpanId", Types.Builtin.LONG); + }); + + // Write 10,000 events with 100 unique trace contexts + for (int contextId = 0; contextId < 100; contextId++) { + long spanId = 10000L + contextId; + long rootSpanId = 20000L + contextId; + + // Each context appears 100 times + for (int repeat = 0; repeat < 100; repeat++) { + writeEvent( + recording, + executionSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", spanId); + valueBuilder.putField("localRootSpanId", rootSpanId); + }); + } + } + } + + Instant start = Instant.now().minusSeconds(60); + Instant end = Instant.now(); + + byte[] result = converter.addFile(jfrFile, start, end).convert(); + + assertNotNull(result, "Result should not be null"); + assertTrue(result.length > 0, "Result should not be empty"); + } + + @Test + void convertRecordingWithRandomStacktraceDepths() throws IOException { + Path jfrFile = tempDir.resolve("random-stacks.jfr"); + + try (Recording recording = Recordings.newRecording(jfrFile)) { + // Get Types instance for creating typed values + Types types = recording.getTypes(); + + // Register event type - stackTrace field is added automatically for event types + Type executionSampleType = + recording.registerEventType( + "datadog.ExecutionSample", + type -> { + type.addField("spanId", Types.Builtin.LONG); + type.addField("localRootSpanId", Types.Builtin.LONG); + }); + + // Generate 1,000 events with random stack traces of varying depths (5-128 frames) + // This tests deduplication with diverse but manageable memory footprint + java.util.Random random = new java.util.Random(42); // Fixed seed for reproducibility + int eventCount = 5000; + + for (int i = 0; i < eventCount; i++) { + // Random stack depth between 5 and 128 frames + int stackDepth = 5 + random.nextInt(124); + + // Generate random stack trace + StackTraceElement[] stackTrace = new StackTraceElement[stackDepth]; + for (int frameIdx = 0; frameIdx < stackDepth; frameIdx++) { + // Create diverse class/method names to test deduplication + int classId = random.nextInt(200); // 200 different classes + int methodId = random.nextInt(50); // 50 different methods per class + int lineNumber = 10 + random.nextInt(990); // Random line numbers + + stackTrace[frameIdx] = + new StackTraceElement( + "com.example.Class" + classId, + "method" + methodId, + "Class" + classId + ".java", + lineNumber); + } + + // Use moderate trace context cardinality (1000 unique contexts) + long contextId = random.nextInt(1000); + final long spanId = 50000L + contextId; + final long rootSpanId = 60000L + contextId; + final StackTraceElement[] finalStackTrace = stackTrace; + + // Write event with manually constructed stack trace + recording.writeEvent( + executionSampleType.asValue( + valueBuilder -> { + valueBuilder.putField("startTime", epochNanos()); + valueBuilder.putField("spanId", spanId); + valueBuilder.putField("localRootSpanId", rootSpanId); + valueBuilder.putField( + "stackTrace", + stackTraceBuilder -> + putStackTrace(types, stackTraceBuilder, finalStackTrace)); + })); + } + } + + Instant start = Instant.now().minusSeconds(60); + Instant end = Instant.now(); + + byte[] result = converter.addFile(jfrFile, start, end).convert(); + + assertNotNull(result, "Result should not be null"); + assertTrue(result.length > 0, "Result should not be empty"); + } + + @Test + void convertEmptyRecordingToJson() throws IOException { + Path jfrFile = tempDir.resolve("empty.jfr"); + + // Create empty JFR file + try (Recording recording = Recordings.newRecording(jfrFile)) { + // Just create an empty recording - no events + } + + Instant start = Instant.now().minusSeconds(10); + Instant end = Instant.now(); + + byte[] result = converter.addFile(jfrFile, start, end).convert(JfrToOtlpConverter.Kind.JSON); + + // Verify JSON output is valid + assertNotNull(result); + String json = new String(result, StandardCharsets.UTF_8); + assertTrue(json.contains("\"resource_profiles\"")); + assertTrue(json.contains("\"dictionary\"")); + System.out.println("JSON output:\n" + json); + } + + @Test + void convertWithOriginalPayloadDisabledByDefault() throws IOException { + Path jfrFile = tempDir.resolve("no-payload.jfr"); + + try (Recording recording = Recordings.newRecording(jfrFile)) { + Type executionSampleType = + recording.registerEventType( + "datadog.ExecutionSample", + type -> { + type.addField("spanId", Types.Builtin.LONG); + type.addField("localRootSpanId", Types.Builtin.LONG); + }); + + writeEvent( + recording, + executionSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", 100L); + valueBuilder.putField("localRootSpanId", 200L); + }); + } + + Instant start = Instant.now().minusSeconds(10); + Instant end = Instant.now(); + + // Convert without setting includeOriginalPayload (default is false) + byte[] result = converter.addFile(jfrFile, start, end).convert(); + + assertNotNull(result); + assertTrue(result.length > 0); + + // Result should be smaller than with payload + // (Note: can't easily verify absence of field in raw protobuf bytes) + } + + @Test + void convertWithOriginalPayloadEnabled() throws IOException { + Path jfrFile = tempDir.resolve("with-payload.jfr"); + long jfrFileSize; + + try (Recording recording = Recordings.newRecording(jfrFile)) { + Type executionSampleType = + recording.registerEventType( + "datadog.ExecutionSample", + type -> { + type.addField("spanId", Types.Builtin.LONG); + type.addField("localRootSpanId", Types.Builtin.LONG); + }); + + writeEvent( + recording, + executionSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", 100L); + valueBuilder.putField("localRootSpanId", 200L); + }); + } + + jfrFileSize = java.nio.file.Files.size(jfrFile); + + Instant start = Instant.now().minusSeconds(10); + Instant end = Instant.now(); + + // Convert WITH original payload + byte[] resultWithPayload = + converter.setIncludeOriginalPayload(true).addFile(jfrFile, start, end).convert(); + + assertNotNull(resultWithPayload); + assertTrue(resultWithPayload.length > 0); + + // Result should be at least as large as the JFR file size (contains JFR + OTLP overhead) + assertTrue( + resultWithPayload.length >= jfrFileSize, + String.format( + "Result size %d should be >= JFR file size %d", resultWithPayload.length, jfrFileSize)); + } + + @Test + void convertMultipleRecordingsWithOriginalPayload() throws IOException { + Path jfrFile1 = tempDir.resolve("payload1.jfr"); + Path jfrFile2 = tempDir.resolve("payload2.jfr"); + long totalJfrSize; + + // Create first recording + try (Recording recording = Recordings.newRecording(jfrFile1)) { + Type executionSampleType = + recording.registerEventType( + "datadog.ExecutionSample", + type -> { + type.addField("spanId", Types.Builtin.LONG); + type.addField("localRootSpanId", Types.Builtin.LONG); + }); + + writeEvent( + recording, + executionSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", 1L); + valueBuilder.putField("localRootSpanId", 2L); + }); + } + + // Create second recording + try (Recording recording = Recordings.newRecording(jfrFile2)) { + Type methodSampleType = + recording.registerEventType( + "datadog.MethodSample", + type -> { + type.addField("spanId", Types.Builtin.LONG); + type.addField("localRootSpanId", Types.Builtin.LONG); + }); + + writeEvent( + recording, + methodSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", 3L); + valueBuilder.putField("localRootSpanId", 4L); + }); + } + + totalJfrSize = java.nio.file.Files.size(jfrFile1) + java.nio.file.Files.size(jfrFile2); + + Instant start = Instant.now().minusSeconds(20); + Instant middle = Instant.now().minusSeconds(10); + Instant end = Instant.now(); + + // Convert both recordings with original payload (creates "uber-JFR") + byte[] result = + converter + .setIncludeOriginalPayload(true) + .addFile(jfrFile1, start, middle) + .addFile(jfrFile2, middle, end) + .convert(); + + assertNotNull(result); + assertTrue(result.length > 0); + + // Result should contain concatenated JFR files + assertTrue( + result.length >= totalJfrSize, + String.format( + "Result size %d should be >= combined JFR size %d", result.length, totalJfrSize)); + } + + @Test + void converterResetsOriginalPayloadSetting() throws IOException { + Path jfrFile = tempDir.resolve("reset-test.jfr"); + + try (Recording recording = Recordings.newRecording(jfrFile)) { + Type executionSampleType = + recording.registerEventType( + "datadog.ExecutionSample", + type -> { + type.addField("spanId", Types.Builtin.LONG); + type.addField("localRootSpanId", Types.Builtin.LONG); + }); + + writeEvent( + recording, + executionSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", 42L); + valueBuilder.putField("localRootSpanId", 42L); + }); + } + + long jfrFileSize = java.nio.file.Files.size(jfrFile); + Instant start = Instant.now().minusSeconds(10); + Instant end = Instant.now(); + + // First conversion WITH payload + byte[] result1 = + converter.setIncludeOriginalPayload(true).addFile(jfrFile, start, end).convert(); + + assertTrue(result1.length >= jfrFileSize, "First conversion should include payload"); + + // Setting is preserved for reuse (not reset after convert()) + byte[] result2 = converter.addFile(jfrFile, start, end).convert(); + + assertTrue(result2.length >= jfrFileSize, "Second conversion should still include payload"); + + // Explicitly disable for third conversion + byte[] result3 = + converter.setIncludeOriginalPayload(false).addFile(jfrFile, start, end).convert(); + + // Third result should be smaller (no payload) + assertTrue( + result3.length < result1.length, "Third conversion without payload should be smaller"); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/JfrTools.java b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/JfrTools.java new file mode 100644 index 00000000000..0a1813aae75 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/JfrTools.java @@ -0,0 +1,88 @@ +package com.datadog.profiling.otel; + +import java.time.Instant; +import java.util.function.Consumer; +import org.openjdk.jmc.flightrecorder.writer.api.Recording; +import org.openjdk.jmc.flightrecorder.writer.api.Type; +import org.openjdk.jmc.flightrecorder.writer.api.TypedValue; +import org.openjdk.jmc.flightrecorder.writer.api.TypedValueBuilder; +import org.openjdk.jmc.flightrecorder.writer.api.Types; + +public final class JfrTools { + /** + * Helper method to write JFR events with automatic startTime field. + * + *

This ensures all events have the required startTime field set, which is necessary for the + * JFR parser to correctly read subsequent field values. + * + * @param recording the JFR recording to write to + * @param eventType the event type to create + * @param fieldSetter consumer that sets additional event fields + */ + public static void writeEvent( + Recording recording, Type eventType, Consumer fieldSetter) { + recording.writeEvent( + eventType.asValue( + valueBuilder -> { + // JMC writer 8.1.0 stores the chunk start epoch nanos as the chunk start ticks (at 1 + // tick/ns), so event ticks must be epoch nanos to convert to wall-clock time + valueBuilder.putField("startTime", epochNanos()); + fieldSetter.accept(valueBuilder); + })); + } + + /** + * Current time as epoch nanos, the event tick value matching the chunk start written by JMC + * writer 8.1.0. + */ + public static long epochNanos() { + Instant now = Instant.now(); + return now.getEpochSecond() * 1_000_000_000L + now.getNano(); + } + + /** + * Helper method to build a JFR stack trace field value from StackTraceElement array. + * + *

Constructs the proper JFR stack trace structure: { frames: StackFrame[], truncated: boolean + * } where each StackFrame contains: { method: { type: { name: String }, name: String }, + * lineNumber: int, bytecodeIndex: int, type: String } + * + * @param types the Types instance from the recording + * @param stackTraceBuilder the builder to construct the stack trace value + * @param stackTrace the stack trace elements to convert + */ + public static void putStackTrace( + Types types, TypedValueBuilder stackTraceBuilder, StackTraceElement[] stackTrace) { + // Get the StackFrame type + Type stackFrameType = types.getType(Types.JDK.STACK_FRAME); + + // Build array of stack frame TypedValues + TypedValue[] frames = new TypedValue[stackTrace.length]; + for (int i = 0; i < stackTrace.length; i++) { + StackTraceElement element = stackTrace[i]; + frames[i] = + stackFrameType.asValue( + frameBuilder -> { + // Build method: { type: Class, name: String } + frameBuilder.putField( + "method", + methodBuilder -> { + // Build type (Class): { name: String } + methodBuilder.putField( + "type", + classBuilder -> { + classBuilder.putField("name", element.getClassName()); + }); + methodBuilder.putField("name", element.getMethodName()); + }); + frameBuilder.putField("lineNumber", element.getLineNumber()); + frameBuilder.putField("bytecodeIndex", -1); + frameBuilder.putField("type", element.isNativeMethod() ? "Native" : "Java"); + }); + } + + // Set the frames array and truncated flag + stackTraceBuilder.putField("frames", frames); + stackTraceBuilder.putField("truncated", stackTrace.length > 8192); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/LongKeyIntMapTest.java b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/LongKeyIntMapTest.java new file mode 100644 index 00000000000..c9a08973bb8 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/LongKeyIntMapTest.java @@ -0,0 +1,72 @@ +package com.datadog.profiling.otel; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class LongKeyIntMapTest { + + @Test + void singleKeyPutGet() { + LongKeyIntMap map = new LongKeyIntMap(1, 16); + assertEquals(-1, map.get1(42L)); + + map.put1(42L, 7); + assertEquals(7, map.get1(42L)); + assertEquals(-1, map.get1(43L)); + + // replacement + map.put1(42L, 9); + assertEquals(9, map.get1(42L)); + } + + @Test + void singleKeyAnyLongValueIsLegal() { + LongKeyIntMap map = new LongKeyIntMap(1, 16); + map.put1(0L, 1); + map.put1(-1L, 2); + map.put1(Long.MIN_VALUE, 3); + map.put1(Long.MAX_VALUE, 4); + + assertEquals(1, map.get1(0L)); + assertEquals(2, map.get1(-1L)); + assertEquals(3, map.get1(Long.MIN_VALUE)); + assertEquals(4, map.get1(Long.MAX_VALUE)); + } + + @Test + void tripleKeyPutGet() { + LongKeyIntMap map = new LongKeyIntMap(3, 16); + assertEquals(-1, map.get3(1L, 2L, 3L)); + + map.put3(1L, 2L, 3L, 10); + assertEquals(10, map.get3(1L, 2L, 3L)); + assertEquals(-1, map.get3(1L, 2L, 4L)); + assertEquals(-1, map.get3(0L, 2L, 3L)); + + map.put3(1L, 2L, 3L, 11); + assertEquals(11, map.get3(1L, 2L, 3L)); + } + + @Test + void growsBeyondInitialCapacity() { + LongKeyIntMap map = new LongKeyIntMap(1, 16); + int count = 10_000; + for (long i = 0; i < count; i++) { + map.put1(i, (int) (i * 3)); + } + assertEquals(count, map.size()); + for (long i = 0; i < count; i++) { + assertEquals((int) (i * 3), map.get1(i)); + } + } + + @Test + void clearResetsContents() { + LongKeyIntMap map = new LongKeyIntMap(3, 16); + map.put3(1L, 2L, 3L, 5); + map.clear(); + assertEquals(0, map.size()); + assertEquals(-1, map.get3(1L, 2L, 3L)); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/ProfcheckValidationTest.java b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/ProfcheckValidationTest.java new file mode 100644 index 00000000000..3cbd08e4429 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/ProfcheckValidationTest.java @@ -0,0 +1,315 @@ +/* + * Copyright 2025 Datadog + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datadog.profiling.otel; + +import static com.datadog.profiling.otel.JfrTools.putStackTrace; +import static com.datadog.profiling.otel.JfrTools.writeEvent; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.junit.jupiter.api.io.TempDir; +import org.openjdk.jmc.flightrecorder.writer.api.Recording; +import org.openjdk.jmc.flightrecorder.writer.api.Recordings; +import org.openjdk.jmc.flightrecorder.writer.api.Type; +import org.openjdk.jmc.flightrecorder.writer.api.Types; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.MountableFile; + +/** + * Integration test that validates OTLP profiles against OpenTelemetry's profcheck conformance + * checker. + * + *

This test: + * + *

    + *
  • Generates synthetic JFR recordings using JMC API + *
  • Converts them to OTLP protobuf format + *
  • Validates with profcheck running in a Docker container + *
+ * + *

Requires Docker to be running. + */ +@Testcontainers +@Tag("docker") +@EnabledIfSystemProperty(named = "run.otlp.validation", matches = "true") +public class ProfcheckValidationTest { + + // Profcheck container built from Dockerfile.profcheck + // Note: We override the entrypoint to keep the container running since profcheck + // normally exits after validation. We use the container for multiple validations. + @Container + private static final GenericContainer profcheckContainer = + new GenericContainer<>("profcheck:latest") + .withCreateContainerCmdModifier(cmd -> cmd.withEntrypoint("/bin/sh")) + .withCommand("-c", "while true; do sleep 1; done"); + + @TempDir Path tempDir; + + @Test + public void testEmptyProfile() throws Exception { + // Generate empty JFR recording + Path jfrFile = tempDir.resolve("empty.jfr"); + try (Recording recording = Recordings.newRecording(jfrFile)) { + // Empty recording - no events + } + + // Convert to OTLP + Path otlpFile = tempDir.resolve("empty.pb"); + byte[] otlpData = convertJfrToOtlp(jfrFile); + Files.write(otlpFile, otlpData); + + // Validate with profcheck (now includes protoc validation) + String result = validateWithProfcheck(otlpFile); + + // Empty profiles should pass protoc structural validation + assertTrue( + result.contains("protoc validation PASSED"), + "Empty profile should pass protoc validation (spec-compliant). Output: " + result); + } + + @Test + public void testCpuProfile() throws Exception { + // Generate JFR recording with CPU samples + Path jfrFile = tempDir.resolve("cpu.jfr"); + + try (Recording recording = Recordings.newRecording(jfrFile)) { + Types types = recording.getTypes(); + + Type executionSampleType = + recording.registerEventType( + "datadog.ExecutionSample", + type -> { + type.addField("spanId", Types.Builtin.LONG); + type.addField("localRootSpanId", Types.Builtin.LONG); + }); + + // Add 100 CPU samples with various stack traces + for (int i = 0; i < 100; i++) { + final long spanId = 10000L + i; + final long rootSpanId = 20000L + (i % 10); + + StackTraceElement[] stackTrace = + new StackTraceElement[] { + new StackTraceElement("com.example.App", "main", "App.java", 42), + new StackTraceElement("com.example.Service", "process", "Service.java", 123), + new StackTraceElement("com.example.Util", "helper", "Util.java", 78) + }; + + writeEvent( + recording, + executionSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", spanId); + valueBuilder.putField("localRootSpanId", rootSpanId); + valueBuilder.putField( + "stackTrace", + stackTraceBuilder -> putStackTrace(types, stackTraceBuilder, stackTrace)); + }); + } + } + + // Convert to OTLP + Path otlpFile = tempDir.resolve("cpu.pb"); + byte[] otlpData = convertJfrToOtlp(jfrFile); + Files.write(otlpFile, otlpData); + + // Validate with profcheck (includes both protoc and profcheck validation) + String result = validateWithProfcheck(otlpFile); + + // Both validators must pass + assertTrue( + result.contains("protoc validation PASSED"), + "CPU profile should pass protoc validation (spec-compliant). Output: " + result); + assertTrue( + result.contains("profcheck validation PASSED"), + "CPU profile should pass profcheck validation. Output: " + result); + } + + @Test + public void testAllocationProfile() throws Exception { + // Generate JFR recording with allocation samples + Path jfrFile = tempDir.resolve("alloc.jfr"); + + try (Recording recording = Recordings.newRecording(jfrFile)) { + Types types = recording.getTypes(); + + Type objectSampleType = + recording.registerEventType( + "datadog.ObjectSample", + type -> { + type.addField("size", Types.Builtin.LONG); + type.addField("weight", Types.Builtin.FLOAT); + type.addField("spanId", Types.Builtin.LONG); + type.addField("localRootSpanId", Types.Builtin.LONG); + }); + + // Add 50 allocation samples + for (int i = 0; i < 50; i++) { + final long weight = 1024L * (i + 1); + final long spanId = 30000L + i; + final long rootSpanId = 40000L + (i % 5); + + StackTraceElement[] stackTrace = + new StackTraceElement[] { + new StackTraceElement("com.example.Factory", "create", "Factory.java", 55), + new StackTraceElement("com.example.Builder", "build", "Builder.java", 89) + }; + + writeEvent( + recording, + objectSampleType, + valueBuilder -> { + valueBuilder.putField("size", weight); + valueBuilder.putField("weight", 0.9f); + valueBuilder.putField("spanId", spanId); + valueBuilder.putField("localRootSpanId", rootSpanId); + valueBuilder.putField( + "stackTrace", + stackTraceBuilder -> putStackTrace(types, stackTraceBuilder, stackTrace)); + }); + } + } + + // Convert to OTLP + Path otlpFile = tempDir.resolve("alloc.pb"); + byte[] otlpData = convertJfrToOtlp(jfrFile); + Files.write(otlpFile, otlpData); + + // Validate with profcheck (includes both protoc and profcheck validation) + String result = validateWithProfcheck(otlpFile); + + // Both validators must pass + assertTrue( + result.contains("protoc validation PASSED"), + "Allocation profile should pass protoc validation (spec-compliant). Output: " + result); + assertTrue( + result.contains("profcheck validation PASSED"), + "Allocation profile should pass profcheck validation. Output: " + result); + } + + @Test + public void testMixedProfile() throws Exception { + // Generate JFR recording with multiple event types + Path jfrFile = tempDir.resolve("mixed.jfr"); + + try (Recording recording = Recordings.newRecording(jfrFile)) { + Types types = recording.getTypes(); + + // CPU samples + Type executionSampleType = + recording.registerEventType( + "datadog.ExecutionSample", + type -> { + type.addField("spanId", Types.Builtin.LONG); + type.addField("localRootSpanId", Types.Builtin.LONG); + }); + + // Wall clock samples + Type methodSampleType = + recording.registerEventType( + "datadog.MethodSample", + type -> { + type.addField("spanId", Types.Builtin.LONG); + type.addField("localRootSpanId", Types.Builtin.LONG); + }); + + StackTraceElement[] stackTrace = + new StackTraceElement[] { + new StackTraceElement("com.example.Main", "run", "Main.java", 100) + }; + + // Add mix of events + for (int i = 0; i < 20; i++) { + final long spanId = 50000L + i; + final long rootSpanId = 60000L; + + // CPU sample + writeEvent( + recording, + executionSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", spanId); + valueBuilder.putField("localRootSpanId", rootSpanId); + valueBuilder.putField( + "stackTrace", + stackTraceBuilder -> putStackTrace(types, stackTraceBuilder, stackTrace)); + }); + + // Wall clock sample + writeEvent( + recording, + methodSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", spanId); + valueBuilder.putField("localRootSpanId", rootSpanId); + valueBuilder.putField( + "stackTrace", + stackTraceBuilder -> putStackTrace(types, stackTraceBuilder, stackTrace)); + }); + } + } + + // Convert to OTLP + Path otlpFile = tempDir.resolve("mixed.pb"); + byte[] otlpData = convertJfrToOtlp(jfrFile); + Files.write(otlpFile, otlpData); + + // Validate with profcheck (includes both protoc and profcheck validation) + String result = validateWithProfcheck(otlpFile); + + // Both validators must pass + assertTrue( + result.contains("protoc validation PASSED"), + "Mixed profile should pass protoc validation (spec-compliant). Output: " + result); + assertTrue( + result.contains("profcheck validation PASSED"), + "Mixed profile should pass profcheck validation. Output: " + result); + } + + private byte[] convertJfrToOtlp(Path jfrFile) throws IOException { + Instant start = Instant.now().minusSeconds(60); + Instant end = Instant.now(); + + JfrToOtlpConverter converter = new JfrToOtlpConverter(); + return converter.addFile(jfrFile, start, end).convert(); + } + + private String validateWithProfcheck(Path otlpFile) throws Exception { + // Copy file into container + profcheckContainer.copyFileToContainer( + MountableFile.forHostPath(otlpFile), "/tmp/" + otlpFile.getFileName()); + + // Run validate-profile script (includes protoc + profcheck) + org.testcontainers.containers.Container.ExecResult result = + profcheckContainer.execInContainer("validate-profile", "/tmp/" + otlpFile.getFileName()); + + String output = result.getStdout() + result.getStderr(); + + // Log output for debugging + System.out.println("Validation output for " + otlpFile.getFileName() + ":"); + System.out.println(output); + + return output; + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/proto/OtlpProtoFieldsTest.java b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/proto/OtlpProtoFieldsTest.java new file mode 100644 index 00000000000..60dee37713a --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/proto/OtlpProtoFieldsTest.java @@ -0,0 +1,100 @@ +package com.datadog.profiling.otel.proto; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.junit.jupiter.api.Test; + +/** + * Checks every {@link OtlpProtoFields} constant against the field numbers in the + * opentelemetry-proto files copied into the test resources. When bumping the pinned proto version, + * replace those files with the ones from the new version. + */ +class OtlpProtoFieldsTest { + + private static final String[] PROTO_FILES = { + "/opentelemetry/proto/profiles/v1development/profiles.proto", + "/opentelemetry/proto/common/v1/common.proto", + "/opentelemetry/proto/resource/v1/resource.proto" + }; + + // top-level messages only: the closing brace of a message is at column 0 + private static final Pattern MESSAGE = + Pattern.compile("^message (\\w+) \\{(.*?)^\\}", Pattern.MULTILINE | Pattern.DOTALL); + // nested message/enum blocks, matched up to the closing brace at the same indentation, so their + // fields are not attributed to the enclosing message + private static final Pattern NESTED = + Pattern.compile( + "^([ \\t]+)(?:message|enum) \\w+ \\{.*?^\\1\\}", Pattern.MULTILINE | Pattern.DOTALL); + private static final Pattern FIELD = Pattern.compile("(\\w+)\\s*=\\s*(\\d+)\\s*;"); + private static final Pattern COMMENT = Pattern.compile("//.*"); + + @Test + void fieldNumbersMatchProtoDefinitions() throws Exception { + Map> messages = parseMessages(); + List mismatches = new ArrayList<>(); + for (Class message : OtlpProtoFields.class.getDeclaredClasses()) { + Map fields = messages.get(message.getSimpleName()); + for (Field constant : message.getDeclaredFields()) { + if (!Modifier.isStatic(constant.getModifiers()) || constant.getType() != int.class) { + continue; + } + Integer expected = fields == null ? null : fields.get(constant.getName()); + int actual = constant.getInt(null); + if (expected == null || expected != actual) { + mismatches.add( + message.getSimpleName() + + "." + + constant.getName() + + " = " + + actual + + ", proto: " + + expected); + } + } + } + assertEquals(new ArrayList(), mismatches); + } + + private static Map> parseMessages() throws IOException { + Map> messages = new HashMap<>(); + for (String file : PROTO_FILES) { + Matcher message = MESSAGE.matcher(read(file)); + while (message.find()) { + Map fields = + messages.computeIfAbsent(message.group(1), k -> new HashMap<>()); + String body = COMMENT.matcher(message.group(2)).replaceAll(""); + Matcher field = FIELD.matcher(NESTED.matcher(body).replaceAll("")); + while (field.find()) { + fields.put(field.group(1).toUpperCase(Locale.ROOT), Integer.parseInt(field.group(2))); + } + } + } + return messages; + } + + private static String read(String resource) throws IOException { + try (InputStream in = OtlpProtoFieldsTest.class.getResourceAsStream(resource)) { + assertNotNull(in, resource); + byte[] buffer = new byte[8192]; + StringBuilder content = new StringBuilder(); + int read; + while ((read = in.read(buffer)) != -1) { + content.append(new String(buffer, 0, read, UTF_8)); + } + return content.toString(); + } + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/proto/ProtobufEncoderTest.java b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/proto/ProtobufEncoderTest.java new file mode 100644 index 00000000000..f3a907f9b02 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/proto/ProtobufEncoderTest.java @@ -0,0 +1,235 @@ +package com.datadog.profiling.otel.proto; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class ProtobufEncoderTest { + + private ProtobufEncoder encoder; + + @BeforeEach + void setUp() { + encoder = new ProtobufEncoder(); + } + + @Test + void writeVarintSingleByte() { + encoder.writeVarint(0); + assertArrayEquals(new byte[] {0}, encoder.toByteArray()); + } + + @Test + void writeVarintSingleByteMax() { + encoder.writeVarint(127); + assertArrayEquals(new byte[] {127}, encoder.toByteArray()); + } + + @Test + void writeVarintTwoBytes() { + encoder.writeVarint(128); + assertArrayEquals(new byte[] {(byte) 0x80, 0x01}, encoder.toByteArray()); + } + + @Test + void writeVarintLargeValue() { + encoder.writeVarint(0xFFFFFFFFL); + // Max 32-bit value + assertArrayEquals( + new byte[] {(byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, 0x0F}, + encoder.toByteArray()); + } + + @Test + void writeFixed64() { + encoder.writeFixed64(0x0102030405060708L); + // Little-endian + assertArrayEquals( + new byte[] {0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01}, encoder.toByteArray()); + } + + @Test + void writeFixed32() { + encoder.writeFixed32(0x01020304); + // Little-endian + assertArrayEquals(new byte[] {0x04, 0x03, 0x02, 0x01}, encoder.toByteArray()); + } + + @Test + void writeTag() { + encoder.writeTag(1, ProtobufEncoder.WIRETYPE_VARINT); + // Field 1, wire type 0 = (1 << 3) | 0 = 0x08 + assertArrayEquals(new byte[] {0x08}, encoder.toByteArray()); + } + + @Test + void writeStringEmpty() { + encoder.writeString(""); + // Length 0 + assertArrayEquals(new byte[] {0x00}, encoder.toByteArray()); + } + + @Test + void writeStringNull() { + encoder.writeString(null); + // Length 0 + assertArrayEquals(new byte[] {0x00}, encoder.toByteArray()); + } + + @Test + void writeStringHello() { + encoder.writeString("hello"); + // Length 5 + "hello" + assertArrayEquals(new byte[] {0x05, 'h', 'e', 'l', 'l', 'o'}, encoder.toByteArray()); + } + + @Test + void writeBytes() { + encoder.writeBytes(new byte[] {0x01, 0x02, 0x03}); + // Length 3 + bytes + assertArrayEquals(new byte[] {0x03, 0x01, 0x02, 0x03}, encoder.toByteArray()); + } + + @Test + void writeVarintField() { + encoder.writeVarintField(1, 150); + // Tag (field 1, varint) + value 150 + // 150 = 0x96 0x01 + assertArrayEquals(new byte[] {0x08, (byte) 0x96, 0x01}, encoder.toByteArray()); + } + + @Test + void writeVarintFieldSkipsZero() { + encoder.writeVarintField(1, 0); + assertEquals(0, encoder.size()); + } + + @Test + void writeStringField() { + encoder.writeStringField(2, "test"); + // Tag (field 2, length-delimited) + length + "test" + assertArrayEquals(new byte[] {0x12, 0x04, 't', 'e', 's', 't'}, encoder.toByteArray()); + } + + @Test + void writeStringFieldSkipsEmpty() { + encoder.writeStringField(2, ""); + assertEquals(0, encoder.size()); + encoder.writeStringField(2, null); + assertEquals(0, encoder.size()); + } + + @Test + void writeBoolFieldTrue() { + encoder.writeBoolField(1, true); + // Tag (field 1, varint) + 1 + assertArrayEquals(new byte[] {0x08, 0x01}, encoder.toByteArray()); + } + + @Test + void writeBoolFieldFalseSkips() { + encoder.writeBoolField(1, false); + assertEquals(0, encoder.size()); + } + + @Test + void writeNestedMessage() { + encoder.writeNestedMessage( + 1, + nested -> { + nested.writeVarintField(1, 42); + }); + // Tag (field 1, length-delimited) + length + nested content + // Nested: tag 0x08 + varint 42 (0x2A) = 2 bytes + assertArrayEquals(new byte[] {0x0A, 0x02, 0x08, 0x2A}, encoder.toByteArray()); + } + + @Test + void writeNestedMessageEmpty() { + encoder.writeNestedMessage( + 1, + nested -> { + // empty message + }); + // Empty nested messages write tag + zero length (required for OTLP index-0 sentinels) + assertArrayEquals(new byte[] {0x0A, 0x00}, encoder.toByteArray()); + } + + @Test + void writePackedVarintFieldInts() { + encoder.writePackedVarintField(1, new int[] {1, 2, 3}); + // Tag (field 1, length-delimited) + length + packed values + // Values: 0x01, 0x02, 0x03 = 3 bytes + assertArrayEquals(new byte[] {0x0A, 0x03, 0x01, 0x02, 0x03}, encoder.toByteArray()); + } + + @Test + void writePackedVarintFieldEmpty() { + encoder.writePackedVarintField(1, new int[0]); + assertEquals(0, encoder.size()); + encoder.writePackedVarintField(1, (int[]) null); + assertEquals(0, encoder.size()); + } + + @Test + void writePackedFixed64Field() { + encoder.writePackedFixed64Field(1, new long[] {0x0102030405060708L}); + // Tag + length (8) + little-endian value + assertArrayEquals( + new byte[] {0x0A, 0x08, 0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01}, + encoder.toByteArray()); + } + + @Test + void reset() { + encoder.writeVarint(123); + assertEquals(1, encoder.size()); + encoder.reset(); + assertEquals(0, encoder.size()); + } + + @Test + void writeSignedVarintPositive() { + encoder.writeSignedVarint(1); + // ZigZag: 1 -> 2 + assertArrayEquals(new byte[] {0x02}, encoder.toByteArray()); + } + + @Test + void writeSignedVarintNegative() { + encoder.writeSignedVarint(-1); + // ZigZag: -1 -> 1 + assertArrayEquals(new byte[] {0x01}, encoder.toByteArray()); + } + + @Test + void writeBytesFieldFromStream() throws IOException { + byte[] payload = "hello".getBytes(StandardCharsets.UTF_8); + encoder.writeVarintField(1, 5); + encoder.writeBytesField(2, new ByteArrayInputStream(payload), payload.length); + + // field 1 varint (tag 0x08, value 0x05) + field 2 length-delimited (tag 0x12, len 5, bytes) + assertArrayEquals( + new byte[] {0x08, 0x05, 0x12, 0x05, 'h', 'e', 'l', 'l', 'o'}, encoder.toByteArray()); + } + + @Test + void writeBytesFieldRollsBackOnShortStream() { + encoder.writeVarintField(1, 5); + int sizeBefore = encoder.size(); + + // stream declares 10 bytes but only provides 3 — the failed field write must leave the + // encoder in its pre-write state so a caller catching the failure can continue encoding + ByteArrayInputStream shortStream = new ByteArrayInputStream(new byte[] {1, 2, 3}); + assertThrows(IOException.class, () -> encoder.writeBytesField(2, shortStream, 10)); + + assertEquals(sizeBefore, encoder.size()); + assertArrayEquals(new byte[] {0x08, 0x05}, encoder.toByteArray()); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/proto/dictionary/FunctionTableTest.java b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/proto/dictionary/FunctionTableTest.java new file mode 100644 index 00000000000..a033f00b8d8 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/proto/dictionary/FunctionTableTest.java @@ -0,0 +1,56 @@ +package com.datadog.profiling.otel.proto.dictionary; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class FunctionTableTest { + + private FunctionTable table; + + @BeforeEach + void setUp() { + table = new FunctionTable(); + } + + @Test + void indexZeroIsNullFunction() { + FunctionTable.FunctionEntry entry = table.get(0); + assertEquals(0, entry.nameIndex); + assertEquals(0, entry.systemNameIndex); + assertEquals(0, entry.filenameIndex); + assertEquals(0, entry.startLine); + assertEquals(1, table.size()); + } + + @Test + void internAllZerosReturnsIndexZero() { + assertEquals(0, table.intern(0, 0, 0, 0)); + } + + @Test + void internReturnsConsistentIndices() { + int idx1 = table.intern(1, 2, 3, 10); + int idx2 = table.intern(1, 2, 3, 10); + assertEquals(idx1, idx2); + } + + @Test + void internDifferentFunctionsReturnsDifferentIndices() { + int idx1 = table.intern(1, 2, 3, 10); + int idx2 = table.intern(1, 2, 3, 20); // different start line + assertNotEquals(idx1, idx2); + } + + @Test + void resetClearsTable() { + table.intern(1, 2, 3, 10); + table.intern(4, 5, 6, 20); + assertEquals(3, table.size()); + + table.reset(); + assertEquals(1, table.size()); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/proto/dictionary/LinkTableTest.java b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/proto/dictionary/LinkTableTest.java new file mode 100644 index 00000000000..96601a70f2f --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/proto/dictionary/LinkTableTest.java @@ -0,0 +1,154 @@ +package com.datadog.profiling.otel.proto.dictionary; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class LinkTableTest { + + private LinkTable table; + + @BeforeEach + void setUp() { + table = new LinkTable(); + } + + @Test + void indexZeroIsEmptyLink() { + LinkTable.LinkEntry entry = table.get(0); + assertEquals(16, entry.traceId.length); + assertEquals(8, entry.spanId.length); + // All zeros + for (byte b : entry.traceId) { + assertEquals(0, b); + } + for (byte b : entry.spanId) { + assertEquals(0, b); + } + assertEquals(1, table.size()); + } + + @Test + void internNullReturnsIndexZero() { + assertEquals(0, table.intern(null, null)); + assertEquals(0, table.intern(new byte[16], null)); + assertEquals(0, table.intern(null, new byte[8])); + } + + @Test + void internAllZerosReturnsIndexZero() { + assertEquals(0, table.intern(new byte[16], new byte[8])); + assertEquals(0, table.intern(0L, 0L)); + } + + @Test + void internReturnsConsistentIndices() { + byte[] traceId = new byte[16]; + traceId[0] = 1; + byte[] spanId = new byte[8]; + spanId[0] = 2; + + int idx1 = table.intern(traceId, spanId); + + byte[] traceId2 = new byte[16]; + traceId2[0] = 1; + byte[] spanId2 = new byte[8]; + spanId2[0] = 2; + + int idx2 = table.intern(traceId2, spanId2); + assertEquals(idx1, idx2); + } + + @Test + void internDifferentLinksReturnsDifferentIndices() { + byte[] traceId1 = new byte[16]; + traceId1[0] = 1; + byte[] spanId1 = new byte[8]; + spanId1[0] = 1; + + byte[] traceId2 = new byte[16]; + traceId2[0] = 2; + byte[] spanId2 = new byte[8]; + spanId2[0] = 2; + + int idx1 = table.intern(traceId1, spanId1); + int idx2 = table.intern(traceId2, spanId2); + assertNotEquals(idx1, idx2); + } + + @Test + void internFromLongValues() { + int idx = table.intern(0x123456789ABCDEF0L, 0xFEDCBA9876543210L); + assertNotEquals(0, idx); + + LinkTable.LinkEntry entry = table.get(idx); + // Trace ID should have value in lower 64 bits (big-endian) + assertEquals(0x12, entry.traceId[8] & 0xFF); + assertEquals(0x34, entry.traceId[9] & 0xFF); + // Span ID should be big-endian + assertEquals((byte) 0xFE, entry.spanId[0]); + assertEquals((byte) 0xDC, entry.spanId[1]); + } + + @Test + void internMakesDefensiveCopy() { + byte[] traceId = new byte[16]; + traceId[0] = 1; + byte[] spanId = new byte[8]; + spanId[0] = 2; + + int idx = table.intern(traceId, spanId); + traceId[0] = 99; // modify original + spanId[0] = 99; + + LinkTable.LinkEntry entry = table.get(idx); + assertEquals(1, entry.traceId[0]); // should be unchanged + assertEquals(2, entry.spanId[0]); + } + + @Test + void resetClearsTable() { + table.intern(1L, 1L); + table.intern(2L, 2L); + assertEquals(3, table.size()); + + table.reset(); + assertEquals(1, table.size()); + } + + @Test + void byteArrayAndLongOverloadsDoNotAlias() { + // a byte[] link with zero high trace-id bytes must not resolve to a long-overload link + // with the same low bits — the two key spaces are disjoint + int longIdx = table.intern(0x1122334455667788L, 0x99AABBCCDDEEFF00L); + assertNotEquals(0, longIdx); + + byte[] traceId = new byte[16]; + traceId[8] = 0x11; + traceId[9] = 0x22; + traceId[10] = 0x33; + traceId[11] = 0x44; + traceId[12] = 0x55; + traceId[13] = 0x66; + traceId[14] = 0x77; + traceId[15] = (byte) 0x88; + byte[] spanId = new byte[8]; + spanId[0] = (byte) 0x99; + spanId[1] = (byte) 0xAA; + spanId[2] = (byte) 0xBB; + spanId[3] = (byte) 0xCC; + spanId[4] = (byte) 0xDD; + spanId[5] = (byte) 0xEE; + spanId[6] = (byte) 0xFF; + spanId[7] = 0x00; + + int byteArrayIdx = table.intern(traceId, spanId); + assertNotEquals(longIdx, byteArrayIdx); + + // repeated interning of the same byte[] link stays stable within its own key space + assertEquals(byteArrayIdx, table.intern(traceId, spanId)); + assertEquals(longIdx, table.intern(0x1122334455667788L, 0x99AABBCCDDEEFF00L)); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/proto/dictionary/StackTableTest.java b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/proto/dictionary/StackTableTest.java new file mode 100644 index 00000000000..f41c786f65a --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/proto/dictionary/StackTableTest.java @@ -0,0 +1,68 @@ +package com.datadog.profiling.otel.proto.dictionary; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class StackTableTest { + + private StackTable table; + + @BeforeEach + void setUp() { + table = new StackTable(); + } + + @Test + void indexZeroIsEmptyStack() { + StackTable.StackEntry entry = table.get(0); + assertEquals(0, entry.locationIndices.length); + assertEquals(1, table.size()); + } + + @Test + void internNullReturnsIndexZero() { + assertEquals(0, table.intern(null)); + } + + @Test + void internEmptyArrayReturnsIndexZero() { + assertEquals(0, table.intern(new int[0])); + } + + @Test + void internReturnsConsistentIndices() { + int[] locations = {1, 2, 3}; + int idx1 = table.intern(locations); + int idx2 = table.intern(new int[] {1, 2, 3}); + assertEquals(idx1, idx2); + } + + @Test + void internDifferentStacksReturnsDifferentIndices() { + int idx1 = table.intern(new int[] {1, 2, 3}); + int idx2 = table.intern(new int[] {1, 2, 4}); + assertNotEquals(idx1, idx2); + } + + @Test + void internMakesDefensiveCopy() { + int[] locations = {1, 2, 3}; + int idx = table.intern(locations); + locations[0] = 999; // modify original + StackTable.StackEntry entry = table.get(idx); + assertEquals(1, entry.locationIndices[0]); // should be unchanged + } + + @Test + void resetClearsTable() { + table.intern(new int[] {1, 2, 3}); + table.intern(new int[] {4, 5, 6}); + assertEquals(3, table.size()); + + table.reset(); + assertEquals(1, table.size()); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/proto/dictionary/StringTableTest.java b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/proto/dictionary/StringTableTest.java new file mode 100644 index 00000000000..7ee76235bc0 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/proto/dictionary/StringTableTest.java @@ -0,0 +1,58 @@ +package com.datadog.profiling.otel.proto.dictionary; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class StringTableTest { + + private StringTable table; + + @BeforeEach + void setUp() { + table = new StringTable(); + } + + @Test + void indexZeroIsEmptyString() { + assertEquals("", table.get(0)); + assertEquals(1, table.size()); + } + + @Test + void internReturnsConsistentIndices() { + int idx1 = table.intern("foo"); + int idx2 = table.intern("foo"); + assertEquals(idx1, idx2); + } + + @Test + void internDifferentStringsReturnsDifferentIndices() { + int idx1 = table.intern("foo"); + int idx2 = table.intern("bar"); + assertNotEquals(idx1, idx2); + } + + @Test + void nullReturnsIndexZero() { + assertEquals(0, table.intern(null)); + } + + @Test + void emptyStringReturnsIndexZero() { + assertEquals(0, table.intern("")); + } + + @Test + void resetClearsTable() { + table.intern("foo"); + table.intern("bar"); + assertEquals(3, table.size()); + + table.reset(); + assertEquals(1, table.size()); + assertEquals("", table.get(0)); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/validation/OtlpCollectorValidationTest.java b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/validation/OtlpCollectorValidationTest.java new file mode 100644 index 00000000000..ae3194f5332 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/validation/OtlpCollectorValidationTest.java @@ -0,0 +1,231 @@ +package com.datadog.profiling.otel.validation; + +import static com.datadog.profiling.otel.JfrTools.writeEvent; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.datadog.profiling.otel.JfrToOtlpConverter; +import java.io.IOException; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.TimeUnit; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.junit.jupiter.api.io.TempDir; +import org.openjdk.jmc.flightrecorder.writer.api.Recording; +import org.openjdk.jmc.flightrecorder.writer.api.Recordings; +import org.openjdk.jmc.flightrecorder.writer.api.Type; +import org.openjdk.jmc.flightrecorder.writer.api.Types; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.MountableFile; + +/** + * Integration tests that validate OTLP profiles against a real OpenTelemetry Collector. + * + *

These tests use Testcontainers to spin up an OTel Collector instance, send generated OTLP + * profiles to it, and verify they are accepted without errors. This validates both protobuf + * encoding correctness and OTLP protocol compliance. + * + *

Note: These tests are disabled by default because they require Docker. Enable with: + * + *

+ * ./gradlew validateOtlp
+ * 
+ * + *

OTLP Profiles Status: As of 2024, OTLP profiles is in Development maturity. The OTel + * Collector may not fully support profiles yet, so these tests validate that the collector can at + * least accept and deserialize our protobuf messages without errors. + * + *

Docker Requirement: If Docker is not available, these tests will be skipped gracefully. + */ +@Tag("otlp-validation") +@EnabledIfSystemProperty(named = "run.otlp.validation", matches = "true") +@Testcontainers(disabledWithoutDocker = true) +class OtlpCollectorValidationTest { + + @TempDir Path tempDir; + + // Using the official OTel Collector Contrib image which has more receivers/exporters + private static final String OTEL_COLLECTOR_IMAGE = "otel/opentelemetry-collector-contrib:0.148.0"; + private static final int OTLP_HTTP_PORT = 4318; + + @Container + private static final GenericContainer otelCollector = + new GenericContainer<>(OTEL_COLLECTOR_IMAGE) + .withExposedPorts(OTLP_HTTP_PORT) + .withCopyFileToContainer( + MountableFile.forClasspathResource("otel-collector-config.yaml"), + "/etc/otelcol/config.yaml") + .withLogConsumer(frame -> System.out.print("[OTEL] " + frame.getUtf8String())) + .waitingFor(Wait.forLogMessage(".*Everything is ready.*", 1)) + .withStartupTimeout(Duration.ofMinutes(2)); + + @Test + void sendGeneratedProfileToCollector() throws Exception { + // Generate a simple OTLP profile + JfrToOtlpConverter converter = new JfrToOtlpConverter(); + + // Create a proper JFR file with actual event + Path tempJfr = createJfrFileWithSample(); + + Instant start = Instant.now().minusSeconds(60); + Instant end = Instant.now(); + + byte[] otlpData = converter.addFile(tempJfr, start, end).convert(); + + assertNotNull(otlpData, "Generated OTLP data should not be null"); + assertTrue(otlpData.length > 0, "Generated OTLP data should not be empty"); + + // Send to OTel Collector via HTTP + String collectorUrl = + String.format( + "http://%s:%d/v1development/profiles", + otelCollector.getHost(), otelCollector.getMappedPort(OTLP_HTTP_PORT)); + + Response response = sendWithRetry(collectorUrl, otlpData, 3); + + // Success criteria: 2xx response or 404 (endpoint not yet implemented for profiles) + // Both indicate the protobuf was at least parseable + int statusCode = response.code(); + String responseBody = response.body() != null ? response.body().string() : ""; + response.close(); + + assertTrue( + statusCode == 200 || statusCode == 202 || statusCode == 404, + String.format("Expected 2xx or 404, got %d. Body: %s", statusCode, responseBody)); + + if (statusCode == 404) { + System.out.println( + "Note: OTel Collector returned 404 - profiles endpoint may not be implemented yet. " + + "This is expected as OTLP profiles is in Development status."); + } else { + System.out.printf( + "Successfully sent OTLP profile to collector. Status: %d, Response: %s%n", + statusCode, responseBody); + } + } + + @Test + void validateProtobufDeserializability() throws Exception { + // Generate OTLP profile data + JfrToOtlpConverter converter = new JfrToOtlpConverter(); + Path tempJfr = createJfrFileWithSample(); + + Instant start = Instant.now().minusSeconds(60); + Instant end = Instant.now(); + + byte[] otlpData = converter.addFile(tempJfr, start, end).convert(); + + assertNotNull(otlpData, "Generated OTLP data should not be null"); + assertTrue(otlpData.length > 0, "Generated OTLP data should not be empty"); + + // Send to profiles endpoint + String collectorUrl = + String.format( + "http://%s:%d/v1development/profiles", + otelCollector.getHost(), otelCollector.getMappedPort(OTLP_HTTP_PORT)); + + Response response = sendWithRetry(collectorUrl, otlpData, 3); + + // We expect 2xx (success) or 404 (endpoint not implemented), but NOT 400 — the collector + // rejecting the payload as invalid protobuf is precisely the failure this test exists to + // catch — and NOT 5xx + int statusCode = response.code(); + String responseBody = response.body() != null ? response.body().string() : ""; + response.close(); + + assertTrue( + statusCode == 200 || statusCode == 202 || statusCode == 404, + String.format( + "Expected 2xx or 404 (protobuf accepted or endpoint missing), got %d — 400/5xx " + + "indicate protobuf parse failure. Body: %s", + statusCode, responseBody)); + + System.out.printf( + "Protobuf deserialization validation: Status %d (accepted or endpoint missing)%n", + statusCode); + } + + @Test + void collectorIsHealthy() { + // Sanity check that the collector container started correctly + assertTrue(otelCollector.isRunning(), "OTel Collector container should be running"); + + String host = otelCollector.getHost(); + Integer port = otelCollector.getMappedPort(OTLP_HTTP_PORT); + + assertNotNull(host, "Collector host should not be null"); + assertNotNull(port, "Collector port should not be null"); + assertTrue(port > 0, "Collector port should be positive"); + + System.out.printf("OTel Collector is healthy and accepting connections at %s:%d%n", host, port); + } + + /** Creates a proper JFR file with a sample event using JMC JFR writer. */ + private Path createJfrFileWithSample() throws IOException { + Path jfrFile = tempDir.resolve("test-profile.jfr"); + + // Create proper JFR recording with execution sample event + try (Recording recording = Recordings.newRecording(jfrFile)) { + Type executionSampleType = + recording.registerEventType( + "datadog.ExecutionSample", + type -> { + type.addField("spanId", Types.Builtin.LONG); + type.addField("localRootSpanId", Types.Builtin.LONG); + }); + + writeEvent( + recording, + executionSampleType, + valueBuilder -> { + valueBuilder.putField("spanId", 12345L); + valueBuilder.putField("localRootSpanId", 67890L); + }); + } + + return jfrFile; + } + + /** + * Sends HTTP request with retry logic to handle transient connection issues during container + * startup. + */ + private Response sendWithRetry(String url, byte[] data, int maxRetries) throws Exception { + OkHttpClient client = + new OkHttpClient.Builder() + .connectTimeout(10, TimeUnit.SECONDS) + .readTimeout(10, TimeUnit.SECONDS) + .writeTimeout(10, TimeUnit.SECONDS) + .build(); + + RequestBody body = RequestBody.create(MediaType.parse("application/x-protobuf"), data); + Request request = new Request.Builder().url(url).post(body).build(); + + Exception lastException = null; + for (int attempt = 1; attempt <= maxRetries; attempt++) { + try { + return client.newCall(request).execute(); + } catch (Exception e) { + lastException = e; + if (attempt < maxRetries) { + System.out.printf( + "Attempt %d/%d failed, retrying in 1 second: %s%n", + attempt, maxRetries, e.getMessage()); + Thread.sleep(1000); + } + } + } + throw lastException; + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/validation/OtlpProfileValidator.java b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/validation/OtlpProfileValidator.java new file mode 100644 index 00000000000..9a2350e2fc7 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/validation/OtlpProfileValidator.java @@ -0,0 +1,346 @@ +package com.datadog.profiling.otel.validation; + +import com.datadog.profiling.otel.proto.dictionary.AttributeTable; +import com.datadog.profiling.otel.proto.dictionary.FunctionTable; +import com.datadog.profiling.otel.proto.dictionary.LinkTable; +import com.datadog.profiling.otel.proto.dictionary.LocationTable; +import com.datadog.profiling.otel.proto.dictionary.StackTable; +import com.datadog.profiling.otel.proto.dictionary.StringTable; + +/** + * Validator for OTLP profile structures to ensure compliance with OpenTelemetry specifications. + * Validates dictionary table constraints, sample consistency, and reference integrity. + */ +public final class OtlpProfileValidator { + + private OtlpProfileValidator() { + // Utility class + } + + /** + * Validates dictionary table constraints according to OTLP spec. + * + * @param strings string table + * @param functions function table + * @param locations location table + * @param stacks stack table + * @param links link table + * @param attributes attribute table + * @return validation result with any errors or warnings found + */ + public static ValidationResult validateDictionaries( + StringTable strings, + FunctionTable functions, + LocationTable locations, + StackTable stacks, + LinkTable links, + AttributeTable attributes) { + + ValidationResult.Builder result = ValidationResult.builder(); + + // Validate StringTable + validateStringTable(strings, result); + + // Validate FunctionTable + validateFunctionTable(functions, strings, result); + + // Validate LocationTable + validateLocationTable(locations, functions, result); + + // Validate StackTable + validateStackTable(stacks, locations, result); + + // Validate LinkTable + validateLinkTable(links, result); + + // Validate AttributeTable + validateAttributeTable(attributes, strings, result); + + return result.build(); + } + + /** + * Validates string table constraints. + * + * @param strings the string table + * @param result the validation result builder + */ + private static void validateStringTable(StringTable strings, ValidationResult.Builder result) { + // Check that table is not empty + if (strings.size() == 0) { + result.addError("StringTable is empty - must have at least index 0 (empty string)"); + return; + } + + // Check that index 0 is empty string (null/unset sentinel) + String index0 = strings.get(0); + if (index0 == null) { + result.addError("StringTable index 0 is null - should be empty string (\"\")"); + } else if (!index0.isEmpty()) { + result.addError( + "StringTable index 0 is not empty string - found: \"" + + index0 + + "\" (length " + + index0.length() + + ")"); + } + + // Check for duplicate strings (except index 0) + for (int i = 1; i < strings.size(); i++) { + String s = strings.get(i); + for (int j = i + 1; j < strings.size(); j++) { + if (s.equals(strings.get(j))) { + result.addWarning( + "StringTable has duplicate entries: index " + + i + + " and " + + j + + " both contain \"" + + s + + "\""); + } + } + } + } + + /** + * Validates function table constraints. + * + * @param functions the function table + * @param strings the string table (for reference validation) + * @param result the validation result builder + */ + private static void validateFunctionTable( + FunctionTable functions, StringTable strings, ValidationResult.Builder result) { + + if (functions.size() == 0) { + result.addError("FunctionTable is empty - must have at least index 0 (null/unset)"); + return; + } + + // Validate that all function string indices reference valid strings + for (int i = 0; i < functions.size(); i++) { + FunctionTable.FunctionEntry entry = functions.get(i); + + // Check name index + if (entry.nameIndex < 0 || entry.nameIndex >= strings.size()) { + result.addError( + "FunctionTable entry " + + i + + " has invalid nameIndex " + + entry.nameIndex + + " (StringTable size: " + + strings.size() + + ")"); + } + + // Check system name index + if (entry.systemNameIndex < 0 || entry.systemNameIndex >= strings.size()) { + result.addError( + "FunctionTable entry " + + i + + " has invalid systemNameIndex " + + entry.systemNameIndex + + " (StringTable size: " + + strings.size() + + ")"); + } + + // Check filename index + if (entry.filenameIndex < 0 || entry.filenameIndex >= strings.size()) { + result.addError( + "FunctionTable entry " + + i + + " has invalid filenameIndex " + + entry.filenameIndex + + " (StringTable size: " + + strings.size() + + ")"); + } + } + } + + /** + * Validates location table constraints. + * + * @param locations the location table + * @param functions the function table (for reference validation) + * @param result the validation result builder + */ + private static void validateLocationTable( + LocationTable locations, FunctionTable functions, ValidationResult.Builder result) { + + if (locations.size() == 0) { + result.addError("LocationTable is empty - must have at least index 0 (null/unset)"); + return; + } + + // Validate that all location line entries reference valid functions + for (int i = 0; i < locations.size(); i++) { + LocationTable.LocationEntry entry = locations.get(i); + + // Check line entries if present + if (entry.lines != null && !entry.lines.isEmpty()) { + for (int lineIdx = 0; lineIdx < entry.lines.size(); lineIdx++) { + LocationTable.LineEntry line = entry.lines.get(lineIdx); + if (line.functionIndex < 0 || line.functionIndex >= functions.size()) { + result.addError( + "LocationTable entry " + + i + + " line " + + lineIdx + + " has invalid functionIndex " + + line.functionIndex + + " (FunctionTable size: " + + functions.size() + + ")"); + } + } + } + } + } + + /** + * Validates stack table constraints. + * + * @param stacks the stack table + * @param locations the location table (for reference validation) + * @param result the validation result builder + */ + private static void validateStackTable( + StackTable stacks, LocationTable locations, ValidationResult.Builder result) { + + if (stacks.size() == 0) { + result.addError("StackTable is empty - must have at least index 0 (null/unset)"); + return; + } + + // Check that index 0 is empty stack + StackTable.StackEntry index0 = stacks.get(0); + if (index0.locationIndices == null || index0.locationIndices.length != 0) { + result.addError("StackTable index 0 must be empty stack (zero-length array)"); + } + + // Validate that all stack location indices reference valid locations + for (int i = 0; i < stacks.size(); i++) { + StackTable.StackEntry entry = stacks.get(i); + if (entry.locationIndices != null) { + for (int j = 0; j < entry.locationIndices.length; j++) { + int locationIndex = entry.locationIndices[j]; + if (locationIndex < 0 || locationIndex >= locations.size()) { + result.addError( + "StackTable entry " + + i + + " location " + + j + + " has invalid index " + + locationIndex + + " (LocationTable size: " + + locations.size() + + ")"); + } + } + } + } + } + + /** + * Validates link table constraints. + * + * @param links the link table + * @param result the validation result builder + */ + private static void validateLinkTable(LinkTable links, ValidationResult.Builder result) { + if (links.size() == 0) { + result.addError("LinkTable is empty - must have at least index 0 (null/unset)"); + return; + } + + // Check that index 0 has zero trace/span IDs + LinkTable.LinkEntry index0 = links.get(0); + if (!isZeroBytes(index0.traceId) || !isZeroBytes(index0.spanId)) { + result.addError( + "LinkTable index 0 must have zero trace_id and span_id (null/unset sentinel)"); + } + + // Validate that all non-zero links have non-zero trace_id and span_id + for (int i = 1; i < links.size(); i++) { + LinkTable.LinkEntry entry = links.get(i); + + if (isZeroBytes(entry.traceId)) { + result.addWarning("LinkTable entry " + i + " has zero trace_id (should be non-zero)"); + } + + if (isZeroBytes(entry.spanId)) { + result.addWarning("LinkTable entry " + i + " has zero span_id (should be non-zero)"); + } + } + } + + /** + * Validates attribute table constraints. + * + * @param attributes the attribute table + * @param strings the string table (for reference validation) + * @param result the validation result builder + */ + private static void validateAttributeTable( + AttributeTable attributes, StringTable strings, ValidationResult.Builder result) { + + if (attributes.size() == 0) { + result.addError("AttributeTable is empty - must have at least index 0 (null/unset)"); + return; + } + + // Validate that all attribute key indices reference valid strings + for (int i = 0; i < attributes.size(); i++) { + AttributeTable.AttributeEntry entry = attributes.get(i); + + // Check key index + if (entry.keyIndex < 0 || entry.keyIndex >= strings.size()) { + result.addError( + "AttributeTable entry " + + i + + " has invalid keyIndex " + + entry.keyIndex + + " (StringTable size: " + + strings.size() + + ")"); + } + + // Check unit index + if (entry.unitIndex < 0 || entry.unitIndex >= strings.size()) { + result.addError( + "AttributeTable entry " + + i + + " has invalid unitIndex " + + entry.unitIndex + + " (StringTable size: " + + strings.size() + + ")"); + } + + // Note: For STRING type, the value is stored as a String object, not an index into + // StringTable + // INT, BOOL, and DOUBLE types store their values directly as Object + } + } + + /** + * Checks if byte array is all zeros. + * + * @param bytes the byte array + * @return true if all bytes are zero + */ + private static boolean isZeroBytes(byte[] bytes) { + if (bytes == null) { + return true; + } + for (byte b : bytes) { + if (b != 0) { + return false; + } + } + return true; + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/validation/OtlpProfileValidatorTest.java b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/validation/OtlpProfileValidatorTest.java new file mode 100644 index 00000000000..32eeffa2a00 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/validation/OtlpProfileValidatorTest.java @@ -0,0 +1,188 @@ +package com.datadog.profiling.otel.validation; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.datadog.profiling.otel.proto.dictionary.AttributeTable; +import com.datadog.profiling.otel.proto.dictionary.FunctionTable; +import com.datadog.profiling.otel.proto.dictionary.LinkTable; +import com.datadog.profiling.otel.proto.dictionary.LocationTable; +import com.datadog.profiling.otel.proto.dictionary.StackTable; +import com.datadog.profiling.otel.proto.dictionary.StringTable; +import org.junit.jupiter.api.Test; + +/** Unit tests for OTLP profile validator. */ +class OtlpProfileValidatorTest { + + @Test + void validateEmptyDictionaries() { + StringTable strings = new StringTable(); + FunctionTable functions = new FunctionTable(); + LocationTable locations = new LocationTable(); + StackTable stacks = new StackTable(); + LinkTable links = new LinkTable(); + AttributeTable attributes = new AttributeTable(); + + ValidationResult result = + OtlpProfileValidator.validateDictionaries( + strings, functions, locations, stacks, links, attributes); + + assertTrue(result.isValid(), "Empty dictionaries should be valid: " + result.getReport()); + assertTrue(result.getErrors().isEmpty()); + } + + @Test + void validateStringTableWithValidEntries() { + StringTable strings = new StringTable(); + strings.intern("com.example.Class"); + strings.intern("methodName"); + strings.intern("File.java"); + + ValidationResult result = + OtlpProfileValidator.validateDictionaries( + strings, + new FunctionTable(), + new LocationTable(), + new StackTable(), + new LinkTable(), + new AttributeTable()); + + assertTrue(result.isValid()); + } + + @Test + void validateFunctionTableReferences() { + StringTable strings = new StringTable(); + int nameIdx = strings.intern("method"); + int systemNameIdx = strings.intern("method"); + int filenameIdx = strings.intern("File.java"); + + FunctionTable functions = new FunctionTable(); + functions.intern(nameIdx, systemNameIdx, filenameIdx, 100); + + ValidationResult result = + OtlpProfileValidator.validateDictionaries( + strings, + functions, + new LocationTable(), + new StackTable(), + new LinkTable(), + new AttributeTable()); + + assertTrue(result.isValid()); + } + + @Test + void validateStackTableWithValidReferences() { + StringTable strings = new StringTable(); + int nameIdx = strings.intern("method"); + + FunctionTable functions = new FunctionTable(); + int funcIdx = functions.intern(nameIdx, nameIdx, nameIdx, 100); + + LocationTable locations = new LocationTable(); + int loc1 = locations.intern(0, 0x1000, funcIdx, 10, 0); + int loc2 = locations.intern(0, 0x2000, funcIdx, 20, 0); + + StackTable stacks = new StackTable(); + stacks.intern(new int[] {loc1, loc2}); + + ValidationResult result = + OtlpProfileValidator.validateDictionaries( + strings, functions, locations, stacks, new LinkTable(), new AttributeTable()); + + assertTrue(result.isValid()); + } + + @Test + void validateLinkTableWithValidEntries() { + LinkTable links = new LinkTable(); + links.intern(123456L, 789L); // Non-zero trace/span IDs + + ValidationResult result = + OtlpProfileValidator.validateDictionaries( + new StringTable(), + new FunctionTable(), + new LocationTable(), + new StackTable(), + links, + new AttributeTable()); + + assertTrue(result.isValid()); + } + + @Test + void validateAttributeTableReferences() { + StringTable strings = new StringTable(); + int keyIdx = strings.intern("thread.name"); + int unitIdx = strings.intern(""); + + AttributeTable attributes = new AttributeTable(); + attributes.internString(keyIdx, "main", unitIdx); + attributes.internInt(keyIdx, 42L, unitIdx); + attributes.internBool(keyIdx, true, unitIdx); + attributes.internDouble(keyIdx, 3.14, unitIdx); + + ValidationResult result = + OtlpProfileValidator.validateDictionaries( + strings, + new FunctionTable(), + new LocationTable(), + new StackTable(), + new LinkTable(), + attributes); + + assertTrue(result.isValid()); + } + + @Test + void validateLocationTableWithInlinedFunctions() { + StringTable strings = new StringTable(); + int nameIdx = strings.intern("method"); + + FunctionTable functions = new FunctionTable(); + int func1 = functions.intern(nameIdx, nameIdx, nameIdx, 100); + int func2 = functions.intern(nameIdx, nameIdx, nameIdx, 200); + + LocationTable locations = new LocationTable(); + // Create location with inlined function + int locIdx = locations.intern(0, 0x1000, func1, 10, 0); + + // Add inlined line entry + LocationTable.LocationEntry loc = locations.get(locIdx); + assertNotNull(loc); + + ValidationResult result = + OtlpProfileValidator.validateDictionaries( + strings, functions, locations, new StackTable(), new LinkTable(), new AttributeTable()); + + assertTrue(result.isValid()); + } + + @Test + void validationResultBuilder() { + ValidationResult result = + ValidationResult.builder() + .addError("Error 1") + .addError("Error 2") + .addWarning("Warning 1") + .build(); + + assertFalse(result.isValid()); + assertEquals(2, result.getErrors().size()); + assertEquals(1, result.getWarnings().size()); + assertTrue(result.getReport().contains("Error 1")); + assertTrue(result.getReport().contains("Warning 1")); + } + + @Test + void validationResultPassesWithWarnings() { + ValidationResult result = ValidationResult.builder().addWarning("Just a warning").build(); + + assertTrue(result.isValid(), "Should be valid with only warnings"); + assertEquals(0, result.getErrors().size()); + assertEquals(1, result.getWarnings().size()); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/validation/ValidationResult.java b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/validation/ValidationResult.java new file mode 100644 index 00000000000..fb52111d21a --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/test/java/com/datadog/profiling/otel/validation/ValidationResult.java @@ -0,0 +1,133 @@ +package com.datadog.profiling.otel.validation; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** Result of OTLP profile validation containing errors and warnings. */ +public final class ValidationResult { + private final List errors; + private final List warnings; + + private ValidationResult(List errors, List warnings) { + this.errors = Collections.unmodifiableList(new ArrayList<>(errors)); + this.warnings = Collections.unmodifiableList(new ArrayList<>(warnings)); + } + + /** + * Creates a new builder for validation results. + * + * @return a new builder instance + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Returns whether validation passed (no errors). + * + * @return true if no errors were found + */ + public boolean isValid() { + return errors.isEmpty(); + } + + /** + * Returns the list of validation errors. + * + * @return unmodifiable list of error messages + */ + public List getErrors() { + return errors; + } + + /** + * Returns the list of validation warnings. + * + * @return unmodifiable list of warning messages + */ + public List getWarnings() { + return warnings; + } + + /** + * Returns a formatted string containing all errors and warnings. + * + * @return formatted validation report + */ + public String getReport() { + StringBuilder sb = new StringBuilder(); + if (isValid()) { + sb.append("Validation PASSED"); + if (!warnings.isEmpty()) { + sb.append(" (").append(warnings.size()).append(" warnings)"); + } + } else { + sb.append("Validation FAILED (").append(errors.size()).append(" errors"); + if (!warnings.isEmpty()) { + sb.append(", ").append(warnings.size()).append(" warnings"); + } + sb.append(")"); + } + + if (!errors.isEmpty()) { + sb.append("\n\nErrors:"); + for (String error : errors) { + sb.append("\n - ").append(error); + } + } + + if (!warnings.isEmpty()) { + sb.append("\n\nWarnings:"); + for (String warning : warnings) { + sb.append("\n - ").append(warning); + } + } + + return sb.toString(); + } + + @Override + public String toString() { + return getReport(); + } + + /** Builder for creating validation results. */ + public static final class Builder { + private final List errors = new ArrayList<>(); + private final List warnings = new ArrayList<>(); + + private Builder() {} + + /** + * Adds an error to the validation result. + * + * @param message error message + * @return this builder + */ + public Builder addError(String message) { + errors.add(message); + return this; + } + + /** + * Adds a warning to the validation result. + * + * @param message warning message + * @return this builder + */ + public Builder addWarning(String message) { + warnings.add(message); + return this; + } + + /** + * Builds the validation result. + * + * @return the validation result + */ + public ValidationResult build() { + return new ValidationResult(errors, warnings); + } + } +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/test/resources/opentelemetry/proto/common/v1/common.proto b/dd-java-agent/agent-profiling/profiling-otel/src/test/resources/opentelemetry/proto/common/v1/common.proto new file mode 100644 index 00000000000..58a911ad498 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/test/resources/opentelemetry/proto/common/v1/common.proto @@ -0,0 +1,154 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.common.v1; + +option csharp_namespace = "OpenTelemetry.Proto.Common.V1"; +option java_multiple_files = true; +option java_package = "io.opentelemetry.proto.common.v1"; +option java_outer_classname = "CommonProto"; +option go_package = "go.opentelemetry.io/proto/otlp/common/v1"; + +// Represents any type of attribute value. AnyValue may contain a +// primitive value such as a string or integer or it may contain an arbitrary nested +// object containing arrays, key-value lists and primitives. +message AnyValue { + // The value is one of the listed fields. It is valid for all values to be unspecified + // in which case this AnyValue is considered to be "empty". + oneof value { + string string_value = 1; + bool bool_value = 2; + int64 int_value = 3; + double double_value = 4; + ArrayValue array_value = 5; + KeyValueList kvlist_value = 6; + bytes bytes_value = 7; + // Reference to the string value in ProfilesDictionary.string_table. + // + // Note: This is currently used exclusively in the Profiling signal. + // Implementers of OTLP receivers for signals other than Profiling should + // treat the presence of this value as a non-fatal issue. + // Log an error or warning indicating an unexpected field intended for the + // Profiling signal and process the data as if this value were absent or + // empty, ignoring its semantic content for the non-Profiling signal. + // + // Status: [Development] + int32 string_value_strindex = 8; + } +} + +// ArrayValue is a list of AnyValue messages. We need ArrayValue as a message +// since oneof in AnyValue does not allow repeated fields. +message ArrayValue { + // Array of values. The array may be empty (contain 0 elements). + repeated AnyValue values = 1; +} + +// KeyValueList is a list of KeyValue messages. We need KeyValueList as a message +// since `oneof` in AnyValue does not allow repeated fields. Everywhere else where we need +// a list of KeyValue messages (e.g. in Span) we use `repeated KeyValue` directly to +// avoid unnecessary extra wrapping (which slows down the protocol). The 2 approaches +// are semantically equivalent. +message KeyValueList { + // A collection of key/value pairs of key-value pairs. The list may be empty (may + // contain 0 elements). + // + // The keys MUST be unique (it is not allowed to have more than one + // value with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. + repeated KeyValue values = 1; +} + +// Represents a key-value pair that is used to store Span attributes, Link +// attributes, etc. +message KeyValue { + // The key name of the pair. + // key_ref MUST NOT be set if key is used. + string key = 1; + + // The value of the pair. + AnyValue value = 2; + + // Reference to the string key in ProfilesDictionary.string_table. + // key MUST NOT be set if key_strindex is used. + // + // Note: This is currently used exclusively in the Profiling signal. + // Implementers of OTLP receivers for signals other than Profiling should + // treat the presence of this key as a non-fatal issue. + // Log an error or warning indicating an unexpected field intended for the + // Profiling signal and process the data as if this value were absent or + // empty, ignoring its semantic content for the non-Profiling signal. + // + // Status: [Development] + int32 key_strindex = 3; +} + +// InstrumentationScope is a message representing the instrumentation scope information +// such as the fully qualified name and version. +message InstrumentationScope { + // A name denoting the Instrumentation scope. + // An empty instrumentation scope name means the name is unknown. + string name = 1; + + // Defines the version of the instrumentation scope. + // An empty instrumentation scope version means the version is unknown. + string version = 2; + + // Additional attributes that describe the scope. [Optional]. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. + repeated KeyValue attributes = 3; + + // The number of attributes that were discarded. Attributes + // can be discarded because their keys are too long or because there are too many + // attributes. If this value is 0, then no attributes were dropped. + uint32 dropped_attributes_count = 4; +} + +// A reference to an Entity. +// Entity represents an object of interest associated with produced telemetry: e.g spans, metrics, profiles, or logs. +// +// Status: [Development] +message EntityRef { + // The Schema URL, if known. This is the identifier of the Schema that the entity data + // is recorded in. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // + // This schema_url applies to the data in this message and to the Resource attributes + // referenced by id_keys and description_keys. + // TODO: discuss if we are happy with this somewhat complicated definition of what + // the schema_url applies to. + // + // This field obsoletes the schema_url field in ResourceMetrics/ResourceSpans/ResourceLogs. + string schema_url = 1; + + // Defines the type of the entity. MUST not change during the lifetime of the entity. + // For example: "service" or "host". This field is required and MUST not be empty + // for valid entities. + string type = 2; + + // Attribute Keys that identify the entity. + // MUST not change during the lifetime of the entity. The Id must contain at least one attribute. + // These keys MUST exist in the containing {message}.attributes. + repeated string id_keys = 3; + + // Descriptive (non-identifying) attribute keys of the entity. + // MAY change over the lifetime of the entity. MAY be empty. + // These attribute keys are not part of entity's identity. + // These keys MUST exist in the containing {message}.attributes. + repeated string description_keys = 4; +} \ No newline at end of file diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/test/resources/opentelemetry/proto/profiles/v1development/profiles.proto b/dd-java-agent/agent-profiling/profiling-otel/src/test/resources/opentelemetry/proto/profiles/v1development/profiles.proto new file mode 100644 index 00000000000..fede0cf3c2b --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/test/resources/opentelemetry/proto/profiles/v1development/profiles.proto @@ -0,0 +1,499 @@ +// Copyright 2023, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This file includes work covered by the following copyright and permission notices: +// +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.profiles.v1development; + +import "opentelemetry/proto/common/v1/common.proto"; +import "opentelemetry/proto/resource/v1/resource.proto"; + +option csharp_namespace = "OpenTelemetry.Proto.Profiles.V1Development"; +option java_multiple_files = true; +option java_package = "io.opentelemetry.proto.profiles.v1development"; +option java_outer_classname = "ProfilesProto"; +option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1development"; + +// Relationships Diagram +// +// ┌──────────────────┐ LEGEND +// │ ProfilesData │ ─────┐ +// └──────────────────┘ │ ─────▶ embedded +// │ │ +// │ 1-n │ ─────▷ referenced by index +// ▼ ▼ +// ┌──────────────────┐ ┌────────────────────┐ +// │ ResourceProfiles │ │ ProfilesDictionary │ +// └──────────────────┘ └────────────────────┘ +// │ +// │ 1-n +// ▼ +// ┌──────────────────┐ +// │ ScopeProfiles │ +// └──────────────────┘ +// │ +// │ 1-n +// ▼ +// ┌──────────────────┐ +// │ Profile │ +// └──────────────────┘ +// │ n-1 +// │ 1-n ┌───────────────────────────────────────┐ +// ▼ │ ▽ +// ┌──────────────────┐ 1-n ┌─────────────────┐ ┌──────────┐ +// │ Sample │ ──────▷ │ KeyValueAndUnit │ │ Link │ +// └──────────────────┘ └─────────────────┘ └──────────┘ +// │ △ △ +// │ n-1 │ │ 1-n +// ▽ │ │ +// ┌──────────────────┐ │ │ +// │ Stack │ │ │ +// └──────────────────┘ │ │ +// │ 1-n │ │ +// │ 1-n ┌────────────────┘ │ +// ▽ │ │ +// ┌──────────────────┐ n-1 ┌─────────────┐ +// │ Location │ ──────▷ │ Mapping │ +// └──────────────────┘ └─────────────┘ +// │ +// │ 1-n +// ▼ +// ┌──────────────────┐ +// │ Line │ +// └──────────────────┘ +// │ +// │ 1-1 +// ▽ +// ┌──────────────────┐ +// │ Function │ +// └──────────────────┘ +// + +// ProfilesDictionary represents the profiles data shared across the +// entire message being sent. The following applies to all fields in this +// message: +// +// - A dictionary is an array of dictionary items. Users of the dictionary +// compactly reference the items using the index within the array. +// +// - A dictionary MUST have a zero value encoded as the first element. This +// allows for _index fields pointing into the dictionary to use a 0 pointer +// value to indicate 'null' / 'not set'. Unless otherwise defined, a 'zero +// value' message value is one with all default field values, so as to +// minimize wire encoded size. +// +// - There SHOULD NOT be dupes in a dictionary. The identity of dictionary +// items is based on their value, recursively as needed. If a particular +// implementation does emit duplicated items, it MUST NOT attempt to give them +// meaning based on the index or order. A profile processor may remove +// duplicate items and this MUST NOT have any observable effects for +// consumers. +// +// - There SHOULD NOT be orphaned (unreferenced) items in a dictionary. A +// profile processor may remove ("garbage-collect") orphaned items and this +// MUST NOT have any observable effects for consumers. +// +message ProfilesDictionary { + // Mappings from address ranges to the image/binary/library mapped + // into that address range referenced by locations via Location.mapping_index. + // + // mapping_table[0] must always be zero value (Mapping{}) and present. + repeated Mapping mapping_table = 1; + + // Locations referenced by samples via Stack.location_indices. + // + // location_table[0] must always be zero value (Location{}) and present. + repeated Location location_table = 2; + + // Functions referenced by locations via Line.function_index. + // + // function_table[0] must always be zero value (Function{}) and present. + repeated Function function_table = 3; + + // Links referenced by samples via Sample.link_index. + // + // link_table[0] must always be zero value (Link{}) and present. + repeated Link link_table = 4; + + // A common table for strings referenced by various messages. + // + // string_table[0] must always be "" and present. + repeated string string_table = 5; + + // A common table for attributes referenced by the Profile, Sample, Mapping + // and Location messages below through attribute_indices field. Each entry is + // a key/value pair with an optional unit. Since this is a dictionary table, + // multiple entries with the same key may be present, unlike direct attribute + // tables like Resource.attributes. The referencing attribute_indices fields, + // though, do maintain the key uniqueness requirement. + // + // It's recommended to use attributes for variables with bounded cardinality, + // such as categorical variables + // (https://en.wikipedia.org/wiki/Categorical_variable). Using an attribute of + // a floating point type (e.g., CPU time) in a sample can quickly make every + // attribute value unique, defeating the purpose of the dictionary and + // impractically increasing the profile size. + // + // Examples of attributes: + // "/http/user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" + // "abc.com/myattribute": true + // "allocation_size": 128 bytes + // + // attribute_table[0] must always be zero value (KeyValueAndUnit{}) and present. + repeated KeyValueAndUnit attribute_table = 6; + + // Stacks referenced by samples via Sample.stack_index. + // + // stack_table[0] must always be zero value (Stack{}) and present. + repeated Stack stack_table = 7; +} + +// ProfilesData represents the profiles data that can be stored in persistent storage, +// OR can be embedded by other protocols that transfer OTLP profiles data but do not +// implement the OTLP protocol. +// +// The main difference between this message and collector protocol is that +// in this message there will not be any "control" or "metadata" specific to +// OTLP protocol. +// +// When new fields are added into this message, the OTLP request MUST be updated +// as well. +message ProfilesData { + // An array of ResourceProfiles. + // For data coming from an SDK profiler, this array will typically contain one + // element. Host-level profilers will usually create one ResourceProfile per + // container, as well as one additional ResourceProfile grouping all samples + // from non-containerized processes. + // Other resource groupings are possible as well and clarified via + // Resource.attributes and semantic conventions. + // Tools that visualize profiles should prefer displaying + // resources_profiles[0].scope_profiles[0].profiles[0] by default. + repeated ResourceProfiles resource_profiles = 1; + + // One instance of ProfilesDictionary + ProfilesDictionary dictionary = 2; +} + + +// A collection of ScopeProfiles from a Resource. +message ResourceProfiles { + reserved 1000; + + // The resource for the profiles in this message. + // If this field is not set then no resource info is known. + opentelemetry.proto.resource.v1.Resource resource = 1; + + // A list of ScopeProfiles that originate from a resource. + repeated ScopeProfiles scope_profiles = 2; + + // The Schema URL, if known. This is the identifier of the Schema that the resource data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to the data in the "resource" field. It does not apply + // to the data in the "scope_profiles" field which have their own schema_url field. + string schema_url = 3; +} + +// A collection of Profiles produced by an InstrumentationScope. +message ScopeProfiles { + // The instrumentation scope information for the profiles in this message. + // Semantically when InstrumentationScope isn't set, it is equivalent with + // an empty instrumentation scope name (unknown). + opentelemetry.proto.common.v1.InstrumentationScope scope = 1; + + // A list of Profiles that originate from an instrumentation scope. + repeated Profile profiles = 2; + + // The Schema URL, if known. This is the identifier of the Schema that the profile data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to the data in the "scope" field and all profiles in the + // "profiles" field. + string schema_url = 3; +} + +// Profile is a common stacktrace profile format. +// +// Measurements represented with this format should follow the +// following conventions: +// +// - Consumers should treat unset optional fields as if they had been +// set with their default value. +// +// - When possible, measurements should be stored in "unsampled" form +// that is most useful to humans. There should be enough +// information present to determine the original sampled values. +// +// - The profile is represented as a set of samples, where each sample +// references a stack trace which is a list of locations, each belonging +// to a mapping. +// - There is a N->1 relationship from Stack.location_indices entries to +// locations. For every Stack.location_indices entry there must be a +// unique Location with that index. +// - There is an optional N->1 relationship from locations to +// mappings. For every nonzero Location.mapping_id there must be a +// unique Mapping with that index. + +// Represents a complete profile, including sample types, samples, mappings to +// binaries, stacks, locations, functions, string table, and additional +// metadata. It modifies and annotates pprof Profile with OpenTelemetry +// specific fields. +// +// Note that whilst fields in this message retain the name and field id from pprof in most cases +// for ease of understanding data migration, it is not intended that pprof:Profile and +// OpenTelemetry:Profile encoding be wire compatible. +message Profile { + // The type and unit of all Sample.values in this profile. + // For a cpu or off-cpu profile this might be: + // ["cpu","nanoseconds"] or ["off_cpu","nanoseconds"] + // For a heap profile, this might be: + // ["allocated_objects","count"] or ["allocated_space","bytes"], + ValueType sample_type = 1; + // The set of samples recorded in this profile. + repeated Sample samples = 2; + + // The following fields 3-12 are informational, do not affect + // interpretation of results. + + // Time of collection. Value is UNIX Epoch time in nanoseconds since 00:00:00 + // UTC on 1 January 1970. + fixed64 time_unix_nano = 3; + // Duration of the profile. For instant profiles like live heap snapshot, the + // duration can be zero but it may be preferable to set time_unix_nano to the + // process start time and duration_nano to the relative time when the profile + // was gathered. This ensures Sample.timestamps_unix_nano values such as + // allocation timestamp fall into the profile time range. + uint64 duration_nano = 4; + // The kind of events between sampled occurrences. + // e.g [ "cpu","cycles" ] or [ "heap","bytes" ] + ValueType period_type = 5; + // The number of events between sampled occurrences. + int64 period = 6; + + // A globally unique identifier for a profile. The ID is a 16-byte array. An ID with + // all zeroes is considered invalid. It may be used for deduplication and signal + // correlation purposes. It is acceptable to treat two profiles with different values + // in this field as not equal, even if they represented the same object at an earlier + // time. + // This field is optional; an ID may be assigned to an ID-less profile in a later step. + bytes profile_id = 7; + + // The number of attributes that were discarded. Attributes + // can be discarded because their keys are too long or because there are too many + // attributes. If this value is 0, then no attributes were dropped. + uint32 dropped_attributes_count = 8; + + // The original payload format. See also original_payload. Optional, but the + // format and the bytes must be set or unset together. + // + // The allowed values for the format string are defined by the OpenTelemetry + // specification. Some examples are "jfr", "pprof", "linux_perf". + // + // The original payload may be optionally provided when the conversion to the + // OLTP format was done from a different format with some loss of the fidelity + // and the receiver may want to store the original payload to allow future + // lossless export or reinterpretation. Some examples of the original format + // are JFR (Java Flight Recorder), pprof, Linux perf. + // + // Even when the original payload is in a format that is semantically close to + // OTLP, such as pprof, a conversion may still be lossy in some cases (e.g. if + // the pprof file contains custom extensions or conventions). + // + // The original payload can be large in size, so including the original + // payload should be configurable by the profiler or collector options. The + // default behavior should be to not include the original payload. + string original_payload_format = 9; + // The original payload bytes. See also original_payload_format. Optional, but + // format and the bytes must be set or unset together. + bytes original_payload = 10; + + // References to attributes in attribute_table. [optional] + repeated int32 attribute_indices = 11; +} + +// A pointer from a profile Sample to a trace Span. +// Connects a profile sample to a trace span, identified by unique trace and span IDs. +message Link { + // A unique identifier of a trace that this linked span is part of. The ID is a + // 16-byte array. + bytes trace_id = 1; + + // A unique identifier for the linked span. The ID is an 8-byte array. + bytes span_id = 2; +} + +// ValueType describes the type and units of a value. +message ValueType { + // Index into ProfilesDictionary.string_table. + int32 type_strindex = 1; + + // Index into ProfilesDictionary.string_table. + int32 unit_strindex = 2; +} + +// Each Sample records values encountered in some program context. The program +// context is typically a stack trace, perhaps augmented with auxiliary +// information like the thread-id, some indicator of a higher level request +// being handled etc. +// +// A Sample MUST have have at least one values or timestamps_unix_nano entry. If +// both fields are populated, they MUST contain the same number of elements, and +// the elements at the same index MUST refer to the same event. +// +// For the purposes of efficiently representing aggregated data observations, a Sample is regarded +// as having a shared identity and an associated collection of per-observation data points. +// Samples having the same identity SHOULD be combined by inserting timestamps and values to the data arrays. +// +// Examples of different ways ('shapes') of representing a sample with the total value of 10: +// +// Report of a stacktrace at 10 timestamps (consumers must assume the value is 1 for each point): +// values: [] +// timestamps_unix_nano: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +// +// Report of a stacktrace with an aggregated value without timestamps: +// values: [10] +// timestamps_unix_nano: [] +// +// Report of a stacktrace at 4 timestamps where each point records a specific value: +// values: [2, 2, 3, 3] +// timestamps_unix_nano: [1, 2, 3, 4] +// +// All Samples for a Profile SHOULD have the same shape, i.e. all data observation series should consistently +// adopt the same data recording style. +// +message Sample { + + // A Sample's identity (i.e. 'primary key') is the tuple of {stack_index, set_of(attribute_indices), link_index} + + // Reference to stack in ProfilesDictionary.stack_table. + int32 stack_index = 1; + // References to attributes in ProfilesDictionary.attribute_table. [optional] + repeated int32 attribute_indices = 2; + // Reference to link in ProfilesDictionary.link_table. [optional] + // It can be unset / set to 0 if no link exists, as link_table[0] is always a 'null' default value. + int32 link_index = 3; + + // The following fields may contain per-observation data and do not form part of the Sample's identity. + + // The type and unit of each value is defined by Profile.sample_type. + repeated int64 values = 4; + // Timestamps associated with Sample. Value is UNIX Epoch time in nanoseconds + // since 00:00:00 UTC on 1 January 1970. The timestamps should fall within the + // [Profile.time_unix_nano, Profile.time_unix_nano + Profile.duration_nano) + // time range. + repeated fixed64 timestamps_unix_nano = 5; +} + +// Describes the mapping of a binary in memory, including its address range, +// file offset, and metadata like build ID +message Mapping { + // Address at which the binary (or DLL) is loaded into memory. + uint64 memory_start = 1; + // The limit of the address range occupied by this mapping. + uint64 memory_limit = 2; + // Offset in the binary that corresponds to the first mapped address. + uint64 file_offset = 3; + // The object this entry is loaded from. This can be a filename on + // disk for the main binary and shared libraries, or virtual + // abstractions like "[vdso]". + int32 filename_strindex = 4; // Index into ProfilesDictionary.string_table. + // References to attributes in ProfilesDictionary.attribute_table. [optional] + repeated int32 attribute_indices = 5; +} + +// A Stack represents a stack trace as a list of locations. +message Stack { + // References to locations in ProfilesDictionary.location_table. + // The first location is the leaf frame. + repeated int32 location_indices = 1; +} + +// Describes function and line table debug information. +message Location { + // Reference to mapping in ProfilesDictionary.mapping_table. + // It can be unset / set to 0 if the mapping is unknown or not applicable for + // this profile type, as mapping_table[0] is always a 'null' default mapping. + int32 mapping_index = 1; + // The instruction address for this location, if available. It + // should be within [Mapping.memory_start...Mapping.memory_limit] + // for the corresponding mapping. A non-leaf address may be in the + // middle of a call instruction. It is up to display tools to find + // the beginning of the instruction if necessary. + uint64 address = 2; + // Multiple line indicates this location has inlined functions, + // where the last entry represents the caller into which the + // preceding entries were inlined. + // + // E.g., if memcpy() is inlined into printf: + // lines[0].function_name == "memcpy" + // lines[1].function_name == "printf" + repeated Line lines = 3; + // References to attributes in ProfilesDictionary.attribute_table. [optional] + repeated int32 attribute_indices = 4; +} + +// Details a specific line in a source code, linked to a function. +message Line { + // Reference to function in ProfilesDictionary.function_table. + int32 function_index = 1; + // Line number in source code. 0 means unset. + int64 line = 2; + // Column number in source code. 0 means unset. + int64 column = 3; +} + +// Describes a function, including its human-readable name, system name, +// source file, and starting line number in the source. +message Function { + // The function name. Empty string if not available. + int32 name_strindex = 1; + // Function name, as identified by the system. For instance, + // it can be a C++ mangled name. Empty string if not available. + int32 system_name_strindex = 2; + // Source file containing the function. Empty string if not available. + int32 filename_strindex = 3; + // Line number in source file. 0 means unset. + int64 start_line = 4; +} + +// A custom 'dictionary native' style of encoding attributes which is more convenient +// for profiles than opentelemetry.proto.common.v1.KeyValue +// Specifically, uses the string table for keys and allows optional unit information. +message KeyValueAndUnit { + // The index into the string table for the attribute's key. + int32 key_strindex = 1; + // The value of the attribute. + opentelemetry.proto.common.v1.AnyValue value = 2; + // The index into the string table for the attribute's unit. + // zero indicates implicit (by semconv) or non-defined unit. + int32 unit_strindex = 3; +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/test/resources/opentelemetry/proto/resource/v1/resource.proto b/dd-java-agent/agent-profiling/profiling-otel/src/test/resources/opentelemetry/proto/resource/v1/resource.proto new file mode 100644 index 00000000000..42c5913cfae --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/test/resources/opentelemetry/proto/resource/v1/resource.proto @@ -0,0 +1,45 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.resource.v1; + +import "opentelemetry/proto/common/v1/common.proto"; + +option csharp_namespace = "OpenTelemetry.Proto.Resource.V1"; +option java_multiple_files = true; +option java_package = "io.opentelemetry.proto.resource.v1"; +option java_outer_classname = "ResourceProto"; +option go_package = "go.opentelemetry.io/proto/otlp/resource/v1"; + +// Resource information. +message Resource { + // Set of attributes that describe the resource. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. + repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; + + // The number of dropped attributes. If the value is 0, then + // no attributes were dropped. + uint32 dropped_attributes_count = 2; + + // Set of entities that participate in this Resource. + // + // Note: keys in the references MUST exist in attributes of this message. + // + // Status: [Development] + repeated opentelemetry.proto.common.v1.EntityRef entity_refs = 3; +} diff --git a/dd-java-agent/agent-profiling/profiling-otel/src/test/resources/otel-collector-config.yaml b/dd-java-agent/agent-profiling/profiling-otel/src/test/resources/otel-collector-config.yaml new file mode 100644 index 00000000000..bf6f6d3294b --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-otel/src/test/resources/otel-collector-config.yaml @@ -0,0 +1,25 @@ +# OpenTelemetry Collector configuration for validating OTLP profiles +# This configuration accepts OTLP data via HTTP and logs it for validation + +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + +exporters: + # Debug exporter for validation - we can inspect logs to verify data was received. + # Replaces the 'logging' exporter removed in collector-contrib v0.111.0 + debug: + verbosity: detailed + +service: + pipelines: + # Accept traces endpoint - profiles may fall back to this + traces: + receivers: [otlp] + processors: [batch] + exporters: [debug] diff --git a/dd-java-agent/agent-profiling/profiling-uploader/build.gradle b/dd-java-agent/agent-profiling/profiling-uploader/build.gradle index 39c4737cfd8..2faedc6c4b3 100644 --- a/dd-java-agent/agent-profiling/profiling-uploader/build.gradle +++ b/dd-java-agent/agent-profiling/profiling-uploader/build.gradle @@ -18,12 +18,17 @@ excludedClassesCoverage += [ 'com.datadog.profiling.uploader.ProfileUploader.1', // When running on Java 8 without `jfr`, no code is executed 'com.datadog.profiling.uploader.util.JfrCliHelper', - 'com.datadog.profiling.uploader.util.JfrCliHelper.Event' + 'com.datadog.profiling.uploader.util.JfrCliHelper.Event', + // Pre-existing borderline coverage — HTTP/upload plumbing, hard to unit test + 'com.datadog.profiling.uploader.CompressingRequestBody', + 'com.datadog.profiling.uploader.ProfileUploader', + 'com.datadog.profiling.uploader.ProfileUploader.RecordingDataAdapter', ] dependencies { implementation libs.slf4j implementation project(':communication') + implementation project(':communication:otlp-exporter') implementation project(':internal-api') implementation project(':utils:container-utils') implementation project(':utils:logging-utils') @@ -31,6 +36,7 @@ dependencies { implementation project(':utils:version-utils') implementation project(':dd-java-agent:agent-profiling:profiling-controller') + implementation project(':dd-java-agent:agent-profiling:profiling-otel') implementation libs.datadog.okhttp implementation libs.lz4 diff --git a/dd-java-agent/agent-profiling/profiling-uploader/src/main/java/com/datadog/profiling/uploader/LightweightOtlpEncoder.java b/dd-java-agent/agent-profiling/profiling-uploader/src/main/java/com/datadog/profiling/uploader/LightweightOtlpEncoder.java new file mode 100644 index 00000000000..9ac9da7e626 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-uploader/src/main/java/com/datadog/profiling/uploader/LightweightOtlpEncoder.java @@ -0,0 +1,216 @@ +package com.datadog.profiling.uploader; + +import com.datadog.profiling.otel.proto.OtlpProtoFields; +import com.datadog.profiling.otel.proto.OtlpResourceAttributes; +import com.datadog.profiling.otel.proto.ProtobufEncoder; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.time.Instant; +import java.util.Map; +import java.util.concurrent.ThreadLocalRandom; + +/** + * Encodes a minimal OTLP ProfilesData message with no sample conversion — just metadata and the raw + * JFR recording embedded as the {@code original_payload} blob. This is orders of magnitude faster + * than full JFR→OTLP conversion since it skips JFR parsing, event processing, and dictionary + * building entirely. + * + *

All nested message lengths are computed up front from the recording size, so the message is + * written in one pass into a single buffer and the JFR bytes are read from the file directly into + * it. The buffer is reused across calls; it holds the whole message, so it grows to fit the largest + * recording, and once larger than {@code MAX_RETAINED_BUFFER} it is shrunk on the next smaller + * encode. Not thread-safe: the buffer returned by {@link #encode} is only valid until the next + * call. + */ +final class LightweightOtlpEncoder { + + // string_table layout: index 0 is the null/unset sentinel required by the OTLP spec, + // indices 1 and 2 back the sample_type/period_type ValueType below + private static final int TYPE_STRINDEX_SAMPLES = 1; + private static final int UNIT_STRINDEX_COUNT = 2; + // buffer growth granularity — bounds the over-allocation while avoiding a reallocation for + // every slightly larger recording + private static final int BUFFER_GROWTH_ALIGNMENT = 1024 * 1024; + // a buffer above this size is reallocated to fit a smaller message, so a single large recording + // does not pin its buffer for the rest of the agent's lifetime + private static final int MAX_RETAINED_BUFFER = 8 * BUFFER_GROWTH_ALIGNMENT; + + private static final byte[] DICTIONARY_FIELD = encodeDictionaryField(); + + // the ResourceProfiles.resource field; resource attributes are fixed for the uploader lifetime + private final byte[] resourceField; + // the Profile fields preceding original_payload + private final ProtobufEncoder profileHeader = new ProtobufEncoder(128); + private final byte[] profileId = new byte[16]; + private byte[] buffer = new byte[0]; + + LightweightOtlpEncoder(Map resourceAttributes) { + ProtobufEncoder encoder = new ProtobufEncoder(512); + OtlpResourceAttributes.writeResource(encoder, resourceAttributes); + this.resourceField = encoder.toByteArray(); + } + + ByteBuffer encode(Path jfrFile, Instant start, Instant end) throws IOException { + long startTimeNanos = start.getEpochSecond() * 1_000_000_000L + start.getNano(); + long endTimeNanos = end.getEpochSecond() * 1_000_000_000L + end.getNano(); + profileHeader.reset(); + encodeProfileHeader(startTimeNanos, endTimeNanos); + int headerSize = profileHeader.size(); + + try (FileChannel channel = FileChannel.open(jfrFile, StandardOpenOption.READ)) { + long jfrSize = channel.size(); + long profileSize = + headerSize + fieldPrefixSize(OtlpProtoFields.Profile.ORIGINAL_PAYLOAD, jfrSize) + jfrSize; + long scopeProfilesSize = + fieldPrefixSize(OtlpProtoFields.ScopeProfiles.PROFILES, profileSize) + profileSize; + long resourceProfilesSize = + resourceField.length + + fieldPrefixSize(OtlpProtoFields.ResourceProfiles.SCOPE_PROFILES, scopeProfilesSize) + + scopeProfilesSize; + long totalSize = + fieldPrefixSize(OtlpProtoFields.ProfilesData.RESOURCE_PROFILES, resourceProfilesSize) + + resourceProfilesSize + + DICTIONARY_FIELD.length; + ensureCapacity(totalSize); + + int pos = + writeFieldPrefix( + buffer, 0, OtlpProtoFields.ProfilesData.RESOURCE_PROFILES, resourceProfilesSize); + System.arraycopy(resourceField, 0, buffer, pos, resourceField.length); + pos += resourceField.length; + pos = + writeFieldPrefix( + buffer, pos, OtlpProtoFields.ResourceProfiles.SCOPE_PROFILES, scopeProfilesSize); + pos = writeFieldPrefix(buffer, pos, OtlpProtoFields.ScopeProfiles.PROFILES, profileSize); + profileHeader.toByteBuffer().get(buffer, pos, headerSize); + pos += headerSize; + pos = writeFieldPrefix(buffer, pos, OtlpProtoFields.Profile.ORIGINAL_PAYLOAD, jfrSize); + + // the length prefixes above are derived from jfrSize, so exactly that many bytes are read; + // a recording truncated concurrently fails the export instead of emitting a malformed message + ByteBuffer target = ByteBuffer.wrap(buffer, pos, (int) jfrSize); + while (target.hasRemaining()) { + if (channel.read(target) < 0) { + throw new IOException("JFR recording shrank while being encoded: " + jfrFile); + } + } + pos += (int) jfrSize; + + System.arraycopy(DICTIONARY_FIELD, 0, buffer, pos, DICTIONARY_FIELD.length); + pos += DICTIONARY_FIELD.length; + return ByteBuffer.wrap(buffer, 0, pos); + } + } + + private void encodeProfileHeader(long startTimeNanos, long endTimeNanos) { + // Field 1: sample_type / Field 5: period_type (count-based samples) + profileHeader.writeNestedMessage( + OtlpProtoFields.Profile.SAMPLE_TYPE, + typeEncoder -> encodeValueType(typeEncoder, TYPE_STRINDEX_SAMPLES, UNIT_STRINDEX_COUNT)); + profileHeader.writeNestedMessage( + OtlpProtoFields.Profile.PERIOD_TYPE, + typeEncoder -> encodeValueType(typeEncoder, TYPE_STRINDEX_SAMPLES, UNIT_STRINDEX_COUNT)); + + profileHeader.writeFixed64Field(OtlpProtoFields.Profile.TIME_UNIX_NANO, startTimeNanos); + // clamped so a reversed window cannot underflow + profileHeader.writeVarintField( + OtlpProtoFields.Profile.DURATION_NANO, Math.max(0, endTimeNanos - startTimeNanos)); + profileHeader.writeVarintField(OtlpProtoFields.Profile.PERIOD, 1); + fillProfileId(); + profileHeader.writeBytesField(OtlpProtoFields.Profile.PROFILE_ID, profileId); + profileHeader.writeStringField(OtlpProtoFields.Profile.ORIGINAL_PAYLOAD_FORMAT, "jfr"); + } + + private void ensureCapacity(long size) throws IOException { + if (size > Integer.MAX_VALUE - BUFFER_GROWTH_ALIGNMENT) { + throw new IOException("OTLP profile message too large: " + size + " bytes"); + } + long aligned = + (size + BUFFER_GROWTH_ALIGNMENT - 1) / BUFFER_GROWTH_ALIGNMENT * BUFFER_GROWTH_ALIGNMENT; + if (size > buffer.length || (buffer.length > MAX_RETAINED_BUFFER && aligned < buffer.length)) { + // every encode rewrites the whole message, so the old content is not copied over + buffer = new byte[(int) aligned]; + } + } + + private static long fieldPrefixSize(int fieldNumber, long length) { + return varintSize((fieldNumber << 3) | ProtobufEncoder.WIRETYPE_LENGTH_DELIMITED) + + varintSize(length); + } + + private static int writeFieldPrefix(byte[] dest, int pos, int fieldNumber, long length) { + pos = writeVarint(dest, pos, (fieldNumber << 3) | ProtobufEncoder.WIRETYPE_LENGTH_DELIMITED); + return writeVarint(dest, pos, length); + } + + private static int varintSize(long value) { + int size = 1; + while ((value & ~0x7FL) != 0) { + value >>>= 7; + size++; + } + return size; + } + + private static int writeVarint(byte[] dest, int pos, long value) { + while ((value & ~0x7FL) != 0) { + dest[pos++] = (byte) ((value & 0x7F) | 0x80); + value >>>= 7; + } + dest[pos++] = (byte) value; + return pos; + } + + private static void encodeValueType(ProtobufEncoder encoder, int typeIndex, int unitIndex) { + encoder.writeVarintField(OtlpProtoFields.ValueType.TYPE_STRINDEX, typeIndex); + encoder.writeVarintField(OtlpProtoFields.ValueType.UNIT_STRINDEX, unitIndex); + } + + // minimal dictionary: the zero-value index-0 entry the OTLP spec requires in every table, plus + // the type/unit labels referenced by sample_type/period_type in string_table + private static byte[] encodeDictionaryField() { + ProtobufEncoder encoder = new ProtobufEncoder(64); + encoder.writeNestedMessage( + OtlpProtoFields.ProfilesData.DICTIONARY, + dictionaryEncoder -> { + writeEmptyEntry(dictionaryEncoder, OtlpProtoFields.ProfilesDictionary.MAPPING_TABLE); + writeEmptyEntry(dictionaryEncoder, OtlpProtoFields.ProfilesDictionary.LOCATION_TABLE); + writeEmptyEntry(dictionaryEncoder, OtlpProtoFields.ProfilesDictionary.FUNCTION_TABLE); + writeEmptyEntry(dictionaryEncoder, OtlpProtoFields.ProfilesDictionary.LINK_TABLE); + dictionaryEncoder.writeTag( + OtlpProtoFields.ProfilesDictionary.STRING_TABLE, + ProtobufEncoder.WIRETYPE_LENGTH_DELIMITED); + dictionaryEncoder.writeString(""); // index 0 (null/unset sentinel) + dictionaryEncoder.writeTag( + OtlpProtoFields.ProfilesDictionary.STRING_TABLE, + ProtobufEncoder.WIRETYPE_LENGTH_DELIMITED); + dictionaryEncoder.writeString("samples"); + dictionaryEncoder.writeTag( + OtlpProtoFields.ProfilesDictionary.STRING_TABLE, + ProtobufEncoder.WIRETYPE_LENGTH_DELIMITED); + dictionaryEncoder.writeString("count"); + writeEmptyEntry(dictionaryEncoder, OtlpProtoFields.ProfilesDictionary.ATTRIBUTE_TABLE); + writeEmptyEntry(dictionaryEncoder, OtlpProtoFields.ProfilesDictionary.STACK_TABLE); + }); + return encoder.toByteArray(); + } + + private static void writeEmptyEntry(ProtobufEncoder encoder, int tableField) { + encoder.writeTag(tableField, ProtobufEncoder.WIRETYPE_LENGTH_DELIMITED); + encoder.writeVarint(0); + } + + private void fillProfileId() { + // ThreadLocalRandom over UUID.randomUUID (SecureRandom) — uniqueness suffices, and + // SecureRandom can block on entropy + long msb = ThreadLocalRandom.current().nextLong(); + long lsb = ThreadLocalRandom.current().nextLong(); + for (int i = 0; i < 8; i++) { + profileId[i] = (byte) ((msb >> (56 - i * 8)) & 0xFF); + profileId[i + 8] = (byte) ((lsb >> (56 - i * 8)) & 0xFF); + } + } +} diff --git a/dd-java-agent/agent-profiling/profiling-uploader/src/main/java/com/datadog/profiling/uploader/OtlpProfileUploader.java b/dd-java-agent/agent-profiling/profiling-uploader/src/main/java/com/datadog/profiling/uploader/OtlpProfileUploader.java new file mode 100644 index 00000000000..fa6ceefc48a --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-uploader/src/main/java/com/datadog/profiling/uploader/OtlpProfileUploader.java @@ -0,0 +1,407 @@ +/* + * Copyright 2025 Datadog + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datadog.profiling.uploader; + +import static datadog.trace.api.config.ProfilingConfig.PROFILING_OTLP_ENABLED; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_OTLP_ENABLED_DEFAULT; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_OTLP_MODE; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_OTLP_MODE_DEFAULT; +import static datadog.trace.api.telemetry.LogCollector.SEND_TELEMETRY; + +import com.datadog.profiling.otel.JfrToOtlpConverter; +import datadog.communication.otlp.OtlpCanonicalResourceAttributes; +import datadog.communication.otlp.OtlpPayload; +import datadog.communication.otlp.OtlpResponse; +import datadog.communication.otlp.OtlpSender; +import datadog.trace.api.Config; +import datadog.trace.api.config.ProfilingConfig; +import datadog.trace.api.profiling.RecordingData; +import datadog.trace.api.profiling.RecordingDataListener; +import datadog.trace.api.profiling.RecordingType; +import datadog.trace.api.telemetry.OtlpTelemetry; +import datadog.trace.bootstrap.config.provider.ConfigProvider; +import datadog.trace.util.AgentThreadFactory; +import datadog.trace.util.TempLocationManager; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.time.Instant; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class OtlpProfileUploader implements RecordingDataListener { + + private static final Logger log = LoggerFactory.getLogger(OtlpProfileUploader.class); + private static final int TERMINATION_TIMEOUT_SEC = 5; + // cap on the raw JFR recording embedded as original_payload in FULL mode — beyond this the + // converted profile is sent without the payload instead of materializing an unbounded blob + private static final long MAX_ORIGINAL_PAYLOAD_BYTES = 64L * 1024 * 1024; + + // canonical resource attribute keys emitted from fixed configuration; a configured profiling + // or global tag with one of these (lower-cased) keys is dropped so it cannot shadow the + // canonical value — mirrors the tracer's OTLP traces export filtering + private static final Set CANONICAL_RESOURCE_KEYS = + new HashSet<>( + java.util.Arrays.asList( + "service", + "env", + "version", + "service.name", + "deployment.environment.name", + "service.version", + "host.name", + "telemetry.sdk.name", + "telemetry.sdk.version", + "telemetry.sdk.language")); + + private final OtlpSender sender; + private final ExecutorService executor; + private final boolean enabled; + private final int terminationTimeout; + private final ProfilingConfig.OtlpMode mode; + private final Map resourceAttributes; + // null unless mode is LIGHT; its reusable buffer is guarded by exportLock + private final LightweightOtlpEncoder lightweightEncoder; + // serializes conversion and send: at most one recording is converted and held in memory at a + // time, and the LIGHT encoder buffer is not overwritten while a send is still reading it + private final ReentrantLock exportLock = new ReentrantLock(); + + public OtlpProfileUploader(final Config config, final ConfigProvider configProvider) { + this(config, configProvider, TERMINATION_TIMEOUT_SEC); + } + + OtlpProfileUploader( + final Config config, final ConfigProvider configProvider, int terminationTimeout) { + boolean enabledConfig = + configProvider.getBoolean(PROFILING_OTLP_ENABLED, PROFILING_OTLP_ENABLED_DEFAULT); + this.terminationTimeout = terminationTimeout; + // the sender owns OkHttp clients and connection pools — only build it when enabled. + // A bad optional-feature configuration must not abort profiling: degrade to disabled + // and keep the classic JFR upload. + OtlpSender createdSender = null; + if (enabledConfig) { + try { + createdSender = createSender(config); + } catch (Exception e) { + log.error("Failed to create OTLP profiles sender; OTLP profile export disabled", e); + } + } + this.enabled = enabledConfig && createdSender != null; + this.sender = createdSender; + if (enabled) { + log.warn( + SEND_TELEMETRY, + "OTLP profiles export is enabled. The OTLP profiles protocol is at Development maturity" + + " (profiles specification: Alpha) and may change incompatibly; do not use it in" + + " production."); + } + this.mode = + configProvider.getEnum( + PROFILING_OTLP_MODE, ProfilingConfig.OtlpMode.class, PROFILING_OTLP_MODE_DEFAULT); + this.resourceAttributes = buildResourceAttributes(config); + this.lightweightEncoder = + mode == ProfilingConfig.OtlpMode.LIGHT + ? new LightweightOtlpEncoder(resourceAttributes) + : null; + // a single worker with no queue: a recording that arrives while the previous export is still + // running is dropped instead of being converted and buffered alongside it + this.executor = + new ThreadPoolExecutor( + 0, + 1, + 60L, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + new AgentThreadFactory(AgentThreadFactory.AgentThread.PROFILER_HTTP_DISPATCHER), + new ThreadPoolExecutor.AbortPolicy()); + log.debug( + "OTLP profile uploader initialized: endpoint={}, protocol={}", + config.getOtlpProfilesEndpoint(), + config.getOtlpProfilesProtocol()); + } + + private static OtlpSender createSender(Config config) { + OtlpSender created = OtlpProfilesSenderFactory.create(config); + if (created == null) { + throw new IllegalStateException( + "Unsupported OTLP profiles protocol: " + config.getOtlpProfilesProtocol()); + } + return created; + } + + @Override + public void onNewData(RecordingType type, RecordingData data, boolean handleSynchronously) { + upload(type, data, handleSynchronously, null); + } + + public void upload(RecordingType type, RecordingData data, boolean sync, Runnable onCompletion) { + if (!enabled) { + data.release(); + if (onCompletion != null) { + onCompletion.run(); + } + return; + } + if (sync) { + // the sync path runs on shutdown/snapshot; bound the wait behind an in-flight async export + // (which may be retrying) instead of blocking for its full duration + boolean locked = false; + try { + locked = exportLock.tryLock(terminationTimeout, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + if (!locked) { + skipExport(data, onCompletion); + return; + } + try { + // reentrant: export acquires exportLock again + export(data, onCompletion); + } finally { + exportLock.unlock(); + } + return; + } + try { + executor.execute(() -> export(data, onCompletion)); + } catch (RejectedExecutionException e) { + skipExport(data, onCompletion); + } + } + + private static void skipExport(RecordingData data, Runnable onCompletion) { + log.warn("OTLP profile export skipped: the previous export is still in progress"); + // the export never started, but the attempt happened — count it so failures cannot exceed + // attempts under backpressure + OtlpTelemetry.getInstance().onProfilesExportAttempt(); + OtlpTelemetry.getInstance().onProfilesExportComplete(false); + data.release(); + if (onCompletion != null) { + onCompletion.run(); + } + } + + private void export(RecordingData data, Runnable onCompletion) { + exportLock.lock(); + try { + try { + OtlpPayload payload; + long conversionNanos; + try { + long conversionStartNanos = System.nanoTime(); + payload = convertToOtlp(data); + conversionNanos = System.nanoTime() - conversionStartNanos; + } finally { + // the payload is fully materialized in memory, so the recording is no longer needed + data.release(); + } + // LIGHT mode does not convert the recording (it exports the raw JFR); recording its + // encoding time would skew the conversion timing telemetry + if (mode != ProfilingConfig.OtlpMode.LIGHT) { + OtlpTelemetry.getInstance().onProfilesConversion(conversionNanos); + } + if (payload == null) { + // skipped by the LIGHT-mode size cap — nothing to export + return; + } + log.debug( + "JFR to OTLP conversion took {} ms (mode={}, bytes={})", + TimeUnit.NANOSECONDS.toMillis(conversionNanos), + mode, + payload.getContentLength()); + send(payload); + } catch (Exception | LinkageError e) { + // not rethrown so that the classic JFR upload continues independently; + // LinkageError covers JVMs where the jafar parser classes cannot link + log.error("Failed to upload OTLP profile", e); + } finally { + if (onCompletion != null) { + onCompletion.run(); + } + } + } finally { + exportLock.unlock(); + } + } + + private void send(OtlpPayload payload) { + try { + OtlpTelemetry.getInstance().onProfilesExportAttempt(); + OtlpResponse response = sender.send(payload); + OtlpTelemetry.getInstance().onProfilesExportComplete(response.success()); + if (!response.success()) { + log.warn( + "OTLP profile upload failed: status={}", + response.status().isPresent() ? response.status().getAsInt() : "unknown"); + } + } catch (Exception e) { + log.error("OTLP profile upload failed", e); + OtlpTelemetry.getInstance().onProfilesExportComplete(false); + } + } + + // canonical attributes are shared with the tracer's OTLP traces export + private static Map buildResourceAttributes(Config config) { + Map attributes = new LinkedHashMap<>(); + OtlpCanonicalResourceAttributes.visit(config, attributes::put); + + // merge the user-configured tags (getMergedProfilingTags: global + profiling + runtime + + // host tags, the same set the classic uploader sends); keys already emitted as canonical + // attributes above are skipped + config + .getMergedProfilingTags() + .forEach( + (key, value) -> { + if (value == null + || value.isEmpty() + || attributes.containsKey(key) + || CANONICAL_RESOURCE_KEYS.contains(key.toLowerCase(Locale.ROOT))) { + return; + } + attributes.put(key, value); + }); + return Collections.unmodifiableMap(attributes); + } + + private OtlpPayload convertToOtlp(RecordingData data) throws IOException { + if (mode == ProfilingConfig.OtlpMode.LIGHT) { + return convertLightweight(data); + } + + JfrToOtlpConverter converter = new JfrToOtlpConverter(); + boolean includePayload = mode == ProfilingConfig.OtlpMode.FULL; + converter.setResourceAttributes(resourceAttributes); + + Path jfrFile = data.getPath(); + if (jfrFile != null) { + applyPayloadCap(converter, jfrFile, includePayload); + converter.addFile(jfrFile, data.getStart(), data.getEnd()); + return new OtlpPayload( + ByteBuffer.wrap(converter.convert(JfrToOtlpConverter.Kind.PROTO)), + OtlpPayload.PROTOBUF_CONTENT_TYPE); + } + + Path tempDir = TempLocationManager.getInstance().getTempDir(); + Path temp = Files.createTempFile(tempDir, "dd-otlp-", ".jfr"); + // the copy, the conversion, and the temp-file cleanup must share one try/finally — + // a failure while streaming would leak the temp file with the full JFR recording + try { + try (InputStream stream = data.getStream()) { + Files.copy(stream, temp, StandardCopyOption.REPLACE_EXISTING); + } + applyPayloadCap(converter, temp, includePayload); + converter.addFile(temp, data.getStart(), data.getEnd()); + return new OtlpPayload( + ByteBuffer.wrap(converter.convert(JfrToOtlpConverter.Kind.PROTO)), + OtlpPayload.PROTOBUF_CONTENT_TYPE); + } finally { + Files.deleteIfExists(temp); + } + } + + // the converted profile plus the embedded raw recording is buffered as one heap byte[]; + // cap the payload so an oversized recording is exported converted-only + private static void applyPayloadCap(JfrToOtlpConverter converter, Path jfrFile, boolean wanted) { + if (!wanted) { + return; + } + try { + long size = Files.size(jfrFile); + if (size > MAX_ORIGINAL_PAYLOAD_BYTES) { + log.warn( + "JFR recording of {} bytes exceeds the original_payload cap of {} bytes; " + + "exporting the converted profile without the original payload", + size, + MAX_ORIGINAL_PAYLOAD_BYTES); + return; + } + } catch (IOException e) { + log.warn("Could not stat JFR recording; exporting without the original payload", e); + return; + } + converter.setIncludeOriginalPayload(true); + } + + private OtlpPayload convertLightweight(RecordingData data) throws IOException { + Path jfrFile = data.getPath(); + if (jfrFile != null) { + return encodeLightweight(jfrFile, data.getStart(), data.getEnd()); + } + + // Fallback: save stream to temp file, then encode; copy, encode, and cleanup share one + // try/finally so a failed copy cannot leak the temp file. + Path tempDir = TempLocationManager.getInstance().getTempDir(); + Path temp = Files.createTempFile(tempDir, "dd-otlp-", ".jfr"); + try { + try (InputStream stream = data.getStream()) { + Files.copy(stream, temp, StandardCopyOption.REPLACE_EXISTING); + } + return encodeLightweight(temp, data.getStart(), data.getEnd()); + } finally { + Files.deleteIfExists(temp); + } + } + + // LIGHT mode exports the raw recording as the profile content, so the recording size bounds + // the payload — recordings above the cap are skipped instead of buffering an unbounded blob + private OtlpPayload encodeLightweight(Path jfrFile, Instant start, Instant end) + throws IOException { + long jfrSize = Files.size(jfrFile); + if (jfrSize > MAX_ORIGINAL_PAYLOAD_BYTES) { + log.warn( + "JFR recording of {} bytes exceeds the {} byte LIGHT-mode export cap; " + + "skipping OTLP profile export", + jfrSize, + MAX_ORIGINAL_PAYLOAD_BYTES); + return null; + } + ByteBuffer encoded = lightweightEncoder.encode(jfrFile, start, end); + return new OtlpPayload(encoded, OtlpPayload.PROTOBUF_CONTENT_TYPE); + } + + public void shutdown() { + log.debug("Shutting down OTLP profile uploader"); + executor.shutdown(); + try { + if (!executor.awaitTermination(terminationTimeout, TimeUnit.SECONDS)) { + log.warn("OTLP uploader executor did not terminate in {} seconds", terminationTimeout); + executor.shutdownNow(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + executor.shutdownNow(); + } + if (sender != null) { + sender.shutdown(); + } + } +} diff --git a/dd-java-agent/agent-profiling/profiling-uploader/src/main/java/com/datadog/profiling/uploader/OtlpProfilesSenderFactory.java b/dd-java-agent/agent-profiling/profiling-uploader/src/main/java/com/datadog/profiling/uploader/OtlpProfilesSenderFactory.java new file mode 100644 index 00000000000..e4334470872 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-uploader/src/main/java/com/datadog/profiling/uploader/OtlpProfilesSenderFactory.java @@ -0,0 +1,46 @@ +package com.datadog.profiling.uploader; + +import static datadog.trace.api.ConfigDefaults.DEFAULT_OTLP_GRPC_PROFILES_ENDPOINT; +import static datadog.trace.api.ConfigDefaults.DEFAULT_OTLP_HTTP_PROFILES_ENDPOINT; + +import datadog.communication.otlp.OtlpGrpcSender; +import datadog.communication.otlp.OtlpHttpSender; +import datadog.communication.otlp.OtlpSender; +import datadog.trace.api.Config; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +// mirrors OtlpMetricsSenderFactory and OtlpLogsService +final class OtlpProfilesSenderFactory { + private static final Logger log = LoggerFactory.getLogger(OtlpProfilesSenderFactory.class); + + private OtlpProfilesSenderFactory() {} + + static OtlpSender create(Config config) { + switch (config.getOtlpProfilesProtocol()) { + case GRPC: + return new OtlpGrpcSender( + config.getOtlpProfilesEndpoint(), + "/" + DEFAULT_OTLP_GRPC_PROFILES_ENDPOINT, + config.getOtlpProfilesHeaders(), + config.getOtlpProfilesTimeout(), + config.getOtlpProfilesCompression()); + case HTTP_PROTOBUF: + return new OtlpHttpSender( + config.getOtlpProfilesEndpoint(), + "/" + DEFAULT_OTLP_HTTP_PROFILES_ENDPOINT, + config.getOtlpProfilesHeaders(), + config.getOtlpProfilesTimeout(), + config.getOtlpProfilesCompression()); + case HTTP_JSON: + // fail fast like the default branch — a silent downgrade ships payloads with a content + // type and serialization the operator did not request + throw new IllegalStateException( + "Unsupported OTLP profiles protocol: " + config.getOtlpProfilesProtocol()); + default: + // fail fast instead of returning a sender that NPEs at first upload + throw new IllegalStateException( + "Unsupported OTLP profiles protocol: " + config.getOtlpProfilesProtocol()); + } + } +} diff --git a/dd-java-agent/agent-profiling/profiling-uploader/src/test/java/com/datadog/profiling/uploader/LightweightOtlpEncoderTest.java b/dd-java-agent/agent-profiling/profiling-uploader/src/test/java/com/datadog/profiling/uploader/LightweightOtlpEncoderTest.java new file mode 100644 index 00000000000..22179a7e9ff --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-uploader/src/test/java/com/datadog/profiling/uploader/LightweightOtlpEncoderTest.java @@ -0,0 +1,180 @@ +package com.datadog.profiling.uploader; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.datadog.profiling.otel.proto.OtlpProtoFields; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class LightweightOtlpEncoderTest { + + private static final Instant START = Instant.ofEpochSecond(1000); + private static final Instant END = Instant.ofEpochSecond(1060); + + @TempDir Path tempDir; + + private final LightweightOtlpEncoder encoder = + new LightweightOtlpEncoder(Collections.singletonMap("service.name", "test-service")); + + @Test + void embedsRecordingInWellFormedMessage() throws IOException { + // 300 bytes needs a two-byte length varint at every nesting level + byte[] jfr = randomBytes(300); + + assertPayload(encoder.encode(write("a.jfr", jfr), START, END), jfr); + } + + @Test + void reusesBufferForSmallerRecording() throws IOException { + byte[] large = randomBytes(200_000); + byte[] small = randomBytes(1_000); + + ByteBuffer first = encoder.encode(write("large.jfr", large), START, END); + assertPayload(first, large); + ByteBuffer second = encoder.encode(write("small.jfr", small), START, END); + assertPayload(second, small); + + assertSame(first.array(), second.array()); + } + + @Test + void growsBufferForLargerRecording() throws IOException { + byte[] small = randomBytes(1_000); + byte[] large = randomBytes(2 * 1024 * 1024); + + ByteBuffer first = encoder.encode(write("small.jfr", small), START, END); + ByteBuffer second = encoder.encode(write("large.jfr", large), START, END); + + assertNotSame(first.array(), second.array()); + assertPayload(second, large); + } + + @Test + void shrinksBufferAboveRetentionCap() throws IOException { + byte[] large = randomBytes(9 * 1024 * 1024); + byte[] small = randomBytes(1_000); + + ByteBuffer first = encoder.encode(write("large.jfr", large), START, END); + int largeCapacity = first.array().length; + ByteBuffer second = encoder.encode(write("small.jfr", small), START, END); + + assertTrue(second.array().length < largeCapacity); + assertPayload(second, small); + } + + @Test + void writesZeroValueEntryInEveryDictionaryTable() throws IOException { + ByteBuffer encoded = encoder.encode(write("a.jfr", randomBytes(100)), START, END); + byte[] message = new byte[encoded.remaining()]; + encoded.duplicate().get(message); + + Map dictionary = + parse(parse(message).get(OtlpProtoFields.ProfilesData.DICTIONARY)); + int[] tables = { + OtlpProtoFields.ProfilesDictionary.MAPPING_TABLE, + OtlpProtoFields.ProfilesDictionary.LOCATION_TABLE, + OtlpProtoFields.ProfilesDictionary.FUNCTION_TABLE, + OtlpProtoFields.ProfilesDictionary.LINK_TABLE, + OtlpProtoFields.ProfilesDictionary.ATTRIBUTE_TABLE, + OtlpProtoFields.ProfilesDictionary.STACK_TABLE + }; + for (int table : tables) { + assertArrayEquals(new byte[0], dictionary.get(table), "table field " + table); + } + } + + private static void assertPayload(ByteBuffer encoded, byte[] expectedJfr) { + byte[] message = new byte[encoded.remaining()]; + encoded.duplicate().get(message); + + Map profilesData = parse(message); + assertNotNull(profilesData.get(OtlpProtoFields.ProfilesData.DICTIONARY)); + Map resourceProfiles = + parse(profilesData.get(OtlpProtoFields.ProfilesData.RESOURCE_PROFILES)); + String resource = + new String( + resourceProfiles.get(OtlpProtoFields.ResourceProfiles.RESOURCE), + StandardCharsets.ISO_8859_1); + assertTrue(resource.contains("test-service")); + Map scopeProfiles = + parse(resourceProfiles.get(OtlpProtoFields.ResourceProfiles.SCOPE_PROFILES)); + Map profile = parse(scopeProfiles.get(OtlpProtoFields.ScopeProfiles.PROFILES)); + + assertEquals( + "jfr", + new String( + profile.get(OtlpProtoFields.Profile.ORIGINAL_PAYLOAD_FORMAT), StandardCharsets.UTF_8)); + assertEquals(16, profile.get(OtlpProtoFields.Profile.PROFILE_ID).length); + assertArrayEquals(expectedJfr, profile.get(OtlpProtoFields.Profile.ORIGINAL_PAYLOAD)); + } + + /** + * Returns the length-delimited fields of a message by field number; fails if any length prefix + * does not match the bytes that follow, which is what a broken precomputed length would cause. + */ + private static Map parse(byte[] message) { + Map fields = new HashMap<>(); + int[] pos = {0}; + while (pos[0] < message.length) { + long tag = readVarint(message, pos); + int fieldNumber = (int) (tag >>> 3); + switch ((int) (tag & 7)) { + case 0: + readVarint(message, pos); + break; + case 1: + pos[0] += 8; + break; + case 2: + int length = (int) readVarint(message, pos); + fields.put(fieldNumber, Arrays.copyOfRange(message, pos[0], pos[0] + length)); + pos[0] += length; + break; + case 5: + pos[0] += 4; + break; + default: + throw new AssertionError("unexpected wire type in tag " + tag); + } + } + assertEquals(message.length, pos[0], "message length does not match its fields"); + return fields; + } + + private static long readVarint(byte[] buf, int[] pos) { + long value = 0; + for (int shift = 0; ; shift += 7) { + byte b = buf[pos[0]++]; + value |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return value; + } + } + } + + private Path write(String name, byte[] content) throws IOException { + return Files.write(tempDir.resolve(name), content); + } + + private static byte[] randomBytes(int size) { + byte[] bytes = new byte[size]; + new Random(size).nextBytes(bytes); + return bytes; + } +} diff --git a/dd-java-agent/agent-profiling/profiling-uploader/src/test/java/com/datadog/profiling/uploader/OtlpProfileUploaderTest.java b/dd-java-agent/agent-profiling/profiling-uploader/src/test/java/com/datadog/profiling/uploader/OtlpProfileUploaderTest.java new file mode 100644 index 00000000000..6cb3449755e --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-uploader/src/test/java/com/datadog/profiling/uploader/OtlpProfileUploaderTest.java @@ -0,0 +1,243 @@ +/* + * Copyright 2025 Datadog + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datadog.profiling.uploader; + +import static datadog.trace.api.config.ProfilingConfig.PROFILING_OTLP_ENABLED; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_OTLP_MODE; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; + +import datadog.trace.api.Config; +import datadog.trace.api.config.OtlpConfig; +import datadog.trace.api.config.ProfilingConfig; +import datadog.trace.api.profiling.ProfilingSnapshot; +import datadog.trace.api.profiling.RecordingData; +import datadog.trace.api.profiling.RecordingInputStream; +import datadog.trace.api.profiling.RecordingType; +import datadog.trace.bootstrap.config.provider.ConfigProvider; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.Collections; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import okhttp3.mockwebserver.Dispatcher; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; +import org.mockito.stubbing.Answer; + +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +public class OtlpProfileUploaderTest { + + private static final String RECORDING_RESOURCE = "/test-recording.jfr"; + private static final RecordingType RECORDING_TYPE = RecordingType.CONTINUOUS; + private static final String RECORDING_NAME = "test-recording"; + private static final int PROFILE_START = 1000; + private static final int PROFILE_END = 1100; + + private final Duration REQUEST_TIMEOUT = Duration.ofSeconds(10); + private final Duration TERMINATION_TIMEOUT = REQUEST_TIMEOUT.plus(Duration.ofSeconds(5)); + + @Mock private Config config; + @Mock private ConfigProvider configProvider; + + private final MockWebServer server = new MockWebServer(); + private String otlpUrl; + + private OtlpProfileUploader uploader; + + @BeforeEach + public void setup() throws IOException { + server.start(); + otlpUrl = server.url("/v1/profiles").toString(); + + when(config.getFinalProfilingUrl()).thenReturn("http://localhost:8126"); + when(config.getProfilingUploadTimeout()).thenReturn((int) REQUEST_TIMEOUT.getSeconds()); + when(config.getProfilingProxyHost()).thenReturn(null); + when(config.getProfilingProxyPort()).thenReturn(8080); + when(config.getProfilingProxyUsername()).thenReturn(null); + when(config.getProfilingProxyPassword()).thenReturn(null); + + when(config.getServiceName()).thenReturn("test-service"); + when(config.getEnv()).thenReturn(""); + when(config.getVersion()).thenReturn(""); + when(config.isReportHostName()).thenReturn(false); + + when(config.getOtlpProfilesProtocol()).thenReturn(OtlpConfig.Protocol.HTTP_PROTOBUF); + when(config.getOtlpProfilesEndpoint()).thenReturn(otlpUrl); + when(config.getOtlpProfilesHeaders()).thenReturn(Collections.emptyMap()); + when(config.getOtlpProfilesTimeout()).thenReturn((int) REQUEST_TIMEOUT.getSeconds()); + when(config.getOtlpProfilesCompression()).thenReturn(OtlpConfig.Compression.NONE); + + when(configProvider.getBoolean(PROFILING_OTLP_ENABLED, false)).thenReturn(true); + when(configProvider.getEnum( + PROFILING_OTLP_MODE, ProfilingConfig.OtlpMode.class, ProfilingConfig.OtlpMode.LIGHT)) + .thenReturn(ProfilingConfig.OtlpMode.LIGHT); + + uploader = + new OtlpProfileUploader(config, configProvider, (int) TERMINATION_TIMEOUT.getSeconds()); + } + + @AfterEach + public void teardown() throws IOException { + uploader.shutdown(); + server.shutdown(); + } + + @Test + public void testDisabledUploader() throws Exception { + when(configProvider.getBoolean(PROFILING_OTLP_ENABLED, false)).thenReturn(false); + + OtlpProfileUploader disabledUploader = + new OtlpProfileUploader(config, configProvider, (int) TERMINATION_TIMEOUT.getSeconds()); + + RecordingData data = mockRecordingData(); + + disabledUploader.onNewData(RECORDING_TYPE, data, true); + + assertEquals(0, server.getRequestCount()); + verify(data).release(); + + disabledUploader.shutdown(); + } + + @Test + public void testLightweightUploadCarriesResourceAttributes() throws Exception { + // OtlpHttpSender retries up to 5 times (HttpRetryPolicy.Factory(5, ...)). Serve an endless + // stream of 200s so a retried attempt never blocks on an empty response queue (which would + // otherwise stall 10s per retry and inflate getRequestCount past 1 under CI contention). + server.setDispatcher( + new Dispatcher() { + @Override + public MockResponse dispatch(RecordedRequest request) { + return new MockResponse(); + } + }); + + RecordingData data = mockRecordingData(); + + uploader.onNewData(RECORDING_TYPE, data, true); + + // The uploader initiates exactly one upload; getRequestCount() may exceed 1 only because of + // transport-level retries, which are outside the scope of this test. + assertTrue(server.getRequestCount() >= 1, "expected at least one upload request"); + RecordedRequest request = server.takeRequest(REQUEST_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); + assertNotNull(request); + byte[] body = request.getBody().readByteArray(); + // String fields are encoded inline on the wire, so the attribute key/value pairs are + // searchable in the raw payload + assertTrue(new String(body, StandardCharsets.ISO_8859_1).contains("service.name")); + assertTrue(new String(body, StandardCharsets.ISO_8859_1).contains("test-service")); + assertTrue(new String(body, StandardCharsets.ISO_8859_1).contains("telemetry.sdk.name")); + verify(data).release(); + } + + @Test + public void syncUploadSkipsWhenAsyncExportDoesNotFinishInTime() throws Exception { + CountDownLatch requestReceived = new CountDownLatch(1); + CountDownLatch unblock = new CountDownLatch(1); + server.setDispatcher( + new Dispatcher() { + @Override + public MockResponse dispatch(RecordedRequest request) throws InterruptedException { + requestReceived.countDown(); + unblock.await(); + return new MockResponse(); + } + }); + OtlpProfileUploader shortWaitUploader = new OtlpProfileUploader(config, configProvider, 1); + try { + shortWaitUploader.onNewData(RECORDING_TYPE, mockRecordingData(), false); + assertTrue(requestReceived.await(REQUEST_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS)); + + RecordingData syncData = mockRecordingData(); + shortWaitUploader.onNewData(RECORDING_TYPE, syncData, true); + + verify(syncData, never()).getStream(); + verify(syncData).release(); + } finally { + unblock.countDown(); + shortWaitUploader.shutdown(); + } + } + + @Test + public void warnsThatOtlpProfilesAreNotProductionReadyWhenEnabled() { + String log = captureStderr(() -> new OtlpProfileUploader(config, configProvider, 1).shutdown()); + + assertTrue(log.contains("WARN"), log); + assertTrue(log.contains("OTLP profiles export is enabled"), log); + assertTrue(log.contains("do not use it in production"), log); + } + + @Test + public void doesNotWarnWhenDisabled() { + when(configProvider.getBoolean(PROFILING_OTLP_ENABLED, false)).thenReturn(false); + + String log = captureStderr(() -> new OtlpProfileUploader(config, configProvider, 1).shutdown()); + + assertFalse(log.contains("OTLP profiles export is enabled"), log); + } + + // the test classpath binds slf4j-simple, which writes to the current System.err + private static String captureStderr(Runnable action) { + PrintStream original = System.err; + ByteArrayOutputStream captured = new ByteArrayOutputStream(); + System.setErr(new PrintStream(captured, true)); + try { + action.run(); + } finally { + System.setErr(original); + } + return new String(captured.toByteArray(), StandardCharsets.UTF_8); + } + + private RecordingData mockRecordingData() throws IOException { + final RecordingData recordingData = mock(RecordingData.class, withSettings().lenient()); + when(recordingData.getStream()) + .then( + (Answer) + invocation -> + new RecordingInputStream(getClass().getResourceAsStream(RECORDING_RESOURCE))); + when(recordingData.getName()).thenReturn(RECORDING_NAME); + when(recordingData.getStart()).thenReturn(Instant.ofEpochSecond(PROFILE_START)); + when(recordingData.getEnd()).thenReturn(Instant.ofEpochSecond(PROFILE_END)); + when(recordingData.getKind()).thenReturn(ProfilingSnapshot.Kind.PERIODIC); + when(recordingData.getPath()).thenReturn(null); // Force stream-based conversion + return recordingData; + } +} diff --git a/dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/CompositeController.java b/dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/CompositeController.java index 236bf52b1ae..581d099ff51 100644 --- a/dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/CompositeController.java +++ b/dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/CompositeController.java @@ -118,7 +118,7 @@ public RecordingInputStream getStream() throws IOException { } @Override - public void release() { + protected void doRelease() { for (RecordingData data : data) { data.release(); } diff --git a/dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/ProfilingAgent.java b/dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/ProfilingAgent.java index 9175e0bc40f..5cbcdfe8865 100644 --- a/dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/ProfilingAgent.java +++ b/dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/ProfilingAgent.java @@ -3,6 +3,8 @@ import static datadog.environment.JavaVirtualMachine.isJavaVersion; import static datadog.environment.JavaVirtualMachine.isJavaVersionAtLeast; import static datadog.environment.JavaVirtualMachine.isOracleJDK8; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_OTLP_ENABLED; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_OTLP_ENABLED_DEFAULT; import static datadog.trace.api.config.ProfilingConfig.PROFILING_SCRUB_ENABLED; import static datadog.trace.api.config.ProfilingConfig.PROFILING_SCRUB_ENABLED_DEFAULT; import static datadog.trace.api.config.ProfilingConfig.PROFILING_SCRUB_FAIL_OPEN; @@ -19,6 +21,7 @@ import com.datadog.profiling.controller.ProfilingSystem; import com.datadog.profiling.controller.UnsupportedEnvironmentException; import com.datadog.profiling.controller.jfr.JFRAccess; +import com.datadog.profiling.uploader.OtlpProfileUploader; import com.datadog.profiling.uploader.ProfileUploader; import com.datadog.profiling.utils.Timestamper; import datadog.trace.api.Config; @@ -55,6 +58,7 @@ public class ProfilingAgent { private static volatile ProfilingSystem profiler; private static volatile ProfileUploader uploader; + private static volatile OtlpProfileUploader otlpUploader; private static class DataDumper implements RecordingDataListener { private final Path path; @@ -144,10 +148,15 @@ public static synchronized boolean run(final boolean earlyStart, Instrumentation final Controller controller = CompositeController.build(configProvider, context); String dumpPath = configProvider.getString(ProfilingConfig.PROFILING_DEBUG_DUMP_PATH); + // local: only the listener chain below consumes the dumper DataDumper dumper = dumpPath != null ? new DataDumper(Paths.get(dumpPath)) : null; uploader = new ProfileUploader(config, configProvider); + if (configProvider.getBoolean(PROFILING_OTLP_ENABLED, PROFILING_OTLP_ENABLED_DEFAULT)) { + otlpUploader = new OtlpProfileUploader(config, configProvider); + } + RecordingDataListener listener = uploader::upload; if (dumper != null) { RecordingDataListener upload = listener; @@ -157,7 +166,13 @@ public static synchronized boolean run(final boolean earlyStart, Instrumentation upload.onNewData(type, data, sync); }; } - // Scrubber wraps the combined dumper+uploader so debug dumps also contain scrubbed data + if (otlpUploader != null) { + OtlpProfileUploader otlp = otlpUploader; + listener = + withOtlpUpload(listener, (type, data, sync) -> otlp.upload(type, data, sync, null)); + } + // the scrubber must wrap the OTLP listener so OTLP receives the scrubbed copy, not the + // raw recording; it also wraps the combined dumper+uploader so debug dumps stay scrubbed // Oracle JDK 8 JFR format has quirks that make scrubbing unreliable — skip it to avoid // corrupting customer data if (configProvider.getBoolean(PROFILING_SCRUB_ENABLED, PROFILING_SCRUB_ENABLED_DEFAULT) @@ -197,7 +212,7 @@ public static synchronized boolean run(final boolean earlyStart, Instrumentation This means that if/when we implement functionality to manually shutdown profiler we would need to not forget to add code that removes this shutdown hook from JVM. */ - Runtime.getRuntime().addShutdownHook(new ShutdownHook(profiler, uploader)); + Runtime.getRuntime().addShutdownHook(new ShutdownHook(profiler, uploader, otlpUploader)); } catch (final IllegalStateException ex) { // The JVM is already shutting down. } @@ -209,6 +224,32 @@ public static synchronized boolean run(final boolean earlyStart, Instrumentation return false; } + /** + * Runs {@code downstream} first and then {@code otlp}, so OTLP export never delays the classic + * upload. The OTLP reference is taken before downstream runs, since downstream may release the + * base reference and free the recording; {@code otlp} must release that reference. If the + * reference cannot be taken, only {@code downstream} runs. + */ + static RecordingDataListener withOtlpUpload( + RecordingDataListener downstream, RecordingDataListener otlp) { + return (type, data, sync) -> { + boolean retained = false; + try { + data.retain(); + retained = true; + } catch (IllegalStateException e) { + log.warn(SEND_TELEMETRY, "OTLP upload skipped, JFR upload will continue", e); + } + try { + downstream.onNewData(type, data, sync); + } finally { + if (retained) { + otlp.onNewData(type, data, sync); + } + } + }; + } + private static RecordingDataListener wrapWithScrubber( RecordingDataListener listener, List excludeEventTypes, boolean failOpen) { try { @@ -226,17 +267,20 @@ private static boolean isStartForceFirstSafe() { } public static void shutdown() { - shutdown(profiler, uploader, false); + shutdown(profiler, uploader, otlpUploader, false); } public static void shutdown(boolean snapshot) { - shutdown(profiler, uploader, snapshot); + shutdown(profiler, uploader, otlpUploader, snapshot); } private static final AtomicBoolean shutDownFlag = new AtomicBoolean(); private static void shutdown( - ProfilingSystem profiler, ProfileUploader uploader, boolean snapshot) { + ProfilingSystem profiler, + ProfileUploader uploader, + OtlpProfileUploader otlpUploader, + boolean snapshot) { if (shutDownFlag.compareAndSet(false, true)) { if (profiler != null) { profiler.shutdown(snapshot); @@ -245,6 +289,10 @@ private static void shutdown( if (uploader != null) { uploader.shutdown(); } + + if (otlpUploader != null) { + otlpUploader.shutdown(); + } } } @@ -252,16 +300,21 @@ private static class ShutdownHook extends Thread { private final WeakReference profilerRef; private final WeakReference uploaderRef; + private final WeakReference otlpUploaderRef; - private ShutdownHook(final ProfilingSystem profiler, final ProfileUploader uploader) { + private ShutdownHook( + final ProfilingSystem profiler, + final ProfileUploader uploader, + final OtlpProfileUploader otlpUploader) { super(AGENT_THREAD_GROUP, "dd-profiler-shutdown-hook"); profilerRef = new WeakReference<>(profiler); uploaderRef = new WeakReference<>(uploader); + otlpUploaderRef = new WeakReference<>(otlpUploader); } @Override public void run() { - shutdown(profilerRef.get(), uploaderRef.get(), false); + shutdown(profilerRef.get(), uploaderRef.get(), otlpUploaderRef.get(), false); } } } diff --git a/dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/ScrubRecordingDataListener.java b/dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/ScrubRecordingDataListener.java index 211d0e23c64..f5931bfa68c 100644 --- a/dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/ScrubRecordingDataListener.java +++ b/dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/ScrubRecordingDataListener.java @@ -139,7 +139,7 @@ public RecordingInputStream getStream() throws IOException { } @Override - public void release() { + protected void doRelease() { try { Files.deleteIfExists(scrubbedFile); } catch (IOException e) { diff --git a/dd-java-agent/agent-profiling/src/test/java/com/datadog/profiling/agent/OtlpUploadListenerTest.java b/dd-java-agent/agent-profiling/src/test/java/com/datadog/profiling/agent/OtlpUploadListenerTest.java new file mode 100644 index 00000000000..adf685f0731 --- /dev/null +++ b/dd-java-agent/agent-profiling/src/test/java/com/datadog/profiling/agent/OtlpUploadListenerTest.java @@ -0,0 +1,99 @@ +package com.datadog.profiling.agent; + +import static com.datadog.profiling.agent.ProfilingAgent.withOtlpUpload; +import static java.util.Arrays.asList; +import static java.util.Collections.singletonList; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.api.profiling.ProfilingSnapshot; +import datadog.trace.api.profiling.RecordingData; +import datadog.trace.api.profiling.RecordingDataListener; +import datadog.trace.api.profiling.RecordingInputStream; +import datadog.trace.api.profiling.RecordingType; +import java.io.ByteArrayInputStream; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.Test; + +class OtlpUploadListenerTest { + + private final List calls = new ArrayList<>(); + private final TestRecordingData data = new TestRecordingData(); + + @Test + void classicUploadRunsBeforeOtlpAndRecordingOutlivesIt() { + RecordingDataListener downstream = + (type, d, sync) -> { + calls.add("downstream"); + d.release(); // drops the base reference, as the classic uploader does + }; + RecordingDataListener otlp = + (type, d, sync) -> { + calls.add("otlp:released=" + data.released); + d.release(); + }; + + withOtlpUpload(downstream, otlp).onNewData(RecordingType.CONTINUOUS, data, false); + + assertEquals(asList("downstream", "otlp:released=false"), calls); + assertTrue(data.released); + } + + @Test + void otlpRunsWhenClassicUploadThrows() { + RecordingDataListener downstream = + (type, d, sync) -> { + d.release(); + throw new IllegalArgumentException("boom"); + }; + RecordingDataListener otlp = + (type, d, sync) -> { + calls.add("otlp"); + d.release(); + }; + + assertThrows( + IllegalArgumentException.class, + () -> withOtlpUpload(downstream, otlp).onNewData(RecordingType.CONTINUOUS, data, false)); + + assertEquals(singletonList("otlp"), calls); + assertTrue(data.released); + } + + @Test + void skipsOtlpWhenRecordingAlreadyReleased() { + data.release(); + RecordingDataListener downstream = (type, d, sync) -> calls.add("downstream"); + RecordingDataListener otlp = (type, d, sync) -> calls.add("otlp"); + + withOtlpUpload(downstream, otlp).onNewData(RecordingType.CONTINUOUS, data, false); + + assertEquals(singletonList("downstream"), calls); + } + + private static final class TestRecordingData extends RecordingData { + boolean released; + + TestRecordingData() { + super(Instant.EPOCH, Instant.EPOCH, ProfilingSnapshot.Kind.PERIODIC); + } + + @Override + public RecordingInputStream getStream() { + return new RecordingInputStream(new ByteArrayInputStream(new byte[0])); + } + + @Override + protected void doRelease() { + released = true; + } + + @Override + public String getName() { + return "test"; + } + } +} diff --git a/dd-java-agent/build.gradle b/dd-java-agent/build.gradle index 3fe773b73a4..94993e58966 100644 --- a/dd-java-agent/build.gradle +++ b/dd-java-agent/build.gradle @@ -434,6 +434,11 @@ dependencies { // do not bring along slf4j and dependent subprojects // (which are loaded on the bootstrap cl) } + sharedShadowInclude project(':communication:otlp-exporter'), { + transitive = false + // do not bring along slf4j and dependent subprojects + // (which are loaded on the bootstrap cl) + } sharedShadowInclude group: 'com.datadoghq', name: 'sketches-java', version: '0.8.3' sharedShadowInclude project(':telemetry'), { transitive = false diff --git a/dd-trace-api/build.gradle.kts b/dd-trace-api/build.gradle.kts index faf02823ec9..8896010ba12 100644 --- a/dd-trace-api/build.gradle.kts +++ b/dd-trace-api/build.gradle.kts @@ -41,6 +41,7 @@ extra["excludedClassesCoverage"] = listOf( "datadog.trace.api.civisibility.noop.NoOpDDTestSuite", "datadog.trace.api.config.AIGuardConfig", "datadog.trace.api.config.ProfilingConfig", + "datadog.trace.api.config.ProfilingConfig.OtlpMode", "datadog.trace.api.interceptor.MutableSpan", "datadog.trace.api.profiling.Profiling", "datadog.trace.api.profiling.Profiling.NoOp", diff --git a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java index bf5bab30d05..12bf2b9bb72 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java @@ -136,6 +136,13 @@ public final class ConfigDefaults { public static final String DEFAULT_OTLP_HTTP_PORT = "4318"; public static final String DEFAULT_OTLP_GRPC_PORT = "4317"; + // Provisional — the OTLP profiles proto is still in v1development; update when stabilized. + // Current collectors expose /v1development/profiles and the v1development ProfilesService. + public static final String DEFAULT_OTLP_HTTP_PROFILES_ENDPOINT = "v1development/profiles"; + public static final String DEFAULT_OTLP_GRPC_PROFILES_ENDPOINT = + "opentelemetry.proto.collector.profiles.v1development.ProfilesService/Export"; + public static final int DEFAULT_OTLP_PROFILES_TIMEOUT = 30_000; // ms + static final int DEFAULT_DOGSTATSD_START_DELAY = 15; // seconds static final boolean DEFAULT_HEALTH_METRICS_ENABLED = true; diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/OtlpConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/OtlpConfig.java index 90fd9fd046f..3abf06f9ea2 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/OtlpConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/OtlpConfig.java @@ -42,6 +42,13 @@ public final class OtlpConfig { public static final String OTLP_TRACES_COMPRESSION = "otlp.traces.compression"; public static final String OTLP_TRACES_TIMEOUT = "otlp.traces.timeout"; + // OTLP Profiles — provisional, proto is still v1development + public static final String OTLP_PROFILES_ENDPOINT = "otlp.profiles.endpoint"; + public static final String OTLP_PROFILES_HEADERS = "otlp.profiles.headers"; + public static final String OTLP_PROFILES_PROTOCOL = "otlp.profiles.protocol"; + public static final String OTLP_PROFILES_COMPRESSION = "otlp.profiles.compression"; + public static final String OTLP_PROFILES_TIMEOUT = "otlp.profiles.timeout"; + public enum Protocol { GRPC, HTTP_PROTOBUF, diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/ProfilingConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/ProfilingConfig.java index 37dc6c3c500..2d479c14320 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/ProfilingConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/ProfilingConfig.java @@ -287,5 +287,22 @@ public final class ProfilingConfig { public static final String PROFILING_SCRUB_EXCLUDE_EVENTS = "profiling.scrub.exclude-events"; + // OTLP Profiles Format Support + public static final String PROFILING_OTLP_ENABLED = "profiling.otlp.enabled"; + public static final boolean PROFILING_OTLP_ENABLED_DEFAULT = false; + + // LIGHT = raw JFR as the original_payload blob, no conversion — the profile carries no samples, + // so only a backend that parses the embedded JFR can use it; + // FULL = converted samples + raw JFR blob; CONVERTED = converted samples only + public static final String PROFILING_OTLP_MODE = "profiling.otlp.mode"; + + public enum OtlpMode { + LIGHT, + FULL, + CONVERTED + } + + public static final ProfilingConfig.OtlpMode PROFILING_OTLP_MODE_DEFAULT = OtlpMode.LIGHT; + private ProfilingConfig() {} } diff --git a/dd-trace-core/build.gradle b/dd-trace-core/build.gradle index 46695a38059..be5504e19e0 100644 --- a/dd-trace-core/build.gradle +++ b/dd-trace-core/build.gradle @@ -80,6 +80,7 @@ dependencies { api project(':dd-trace-api') api project(':communication') + api project(':communication:otlp-exporter') api project(':internal-api') api project(':products:metrics:metrics-agent') // Note: dd-trace-core should NOT depend on metrics-lib directly diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/OtlpPayloadDispatcher.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/OtlpPayloadDispatcher.java index ed69f08c37e..1506d45d792 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/OtlpPayloadDispatcher.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/OtlpPayloadDispatcher.java @@ -1,9 +1,10 @@ package datadog.trace.common.writer; +import datadog.communication.otlp.OtlpPayload; +import datadog.communication.otlp.OtlpResponse; +import datadog.communication.otlp.OtlpSender; import datadog.trace.api.telemetry.OtlpTelemetry; import datadog.trace.core.CoreSpan; -import datadog.trace.core.otlp.common.OtlpPayload; -import datadog.trace.core.otlp.common.OtlpSender; import datadog.trace.core.otlp.trace.OtlpTraceCollector; import java.util.Collection; import java.util.Collections; @@ -39,7 +40,7 @@ public void flush() { OtlpPayload payload = collector.collectTraces(); if (payload != OtlpPayload.EMPTY) { OtlpTelemetry.getInstance().onTracesExportAttempt(); - RemoteApi.Response response = sender.send(payload); + OtlpResponse response = sender.send(payload); OtlpTelemetry.getInstance().onTracesExportComplete(response.success()); } } catch (RuntimeException e) { // don't catch severe Errors diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/OtlpWriter.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/OtlpWriter.java index 1650a77b52a..a84374ccbb2 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/OtlpWriter.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/OtlpWriter.java @@ -6,13 +6,13 @@ import static datadog.trace.api.ConfigDefaults.DEFAULT_OTLP_TRACES_TIMEOUT; import datadog.communication.ddagent.DroppingPolicy; +import datadog.communication.otlp.OtlpGrpcSender; +import datadog.communication.otlp.OtlpHttpSender; +import datadog.communication.otlp.OtlpSender; import datadog.trace.api.config.OtlpConfig; import datadog.trace.common.sampling.SingleSpanSampler; import datadog.trace.common.writer.ddagent.Prioritization; import datadog.trace.core.monitor.HealthMetrics; -import datadog.trace.core.otlp.common.OtlpGrpcSender; -import datadog.trace.core.otlp.common.OtlpHttpSender; -import datadog.trace.core.otlp.common.OtlpSender; import datadog.trace.core.otlp.trace.OtlpTraceCollector; import datadog.trace.core.otlp.trace.OtlpTraceJsonCollector; import datadog.trace.core.otlp.trace.OtlpTraceProtoCollector; diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpProtoBuffer.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpProtoBuffer.java index 81196bfaa23..13e4ec10d0c 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpProtoBuffer.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpProtoBuffer.java @@ -1,11 +1,12 @@ package datadog.trace.core.otlp.common; +import static datadog.communication.otlp.OtlpPayload.PROTOBUF_CONTENT_TYPE; import static datadog.trace.core.otlp.common.OtlpCommonProto.LEN_WIRE_TYPE; import static datadog.trace.core.otlp.common.OtlpCommonProto.sizeVarInt; import static datadog.trace.core.otlp.common.OtlpCommonProto.writeVarInt; -import static datadog.trace.core.otlp.common.OtlpPayload.PROTOBUF_CONTENT_TYPE; import static datadog.trace.util.BitUtils.nextPowerOfTwo; +import datadog.communication.otlp.OtlpPayload; import datadog.communication.serialization.GrowableBuffer; import java.nio.ByteBuffer; diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpResourceAttributes.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpResourceAttributes.java index 6489c7ae5bf..05686b3b892 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpResourceAttributes.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpResourceAttributes.java @@ -1,9 +1,9 @@ package datadog.trace.core.otlp.common; -import static datadog.communication.ddagent.TracerVersion.TRACER_VERSION; import static datadog.trace.api.DDTags.SDK_OTLP_EXPORT; import static java.util.Arrays.asList; +import datadog.communication.otlp.OtlpCanonicalResourceAttributes; import datadog.trace.api.Config; import datadog.trace.api.ProcessTags; import java.util.HashSet; @@ -46,26 +46,7 @@ private OtlpResourceAttributes() {} */ static void visitResourceAttributes( Config config, Map extraAttributes, BiConsumer visitor) { - String serviceName = config.getServiceName(); - String env = config.getEnv(); - String version = config.getVersion(); - - visitor.accept("service.name", serviceName); - if (!env.isEmpty()) { - visitor.accept("deployment.environment.name", env); - } - if (!version.isEmpty()) { - visitor.accept("service.version", version); - } - if (config.isReportHostName()) { - String hostName = config.getHostName(); - if (hostName != null && !hostName.isEmpty()) { - visitor.accept("host.name", hostName); - } - } - visitor.accept("telemetry.sdk.name", "datadog"); - visitor.accept("telemetry.sdk.version", TRACER_VERSION); - visitor.accept("telemetry.sdk.language", "java"); + OtlpCanonicalResourceAttributes.visit(config, visitor::accept); config .getGlobalTags() diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpSender.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpSender.java deleted file mode 100644 index 81172279b85..00000000000 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpSender.java +++ /dev/null @@ -1,10 +0,0 @@ -package datadog.trace.core.otlp.common; - -import datadog.trace.common.writer.RemoteApi; - -/** Sends chunks of OTLP data. */ -public interface OtlpSender { - RemoteApi.Response send(OtlpPayload payload); - - void shutdown(); -} diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/logs/OtlpLogsCollector.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/logs/OtlpLogsCollector.java index 0e699e22623..97950e5ece7 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/logs/OtlpLogsCollector.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/logs/OtlpLogsCollector.java @@ -1,6 +1,6 @@ package datadog.trace.core.otlp.logs; -import datadog.trace.core.otlp.common.OtlpPayload; +import datadog.communication.otlp.OtlpPayload; /** Collects logs ready for export. */ public abstract class OtlpLogsCollector { diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/logs/OtlpLogsJsonCollector.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/logs/OtlpLogsJsonCollector.java index b81822b6705..f168cdb3f23 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/logs/OtlpLogsJsonCollector.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/logs/OtlpLogsJsonCollector.java @@ -1,11 +1,12 @@ package datadog.trace.core.otlp.logs; +import static datadog.communication.otlp.OtlpPayload.JSON_CONTENT_TYPE; import static datadog.trace.core.otlp.common.OtlpCommonJson.writeAttribute; import static datadog.trace.core.otlp.common.OtlpCommonJson.writeScopeAndSchema; -import static datadog.trace.core.otlp.common.OtlpPayload.JSON_CONTENT_TYPE; import static datadog.trace.core.otlp.common.OtlpResourceJson.RESOURCE_FRAGMENT; import static datadog.trace.core.otlp.logs.OtlpLogsJson.writeLogRecordFields; +import datadog.communication.otlp.OtlpPayload; import datadog.json.JsonWriter; import datadog.trace.bootstrap.otel.common.OtelInstrumentationScope; import datadog.trace.bootstrap.otel.logs.data.OtelLogRecordProcessor; @@ -13,7 +14,6 @@ import datadog.trace.bootstrap.otlp.logs.OtlpLogsVisitor; import datadog.trace.bootstrap.otlp.logs.OtlpScopedLogsVisitor; import datadog.trace.core.otlp.common.LazyJsonArray; -import datadog.trace.core.otlp.common.OtlpPayload; import java.nio.ByteBuffer; import java.util.function.ObjIntConsumer; diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/logs/OtlpLogsProtoCollector.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/logs/OtlpLogsProtoCollector.java index 94fb0c6d1fa..98ffeaf691b 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/logs/OtlpLogsProtoCollector.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/logs/OtlpLogsProtoCollector.java @@ -7,6 +7,7 @@ import static datadog.trace.core.otlp.logs.OtlpLogsProto.recordLogRecordMessage; import static datadog.trace.core.otlp.logs.OtlpLogsProto.recordScopedLogsMessage; +import datadog.communication.otlp.OtlpPayload; import datadog.communication.serialization.GrowableBuffer; import datadog.trace.bootstrap.otel.common.OtelInstrumentationScope; import datadog.trace.bootstrap.otel.logs.data.OtelLogRecordProcessor; @@ -14,7 +15,6 @@ import datadog.trace.bootstrap.otlp.logs.OtlpLogsVisitor; import datadog.trace.bootstrap.otlp.logs.OtlpScopedLogsVisitor; import datadog.trace.core.otlp.common.OtlpCommonProto; -import datadog.trace.core.otlp.common.OtlpPayload; import datadog.trace.core.otlp.common.OtlpProtoBuffer; import java.util.function.ObjIntConsumer; diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/logs/OtlpLogsService.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/logs/OtlpLogsService.java index e65a3757782..496d169b75c 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/logs/OtlpLogsService.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/logs/OtlpLogsService.java @@ -3,13 +3,13 @@ import static datadog.trace.util.AgentThreadFactory.AgentThread.OTLP_LOGS_EXPORTER; import static datadog.trace.util.AgentThreadFactory.newAgentThread; +import datadog.communication.otlp.OtlpGrpcSender; +import datadog.communication.otlp.OtlpHttpSender; +import datadog.communication.otlp.OtlpPayload; +import datadog.communication.otlp.OtlpResponse; +import datadog.communication.otlp.OtlpSender; import datadog.trace.api.Config; import datadog.trace.api.telemetry.OtlpTelemetry; -import datadog.trace.common.writer.RemoteApi; -import datadog.trace.core.otlp.common.OtlpGrpcSender; -import datadog.trace.core.otlp.common.OtlpHttpSender; -import datadog.trace.core.otlp.common.OtlpPayload; -import datadog.trace.core.otlp.common.OtlpSender; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -111,7 +111,7 @@ private void export() { OtlpPayload payload = collector.waitForLogs(intervalMillis); if (payload != OtlpPayload.EMPTY) { int logRecordCount = collector.getLogRecordCount(); - RemoteApi.Response response = sender.send(payload); + OtlpResponse response = sender.send(payload); if (response.success()) { OtlpTelemetry.getInstance().onLogRecordsSubmitted(logRecordCount); } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpMetricsCollector.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpMetricsCollector.java index 3809160aadb..df18c9c5a22 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpMetricsCollector.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpMetricsCollector.java @@ -1,7 +1,7 @@ package datadog.trace.core.otlp.metrics; +import datadog.communication.otlp.OtlpPayload; import datadog.trace.bootstrap.otlp.metrics.OtlpMetricsVisitor; -import datadog.trace.core.otlp.common.OtlpPayload; import java.util.function.Consumer; /** Collects metrics ready for export. */ diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpMetricsJsonCollector.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpMetricsJsonCollector.java index 46fe6ae0abe..3bdb22c22ac 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpMetricsJsonCollector.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpMetricsJsonCollector.java @@ -1,15 +1,16 @@ package datadog.trace.core.otlp.metrics; +import static datadog.communication.otlp.OtlpPayload.JSON_CONTENT_TYPE; import static datadog.trace.bootstrap.otel.metrics.OtelInstrumentType.GAUGE; import static datadog.trace.bootstrap.otel.metrics.OtelInstrumentType.OBSERVABLE_GAUGE; import static datadog.trace.core.otlp.common.OtlpCommonJson.writeAttribute; import static datadog.trace.core.otlp.common.OtlpCommonJson.writeScopeAndSchema; -import static datadog.trace.core.otlp.common.OtlpPayload.JSON_CONTENT_TYPE; import static datadog.trace.core.otlp.common.OtlpResourceJson.RESOURCE_FRAGMENT; import static datadog.trace.core.otlp.metrics.OtlpMetricsJson.closeMetric; import static datadog.trace.core.otlp.metrics.OtlpMetricsJson.openMetric; import static datadog.trace.core.otlp.metrics.OtlpMetricsJson.writeDataPointValue; +import datadog.communication.otlp.OtlpPayload; import datadog.json.JsonWriter; import datadog.trace.api.time.TimeSource; import datadog.trace.bootstrap.otel.common.OtelInstrumentationScope; @@ -21,7 +22,6 @@ import datadog.trace.bootstrap.otlp.metrics.OtlpMetricsVisitor; import datadog.trace.bootstrap.otlp.metrics.OtlpScopedMetricsVisitor; import datadog.trace.core.otlp.common.LazyJsonArray; -import datadog.trace.core.otlp.common.OtlpPayload; import java.nio.ByteBuffer; import java.util.function.Consumer; diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpMetricsProtoCollector.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpMetricsProtoCollector.java index 82029367669..2a78bf8d37a 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpMetricsProtoCollector.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpMetricsProtoCollector.java @@ -13,6 +13,7 @@ import static datadog.trace.core.otlp.metrics.OtlpMetricsProto.recordMetricMessage; import static datadog.trace.core.otlp.metrics.OtlpMetricsProto.recordScopedMetricsMessage; +import datadog.communication.otlp.OtlpPayload; import datadog.communication.serialization.GrowableBuffer; import datadog.trace.api.time.TimeSource; import datadog.trace.bootstrap.otel.common.OtelInstrumentationScope; @@ -24,7 +25,6 @@ import datadog.trace.bootstrap.otlp.metrics.OtlpMetricsVisitor; import datadog.trace.bootstrap.otlp.metrics.OtlpScopedMetricsVisitor; import datadog.trace.core.otlp.common.OtlpCommonProto; -import datadog.trace.core.otlp.common.OtlpPayload; import datadog.trace.core.otlp.common.OtlpProtoBuffer; import java.util.function.Consumer; diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpMetricsSenderFactory.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpMetricsSenderFactory.java index 6ab62caeb8d..b16d91b9bda 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpMetricsSenderFactory.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpMetricsSenderFactory.java @@ -1,9 +1,9 @@ package datadog.trace.core.otlp.metrics; +import datadog.communication.otlp.OtlpGrpcSender; +import datadog.communication.otlp.OtlpHttpSender; +import datadog.communication.otlp.OtlpSender; import datadog.trace.api.Config; -import datadog.trace.core.otlp.common.OtlpGrpcSender; -import datadog.trace.core.otlp.common.OtlpHttpSender; -import datadog.trace.core.otlp.common.OtlpSender; import javax.annotation.Nullable; /** diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpMetricsService.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpMetricsService.java index 4a7580aa061..8ccab0dee48 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpMetricsService.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpMetricsService.java @@ -2,13 +2,13 @@ import static datadog.trace.util.AgentThreadFactory.AgentThread.OTLP_METRICS_EXPORTER; +import datadog.communication.otlp.OtlpPayload; +import datadog.communication.otlp.OtlpResponse; +import datadog.communication.otlp.OtlpSender; import datadog.trace.api.Config; import datadog.trace.api.config.OtlpConfig; import datadog.trace.api.telemetry.OtlpTelemetry; import datadog.trace.api.time.SystemTimeSource; -import datadog.trace.common.writer.RemoteApi; -import datadog.trace.core.otlp.common.OtlpPayload; -import datadog.trace.core.otlp.common.OtlpSender; import datadog.trace.util.AgentTaskScheduler; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; @@ -93,7 +93,7 @@ private void export() { OtlpPayload payload = collector.collectMetrics(); if (payload != OtlpPayload.EMPTY) { OtlpTelemetry.getInstance().onMetricsExportAttempt(); - RemoteApi.Response response = sender.send(payload); + OtlpResponse response = sender.send(payload); OtlpTelemetry.getInstance().onMetricsExportComplete(response.success()); } } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpStatsMetricWriter.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpStatsMetricWriter.java index 5e69d20e1f0..7acc5300901 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpStatsMetricWriter.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpStatsMetricWriter.java @@ -6,6 +6,9 @@ import static datadog.trace.bootstrap.otlp.common.OtlpAttributeVisitor.STRING_ARRAY_ATTRIBUTE; import static datadog.trace.bootstrap.otlp.common.OtlpAttributeVisitor.STRING_ATTRIBUTE; +import datadog.communication.otlp.OtlpPayload; +import datadog.communication.otlp.OtlpResponse; +import datadog.communication.otlp.OtlpSender; import datadog.metrics.api.Histogram; import datadog.trace.api.Config; import datadog.trace.api.config.OtlpConfig; @@ -20,11 +23,8 @@ import datadog.trace.bootstrap.otlp.metrics.OtlpMetricsVisitor; import datadog.trace.common.metrics.AggregateEntry; import datadog.trace.common.metrics.MetricWriter; -import datadog.trace.common.writer.RemoteApi; -import datadog.trace.core.otlp.common.OtlpPayload; import datadog.trace.core.otlp.common.OtlpResourceJson; import datadog.trace.core.otlp.common.OtlpResourceProto; -import datadog.trace.core.otlp.common.OtlpSender; import java.util.ArrayList; import java.util.List; import javax.annotation.Nullable; @@ -162,7 +162,7 @@ public void finishBucket() { OtlpPayload payload = collector.collectMetrics(this::emit, startNanos, endNanos); if (payload != OtlpPayload.EMPTY) { OtlpTelemetry.getInstance().onMetricsExportAttempt(); - RemoteApi.Response response = sender.send(payload); + OtlpResponse response = sender.send(payload); OtlpTelemetry.getInstance().onMetricsExportComplete(response.success()); } } finally { diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceCollector.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceCollector.java index 127e27af467..4931877e0b1 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceCollector.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceCollector.java @@ -2,8 +2,8 @@ import static datadog.trace.core.DDSpanContext.SPAN_SAMPLING_MECHANISM_TAG; +import datadog.communication.otlp.OtlpPayload; import datadog.trace.core.CoreSpan; -import datadog.trace.core.otlp.common.OtlpPayload; import java.util.List; /** Collects traces ready for export. */ diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceJsonCollector.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceJsonCollector.java index cbb39ed615b..38baaa8c566 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceJsonCollector.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceJsonCollector.java @@ -1,17 +1,17 @@ package datadog.trace.core.otlp.trace; +import static datadog.communication.otlp.OtlpPayload.JSON_CONTENT_TYPE; import static datadog.trace.core.otlp.common.OtlpCommonJson.writeScopeAndSchema; -import static datadog.trace.core.otlp.common.OtlpPayload.JSON_CONTENT_TYPE; import static datadog.trace.core.otlp.common.OtlpProtoBuffer.MAX_CAPACITY_BYTES; import static datadog.trace.core.otlp.common.OtlpResourceJson.TRACE_RESOURCE_FRAGMENT; import static datadog.trace.core.otlp.trace.OtlpTraceJson.writeSpan; +import datadog.communication.otlp.OtlpPayload; import datadog.json.JsonWriter; import datadog.trace.bootstrap.instrumentation.api.AgentSpanLink; import datadog.trace.bootstrap.otel.common.OtelInstrumentationScope; import datadog.trace.core.CoreSpan; import datadog.trace.core.DDSpan; -import datadog.trace.core.otlp.common.OtlpPayload; import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProtoCollector.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProtoCollector.java index ba51d9f9f3d..164755e413b 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProtoCollector.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProtoCollector.java @@ -5,13 +5,13 @@ import static datadog.trace.core.otlp.trace.OtlpTraceProto.recordSpanLinkMessage; import static datadog.trace.core.otlp.trace.OtlpTraceProto.recordSpanMessage; +import datadog.communication.otlp.OtlpPayload; import datadog.communication.serialization.GrowableBuffer; import datadog.trace.bootstrap.instrumentation.api.AgentSpanLink; import datadog.trace.bootstrap.otel.common.OtelInstrumentationScope; import datadog.trace.core.CoreSpan; import datadog.trace.core.DDSpan; import datadog.trace.core.otlp.common.OtlpCommonProto; -import datadog.trace.core.otlp.common.OtlpPayload; import datadog.trace.core.otlp.common.OtlpProtoBuffer; import java.util.List; diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/OtlpPayloadDispatcherTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/OtlpPayloadDispatcherTest.java index 280e37f0f41..9f9b4dd0ecd 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/OtlpPayloadDispatcherTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/OtlpPayloadDispatcherTest.java @@ -12,11 +12,12 @@ import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; +import datadog.communication.otlp.OtlpPayload; +import datadog.communication.otlp.OtlpResponse; +import datadog.communication.otlp.OtlpSender; import datadog.trace.api.sampling.PrioritySampling; import datadog.trace.api.telemetry.OtlpTelemetry; import datadog.trace.core.CoreSpan; -import datadog.trace.core.otlp.common.OtlpPayload; -import datadog.trace.core.otlp.common.OtlpSender; import datadog.trace.core.otlp.trace.OtlpTraceCollector; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -39,7 +40,7 @@ class OtlpPayloadDispatcherTest { @BeforeEach void stubSuccessfulSend() { - lenient().when(sender.send(any())).thenReturn(RemoteApi.Response.success(200)); + lenient().when(sender.send(any())).thenReturn(OtlpResponse.success(200)); OtlpTelemetry.getInstance().prepareMetrics(); OtlpTelemetry.getInstance().drain(); } @@ -172,7 +173,7 @@ void dispatcherRemainsUsableAfterCollectorFailure() { @Test void flushRecordsSuccessfulExportTelemetry() { - RemoteApi.Response response = RemoteApi.Response.success(200); + OtlpResponse response = OtlpResponse.success(200); when(sender.send(any())).thenReturn(response); OtlpPayloadDispatcher dispatcher = new OtlpPayloadDispatcher(sender, collector); @@ -187,7 +188,7 @@ void flushRecordsSuccessfulExportTelemetry() { @Test void flushRecordsFailedExportTelemetry() { - RemoteApi.Response response = RemoteApi.Response.failed(500); + OtlpResponse response = OtlpResponse.failed(500); when(sender.send(any())).thenReturn(response); OtlpPayloadDispatcher dispatcher = new OtlpPayloadDispatcher(sender, collector); diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/OtlpWriterTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/OtlpWriterTest.java index df86d4384e3..159a45467eb 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/OtlpWriterTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/OtlpWriterTest.java @@ -6,9 +6,9 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.verify; +import datadog.communication.otlp.OtlpGrpcSender; +import datadog.communication.otlp.OtlpSender; import datadog.trace.api.config.OtlpConfig; -import datadog.trace.core.otlp.common.OtlpGrpcSender; -import datadog.trace.core.otlp.common.OtlpSender; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/WriterFactoryTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/WriterFactoryTest.java index 9a7c8bdc946..363c14d32a7 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/WriterFactoryTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/WriterFactoryTest.java @@ -219,10 +219,10 @@ void testWriterCreationForLlmObservability( } @TableTest({ - "scenario | protocol | compression | endpoint | expectedSenderClass | expectedUrl | expectedGzip", - "http no gzip | HTTP_PROTOBUF | NONE | 'http://otel-collector:4318/v1/traces' | datadog.trace.core.otlp.common.OtlpHttpSender | 'http://otel-collector:4318/v1/traces' | false ", - "http gzip | HTTP_PROTOBUF | GZIP | 'http://otel-collector:4318/v1/traces' | datadog.trace.core.otlp.common.OtlpHttpSender | 'http://otel-collector:4318/v1/traces' | true ", - "grpc no gzip | GRPC | NONE | 'http://otel-collector:4317' | datadog.trace.core.otlp.common.OtlpGrpcSender | 'http://otel-collector:4317/opentelemetry.proto.collector.trace.v1.TraceService/Export' | false " + "scenario | protocol | compression | endpoint | expectedSenderClass | expectedUrl | expectedGzip", + "http no gzip | HTTP_PROTOBUF | NONE | 'http://otel-collector:4318/v1/traces' | datadog.communication.otlp.OtlpHttpSender | 'http://otel-collector:4318/v1/traces' | false ", + "http gzip | HTTP_PROTOBUF | GZIP | 'http://otel-collector:4318/v1/traces' | datadog.communication.otlp.OtlpHttpSender | 'http://otel-collector:4318/v1/traces' | true ", + "grpc no gzip | GRPC | NONE | 'http://otel-collector:4317' | datadog.communication.otlp.OtlpGrpcSender | 'http://otel-collector:4317/opentelemetry.proto.collector.trace.v1.TraceService/Export' | false " }) void testWriterCreationForOtlpWriter( OtlpConfig.Protocol protocol, diff --git a/dd-trace-core/src/test/java/datadog/trace/core/otlp/common/OtlpProtoBufferTest.java b/dd-trace-core/src/test/java/datadog/trace/core/otlp/common/OtlpProtoBufferTest.java index 794b36b8ee4..5e9d00705a3 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/otlp/common/OtlpProtoBufferTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/otlp/common/OtlpProtoBufferTest.java @@ -8,6 +8,7 @@ import com.google.protobuf.CodedInputStream; import com.google.protobuf.WireFormat; +import datadog.communication.otlp.OtlpPayload; import datadog.communication.serialization.GrowableBuffer; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/dd-trace-core/src/test/java/datadog/trace/core/otlp/logs/OtlpLogsJsonCollectorTest.java b/dd-trace-core/src/test/java/datadog/trace/core/otlp/logs/OtlpLogsJsonCollectorTest.java index 75c4cef2978..c1a8e7d3e05 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/otlp/logs/OtlpLogsJsonCollectorTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/otlp/logs/OtlpLogsJsonCollectorTest.java @@ -9,6 +9,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import datadog.communication.otlp.OtlpPayload; import datadog.json.JsonMapper; import datadog.trace.api.sampling.PrioritySampling; import datadog.trace.api.sampling.SamplingMechanism; @@ -20,7 +21,6 @@ import datadog.trace.common.writer.LoggingWriter; import datadog.trace.core.CoreTracer; import datadog.trace.core.DDSpan; -import datadog.trace.core.otlp.common.OtlpPayload; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Collections; diff --git a/dd-trace-core/src/test/java/datadog/trace/core/otlp/logs/OtlpLogsProtoTest.java b/dd-trace-core/src/test/java/datadog/trace/core/otlp/logs/OtlpLogsProtoTest.java index 735653921cb..44254fb189e 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/otlp/logs/OtlpLogsProtoTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/otlp/logs/OtlpLogsProtoTest.java @@ -17,6 +17,7 @@ import com.google.protobuf.CodedInputStream; import com.google.protobuf.WireFormat; +import datadog.communication.otlp.OtlpPayload; import datadog.trace.api.DD128bTraceId; import datadog.trace.api.TracePropagationStyle; import datadog.trace.api.sampling.PrioritySampling; @@ -30,7 +31,6 @@ import datadog.trace.common.writer.LoggingWriter; import datadog.trace.core.CoreTracer; import datadog.trace.core.DDSpan; -import datadog.trace.core.otlp.common.OtlpPayload; import datadog.trace.core.propagation.ExtractedContext; import datadog.trace.core.propagation.PropagationTags; import java.io.IOException; diff --git a/dd-trace-core/src/test/java/datadog/trace/core/otlp/logs/OtlpLogsServiceTest.java b/dd-trace-core/src/test/java/datadog/trace/core/otlp/logs/OtlpLogsServiceTest.java index 0954e0e56d2..d03712a7a6a 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/otlp/logs/OtlpLogsServiceTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/otlp/logs/OtlpLogsServiceTest.java @@ -5,8 +5,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import datadog.communication.otlp.OtlpHttpSender; import datadog.trace.api.Config; -import datadog.trace.core.otlp.common.OtlpHttpSender; import java.util.Properties; import org.junit.jupiter.api.Test; diff --git a/dd-trace-core/src/test/java/datadog/trace/core/otlp/metrics/OtlpMetricsJsonCollectorTest.java b/dd-trace-core/src/test/java/datadog/trace/core/otlp/metrics/OtlpMetricsJsonCollectorTest.java index a3cbf6a0637..b55e83c1ce2 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/otlp/metrics/OtlpMetricsJsonCollectorTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/otlp/metrics/OtlpMetricsJsonCollectorTest.java @@ -8,6 +8,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import datadog.communication.otlp.OtlpPayload; import datadog.json.JsonMapper; import datadog.trace.api.time.ControllableTimeSource; import datadog.trace.bootstrap.otel.common.OtelInstrumentationScope; @@ -18,7 +19,6 @@ import datadog.trace.bootstrap.otlp.metrics.OtlpMetricVisitor; import datadog.trace.bootstrap.otlp.metrics.OtlpMetricsVisitor; import datadog.trace.bootstrap.otlp.metrics.OtlpScopedMetricsVisitor; -import datadog.trace.core.otlp.common.OtlpPayload; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Arrays; diff --git a/dd-trace-core/src/test/java/datadog/trace/core/otlp/metrics/OtlpMetricsProtoTest.java b/dd-trace-core/src/test/java/datadog/trace/core/otlp/metrics/OtlpMetricsProtoTest.java index 8cc2ac4ea81..0bd5babd969 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/otlp/metrics/OtlpMetricsProtoTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/otlp/metrics/OtlpMetricsProtoTest.java @@ -20,6 +20,7 @@ import com.google.protobuf.CodedInputStream; import com.google.protobuf.WireFormat; +import datadog.communication.otlp.OtlpPayload; import datadog.trace.api.time.ControllableTimeSource; import datadog.trace.bootstrap.otel.common.OtelInstrumentationScope; import datadog.trace.bootstrap.otel.metrics.OtelInstrumentDescriptor; @@ -30,7 +31,6 @@ import datadog.trace.bootstrap.otlp.metrics.OtlpLongPoint; import datadog.trace.bootstrap.otlp.metrics.OtlpMetricVisitor; import datadog.trace.bootstrap.otlp.metrics.OtlpScopedMetricsVisitor; -import datadog.trace.core.otlp.common.OtlpPayload; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; diff --git a/dd-trace-core/src/test/java/datadog/trace/core/otlp/metrics/OtlpMetricsServiceTest.java b/dd-trace-core/src/test/java/datadog/trace/core/otlp/metrics/OtlpMetricsServiceTest.java index 70e719a3dbb..60a0575f7ad 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/otlp/metrics/OtlpMetricsServiceTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/otlp/metrics/OtlpMetricsServiceTest.java @@ -5,8 +5,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import datadog.communication.otlp.OtlpHttpSender; import datadog.trace.api.Config; -import datadog.trace.core.otlp.common.OtlpHttpSender; import java.util.Properties; import org.junit.jupiter.api.Test; diff --git a/dd-trace-core/src/test/java/datadog/trace/core/otlp/metrics/OtlpStatsMetricWriterTest.java b/dd-trace-core/src/test/java/datadog/trace/core/otlp/metrics/OtlpStatsMetricWriterTest.java index e20f8713b84..01658ad92e6 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/otlp/metrics/OtlpStatsMetricWriterTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/otlp/metrics/OtlpStatsMetricWriterTest.java @@ -1,6 +1,6 @@ package datadog.trace.core.otlp.metrics; -import static datadog.trace.common.writer.RemoteApi.Response.success; +import static datadog.communication.otlp.OtlpResponse.success; import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -11,14 +11,14 @@ import com.google.protobuf.CodedInputStream; import com.google.protobuf.WireFormat; +import datadog.communication.otlp.OtlpPayload; +import datadog.communication.otlp.OtlpResponse; +import datadog.communication.otlp.OtlpSender; import datadog.metrics.api.Histograms; import datadog.metrics.impl.DDSketchHistograms; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.common.metrics.AggregateEntry; import datadog.trace.common.metrics.AggregateEntryTestUtils; -import datadog.trace.common.writer.RemoteApi; -import datadog.trace.core.otlp.common.OtlpPayload; -import datadog.trace.core.otlp.common.OtlpSender; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -68,7 +68,7 @@ private static final class CapturingSender implements OtlpSender { byte[] lastPayload; @Override - public RemoteApi.Response send(OtlpPayload payload) { + public OtlpResponse send(OtlpPayload payload) { sendCount++; java.nio.ByteBuffer content = payload.getContent(); byte[] bytes = new byte[content.remaining()]; diff --git a/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpTraceJsonCollectorTest.java b/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpTraceJsonCollectorTest.java index 3f836227a39..bf3406eb539 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpTraceJsonCollectorTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpTraceJsonCollectorTest.java @@ -15,6 +15,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import datadog.communication.otlp.OtlpPayload; import datadog.json.JsonMapper; import datadog.trace.api.DDTraceId; import datadog.trace.api.TracePropagationStyle; @@ -27,7 +28,6 @@ import datadog.trace.core.CoreSpan; import datadog.trace.core.CoreTracer; import datadog.trace.core.DDSpan; -import datadog.trace.core.otlp.common.OtlpPayload; import datadog.trace.core.propagation.ExtractedContext; import datadog.trace.core.propagation.PropagationTags; import java.io.IOException; diff --git a/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpTraceProtoTest.java b/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpTraceProtoTest.java index e326a6958b8..880f11c7749 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpTraceProtoTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpTraceProtoTest.java @@ -24,6 +24,7 @@ import com.google.protobuf.CodedInputStream; import com.google.protobuf.WireFormat; +import datadog.communication.otlp.OtlpPayload; import datadog.trace.api.DD128bTraceId; import datadog.trace.api.DDTraceId; import datadog.trace.api.TracePropagationStyle; @@ -36,7 +37,6 @@ import datadog.trace.core.CoreSpan; import datadog.trace.core.CoreTracer; import datadog.trace.core.DDSpan; -import datadog.trace.core.otlp.common.OtlpPayload; import datadog.trace.core.propagation.ExtractedContext; import datadog.trace.core.propagation.PropagationTags; import java.io.IOException; diff --git a/docker/Dockerfile.profcheck b/docker/Dockerfile.profcheck new file mode 100644 index 00000000000..1187c646e56 --- /dev/null +++ b/docker/Dockerfile.profcheck @@ -0,0 +1,118 @@ +# Dockerfile for building and running OpenTelemetry profcheck tool +# Usage: +# docker build -f Dockerfile.profcheck -t profcheck . +# docker run --rm -v $(pwd):/data profcheck /data/output.pb + +FROM golang:1.25-alpine AS builder + +# Install git +RUN apk add --no-cache git + +# Clone the sig-profiling repo and checkout a pinned main commit +ARG SIG_PROFILING_COMMIT=0dd094852c0b8145a3cdc625066d68315336a16b +WORKDIR /build +RUN git clone https://github.com/open-telemetry/sig-profiling.git && \ + cd sig-profiling && \ + git checkout ${SIG_PROFILING_COMMIT} + +# Build profcheck +WORKDIR /build/sig-profiling/profcheck +RUN go build -o /profcheck ./cmd/profcheck + +# Create runtime image with protoc for validation +FROM alpine:3.24.1 + +# Install protoc and required dependencies +RUN apk add --no-cache protobuf protobuf-dev git + +# Clone OTLP proto definitions (v1.10.0) for protoc validation; keep in sync with OtlpProtoFields +ARG OTLP_PROTO_COMMIT=ca839c51f706f5d53bfb46f06c3e90c3af3a52c6 +WORKDIR /proto +RUN git init opentelemetry-proto && \ + cd opentelemetry-proto && \ + git fetch --depth=1 https://github.com/open-telemetry/opentelemetry-proto.git ${OTLP_PROTO_COMMIT} && \ + git checkout FETCH_HEAD && \ + # Get the commit hash for reference + git rev-parse HEAD > /proto/commit-hash.txt + +# Copy profcheck binary +COPY --from=builder /profcheck /usr/local/bin/profcheck + +# Create validation wrapper script +RUN cat > /usr/local/bin/validate-profile << 'EOF' +#!/bin/sh +set -e + +FILE="$1" +if [ -z "$FILE" ]; then + echo "Usage: validate-profile " + exit 1 +fi + +if [ ! -f "$FILE" ]; then + echo "Error: File not found: $FILE" + exit 1 +fi + +echo "================================" +echo "OTLP Profile Validation" +echo "================================" +echo "" + +# Run protoc validation (this is the authoritative check) +echo "[1/2] Validating with protoc (official Protocol Buffers compiler)..." +cd /proto/opentelemetry-proto +if protoc --decode=opentelemetry.proto.profiles.v1development.ProfilesData \ + --proto_path=. \ + opentelemetry/proto/profiles/v1development/profiles.proto \ + < "$FILE" > /tmp/decoded.txt 2>&1; then + echo "✓ protoc validation PASSED - profile is spec-compliant" + PROTOC_STATUS=0 +else + echo "✗ protoc validation FAILED - profile has structural errors" + cat /tmp/decoded.txt + PROTOC_STATUS=1 +fi + +echo "" +echo "[2/2] Running profcheck (OpenTelemetry conformance checker)..." +echo "" + +# POSIX sh pipelines report the LAST command's status (tee), masking profcheck's exit code — +# redirect to a file and capture the status directly; || keeps set -e from tripping on failure. +PROFCHECK_STATUS=0 +profcheck "$FILE" > /tmp/profcheck.txt 2>&1 || PROFCHECK_STATUS=$? +if [ $PROFCHECK_STATUS -eq 0 ]; then + echo "✓ profcheck validation PASSED" +else + echo "✗ profcheck validation FAILED" + echo "" + echo "Full profcheck output:" + echo "--- BEGIN PROFCHECK OUTPUT ---" + cat /tmp/profcheck.txt + echo "--- END PROFCHECK OUTPUT ---" +fi + +echo "" +echo "================================" +echo "Validation Summary" +echo "================================" +echo "protoc: $([ $PROTOC_STATUS -eq 0 ] && echo 'PASS ✓' || echo 'FAIL ✗')" +echo "profcheck: $([ $PROFCHECK_STATUS -eq 0 ] && echo 'PASS ✓' || echo 'FAIL ✗')" +echo "" + +# Both validators must pass +if [ $PROTOC_STATUS -eq 0 ] && [ $PROFCHECK_STATUS -eq 0 ]; then + exit 0 +else + exit 1 +fi +EOF + +RUN chmod +x /usr/local/bin/validate-profile + +# Set working directory +WORKDIR /data + +# Set entrypoint to validation wrapper +ENTRYPOINT ["/usr/local/bin/validate-profile"] diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index f781cbc134a..8e51846d6e2 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -20,6 +20,7 @@ final class CachedData { filter.exclude(filter.project(':internal-api')) filter.exclude(filter.project(':internal-api:internal-api-9')) filter.exclude(filter.project(':communication')) + filter.exclude(filter.project(':communication:otlp-exporter')) filter.exclude(filter.project(':components:context')) filter.exclude(filter.project(':components:environment')) filter.exclude(filter.project(':components:json')) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 27ecd8e76a7..5da331f332e 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -50,7 +50,7 @@ jmh = "1.37" # Profiling jmc = "8.1.0" -jafar = "0.16.0" +jafar = "0.26.2" jol = "0.17" # Web & Network @@ -136,6 +136,8 @@ jmc-common = { module = "org.openjdk.jmc:common", version.ref = "jmc" } jmc-flightrecorder = { module = "org.openjdk.jmc:flightrecorder", version.ref = "jmc" } jafar-tools = { module = "io.btrace:jafar-tools", version.ref = "jafar" } jol-core = { module = "org.openjdk.jol:jol-core", version.ref = "jol" } +jmc-flightrecorder-writer = { module = "org.openjdk.jmc:flightrecorder.writer", version.ref = "jmc" } +jafar-parser = { module = "io.btrace:jafar-parser", version.ref = "jafar" } # Web & Network jnr-unixsocket = { module = "com.github.jnr:jnr-unixsocket", version.ref = "jnr-unixsocket"} diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 2db203dcbd4..0628d8e6265 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -133,7 +133,9 @@ import static datadog.trace.api.ConfigDefaults.DEFAULT_OTLP_HTTP_LOGS_ENDPOINT; import static datadog.trace.api.ConfigDefaults.DEFAULT_OTLP_HTTP_METRICS_ENDPOINT; import static datadog.trace.api.ConfigDefaults.DEFAULT_OTLP_HTTP_PORT; +import static datadog.trace.api.ConfigDefaults.DEFAULT_OTLP_HTTP_PROFILES_ENDPOINT; import static datadog.trace.api.ConfigDefaults.DEFAULT_OTLP_HTTP_TRACES_ENDPOINT; +import static datadog.trace.api.ConfigDefaults.DEFAULT_OTLP_PROFILES_TIMEOUT; import static datadog.trace.api.ConfigDefaults.DEFAULT_OTLP_TRACES_TIMEOUT; import static datadog.trace.api.ConfigDefaults.DEFAULT_PARTIAL_FLUSH_MIN_SPANS; import static datadog.trace.api.ConfigDefaults.DEFAULT_PERF_METRICS_ENABLED; @@ -508,6 +510,11 @@ import static datadog.trace.api.config.OtlpConfig.OTLP_METRICS_PROTOCOL; import static datadog.trace.api.config.OtlpConfig.OTLP_METRICS_TEMPORALITY_PREFERENCE; import static datadog.trace.api.config.OtlpConfig.OTLP_METRICS_TIMEOUT; +import static datadog.trace.api.config.OtlpConfig.OTLP_PROFILES_COMPRESSION; +import static datadog.trace.api.config.OtlpConfig.OTLP_PROFILES_ENDPOINT; +import static datadog.trace.api.config.OtlpConfig.OTLP_PROFILES_HEADERS; +import static datadog.trace.api.config.OtlpConfig.OTLP_PROFILES_PROTOCOL; +import static datadog.trace.api.config.OtlpConfig.OTLP_PROFILES_TIMEOUT; import static datadog.trace.api.config.OtlpConfig.OTLP_TRACES_COMPRESSION; import static datadog.trace.api.config.OtlpConfig.OTLP_TRACES_ENDPOINT; import static datadog.trace.api.config.OtlpConfig.OTLP_TRACES_HEADERS; @@ -1054,6 +1061,12 @@ public static String getHostName() { private final OtlpConfig.Compression otlpTracesCompression; private final int otlpTracesTimeout; + private final String otlpProfilesEndpoint; + private final Map otlpProfilesHeaders; + private final OtlpConfig.Protocol otlpProfilesProtocol; + private final OtlpConfig.Compression otlpProfilesCompression; + private final int otlpProfilesTimeout; + // These values are default-ed to those of jmx fetch values as needed private final boolean healthMetricsEnabled; private final String healthMetricsStatsdHost; @@ -2258,6 +2271,40 @@ && isMetricsOtelEnabled() } otlpTracesEndpoint = otlpTracesEndpointFromEnvironment; + // OTLP Profiles config — mirrors the traces/logs/metrics pattern + int profilesTimeout = + configProvider.getInteger(OTLP_PROFILES_TIMEOUT, DEFAULT_OTLP_PROFILES_TIMEOUT); + if (profilesTimeout < 0) { + log.warn( + "Invalid OTLP profiles timeout: {}. The value must not be negative", profilesTimeout); + profilesTimeout = DEFAULT_OTLP_PROFILES_TIMEOUT; + } + otlpProfilesTimeout = profilesTimeout; + + otlpProfilesHeaders = configProvider.getMergedMap(OTLP_PROFILES_HEADERS, '='); + otlpProfilesProtocol = + configProvider.getEnum( + OTLP_PROFILES_PROTOCOL, OtlpConfig.Protocol.class, OtlpConfig.Protocol.HTTP_PROTOBUF); + otlpProfilesCompression = + configProvider.getEnum( + OTLP_PROFILES_COMPRESSION, OtlpConfig.Compression.class, OtlpConfig.Compression.NONE); + + String otlpProfilesEndpointFromEnvironment = configProvider.getString(OTLP_PROFILES_ENDPOINT); + if (otlpProfilesEndpointFromEnvironment == null) { + if (otlpProfilesProtocol == OtlpConfig.Protocol.GRPC) { + otlpProfilesEndpointFromEnvironment = "http://" + agentHost + ':' + DEFAULT_OTLP_GRPC_PORT; + } else { + otlpProfilesEndpointFromEnvironment = + "http://" + + agentHost + + ':' + + DEFAULT_OTLP_HTTP_PORT + + '/' + + DEFAULT_OTLP_HTTP_PROFILES_ENDPOINT; + } + } + otlpProfilesEndpoint = otlpProfilesEndpointFromEnvironment; + // Runtime metrics are disabled if Otel metrics are enabled and the metrics exporter is none runtimeMetricsEnabled = configProvider.getBoolean(RUNTIME_METRICS_ENABLED, true); @@ -5989,6 +6036,26 @@ public int getOtlpTracesTimeout() { return otlpTracesTimeout; } + public String getOtlpProfilesEndpoint() { + return otlpProfilesEndpoint; + } + + public Map getOtlpProfilesHeaders() { + return otlpProfilesHeaders; + } + + public OtlpConfig.Protocol getOtlpProfilesProtocol() { + return otlpProfilesProtocol; + } + + public OtlpConfig.Compression getOtlpProfilesCompression() { + return otlpProfilesCompression; + } + + public int getOtlpProfilesTimeout() { + return otlpProfilesTimeout; + } + public boolean isRuleEnabled(final String name) { return isRuleEnabled(name, true); } diff --git a/internal-api/src/main/java/datadog/trace/api/profiling/RecordingData.java b/internal-api/src/main/java/datadog/trace/api/profiling/RecordingData.java index d62f1541c3e..9ab74066593 100644 --- a/internal-api/src/main/java/datadog/trace/api/profiling/RecordingData.java +++ b/internal-api/src/main/java/datadog/trace/api/profiling/RecordingData.java @@ -21,12 +21,29 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; -/** Platform-agnostic API for operations required when retrieving data using the ProfilingSystem. */ +/** + * Platform-agnostic API for operations required when retrieving data using the ProfilingSystem. + * + *

Subclassing note: {@link #release()} is final and reference-counted; subclasses implement + * resource cleanup by overriding {@link #doRelease()}, invoked exactly once when the reference + * count reaches zero. + */ public abstract class RecordingData implements ProfilingSnapshot { protected final Instant start; protected final Instant end; protected final Kind kind; + // Reference counting for multiple listeners. Starts at 1 (the base reference owned by the + // first listener in the chain). Additional listeners call retain() before use and release() + // when done. doRelease() fires when the count reaches 0. + // Plain fields suffice: both are only ever accessed under the private lock below. + private int refCount = 1; + private boolean released = false; + + // Private lock so the intrinsic lock of this publicly exposed class is not used for + // synchronization (SpotBugs USO_UNSAFE_METHOD_SYNCHRONIZATION / CERT LCK00-J). + private final Object lock = new Object(); + public RecordingData(final Instant start, final Instant end, Kind kind) { this.start = start; this.end = end; @@ -40,10 +57,36 @@ public RecordingData(final Instant start, final Instant end, Kind kind) { @Nonnull public abstract RecordingInputStream getStream() throws IOException; + /** + * Increment reference count. Must be called once for each *additional* handler that will process + * this RecordingData beyond the base reference. + * + *

The reference count starts at 1, representing the base reference that the primary listener + * releases via {@code release()}. Each additional handler must call {@code retain()} before + * processing and {@code release()} when done. + * + * @return this instance for chaining + * @throws IllegalStateException if the recording has already been released + */ + @Nonnull + public final RecordingData retain() { + synchronized (lock) { + if (released) { + throw new IllegalStateException("Cannot retain released RecordingData"); + } + refCount++; + return this; + } + } + /** * Releases the resources associated with the recording, for example the underlying file. * - *

Forgetting to releasing this when done streaming, will lead to one or more of the following: + *

This method uses reference counting to support multiple handlers. The base reference count + * is 1; each call to {@link #retain()} must be matched with a call to {@code release()}. The + * actual resource cleanup via {@link #doRelease()} happens when the reference count reaches zero. + * + *

Forgetting to release this when done streaming will lead to one or more of the following: * *

    *
  • Memory leak @@ -51,8 +94,37 @@ public RecordingData(final Instant start, final Instant end, Kind kind) { *
* *

Please don't forget to call release when done streaming... + * + *

Releasing an already fully-released recording is a silent no-op — the {@code released} flag + * guards against double release. Retaining after full release, in contrast, throws {@link + * IllegalStateException}. + */ + public final void release() { + boolean shouldRelease = false; + synchronized (lock) { + if (released) { + // retain() refuses to hand out new references once released, so the count can never + // reach zero a second time + return; + } + int remaining = --refCount; + if (remaining == 0) { + released = true; + shouldRelease = true; + } + } + if (shouldRelease) { + doRelease(); + } + } + + /** + * Actual resource cleanup implementation. Subclasses must override this method instead of {@link + * #release()}. + * + *

This method is called exactly once when the reference count reaches zero. */ - public abstract void release(); + protected abstract void doRelease(); /** * Returns the name of the recording from which the data is originating. diff --git a/internal-api/src/main/java/datadog/trace/api/telemetry/OtlpTelemetry.java b/internal-api/src/main/java/datadog/trace/api/telemetry/OtlpTelemetry.java index a36bcff67c8..3323a10a4d0 100644 --- a/internal-api/src/main/java/datadog/trace/api/telemetry/OtlpTelemetry.java +++ b/internal-api/src/main/java/datadog/trace/api/telemetry/OtlpTelemetry.java @@ -8,11 +8,15 @@ import java.util.List; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.atomic.LongAccumulator; import java.util.concurrent.atomic.LongAdder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** Collects telemetry metrics for the OTLP trace, metrics, and log exporters. */ public class OtlpTelemetry implements MetricCollector { private static final String NAMESPACE = "tracers"; + private static final Logger log = LoggerFactory.getLogger(OtlpTelemetry.class); private static final OtlpTelemetry INSTANCE = new OtlpTelemetry(); @@ -24,6 +28,16 @@ public static OtlpTelemetry getInstance() { private final String[] metricsTags = tagsFor(Config.get().getOtlpMetricsProtocol()); private final String[] logsTags = tagsFor(Config.get().getOtlpLogsProtocol()); + private final String[] profilesTags = tagsFor(Config.get().getOtlpProfilesProtocol()); + private final ExportCounters profilesExport = new ExportCounters("profiles"); + + // JFR→OTLP conversion timings, reported as avg/max gauges per flush window + private final LongAdder profilesConversionNanosTotal = new LongAdder(); + private final LongAdder profilesConversionCount = new LongAdder(); + // guards the (nanosTotal, count) pair so window snapshots are consistent + private final Object conversionMetricsLock = new Object(); + private final LongAccumulator profilesConversionNanosMax = new LongAccumulator(Long::max, 0); + private final ExportCounters tracesExport = new ExportCounters("traces"); private final ExportCounters metricsExport = new ExportCounters("metrics"); private final LongAdder logRecords = new LongAdder(); @@ -44,10 +58,29 @@ public void onMetricsExportAttempt() { metricsExport.attempts.increment(); } + public void onProfilesExportAttempt() { + profilesExport.attempts.increment(); + } + public void onMetricsExportComplete(boolean success) { metricsExport.complete(success); } + public void onProfilesExportComplete(boolean success) { + profilesExport.complete(success); + } + + /** Records the duration of a single JFR→OTLP profile conversion. */ + public void onProfilesConversion(long nanos) { + // (total, count, max) must stay a consistent window snapshot; the conversion metrics lock + // also guards the snapshot+reset in prepareMetrics() + synchronized (conversionMetricsLock) { + profilesConversionNanosTotal.add(nanos); + profilesConversionCount.increment(); + profilesConversionNanosMax.accumulate(nanos); + } + } + public void onLogRecordsSubmitted(long count) { if (count > 0) { logRecords.add(count); @@ -64,12 +97,43 @@ private static String[] tagsFor(OtlpConfig.Protocol protocol) { public void prepareMetrics() { tracesExport.stageInto(telemetryQueue, tracesTags); metricsExport.stageInto(telemetryQueue, metricsTags); + profilesExport.stageInto(telemetryQueue, profilesTags); + long conversionCount; + long conversionNanosTotal; + long maxNanos; + synchronized (conversionMetricsLock) { + // snapshot+reset (count, total, max) atomically so each flush window reports exactly the + // conversions that completed within it + conversionCount = profilesConversionCount.sumThenReset(); + conversionNanosTotal = profilesConversionNanosTotal.sumThenReset(); + maxNanos = profilesConversionNanosMax.getThenReset(); + } + if (conversionCount > 0) { + double avgNanos = conversionNanosTotal / (double) conversionCount; + offerMetric( + new OtlpMetric( + "otel.profiles_conversion_ms", "gauge", nanosToMillis(avgNanos), profilesTags)); + offerMetric( + new OtlpMetric( + "otel.profiles_conversion_max_ms", "gauge", nanosToMillis(maxNanos), profilesTags)); + } long logRecordCount = logRecords.sumThenReset(); if (logRecordCount > 0) { - telemetryQueue.offer(new OtlpMetric("otel.log_records", logRecordCount, logsTags)); + offerMetric(new OtlpMetric("otel.log_records", logRecordCount, logsTags)); + } + } + + // the queue is bounded and shared by all OTLP surfaces; make a rejected offer visible + private void offerMetric(OtlpMetric metric) { + if (!telemetryQueue.offer(metric)) { + log.debug("Telemetry queue full, dropping metric {}", metric.metricName); } } + private static double nanosToMillis(double nanos) { + return nanos / 1_000_000.0; + } + @Override public Collection drain() { if (telemetryQueue.isEmpty()) { @@ -109,15 +173,19 @@ void stageInto(BlockingQueue out, String[] tags) { private static void addIfNonZero( BlockingQueue out, String metricName, LongAdder counter, String[] tags) { long value = counter.sumThenReset(); - if (value > 0) { - out.offer(new OtlpMetric(metricName, value, tags)); + if (value > 0 && !out.offer(new OtlpMetric(metricName, value, tags))) { + log.debug("Telemetry queue full, dropping metric {}", metricName); } } } public static class OtlpMetric extends MetricCollector.Metric { public OtlpMetric(String metricName, long value, String... tags) { - super(NAMESPACE, true, metricName, "count", value, tags); + this(metricName, "count", value, tags); + } + + OtlpMetric(String metricName, String type, Number value, String... tags) { + super(NAMESPACE, true, metricName, type, value, tags); } } } diff --git a/internal-api/src/test/java/datadog/trace/api/telemetry/OtlpTelemetryTest.java b/internal-api/src/test/java/datadog/trace/api/telemetry/OtlpTelemetryTest.java index 53877bd8dc6..18888b70cdb 100644 --- a/internal-api/src/test/java/datadog/trace/api/telemetry/OtlpTelemetryTest.java +++ b/internal-api/src/test/java/datadog/trace/api/telemetry/OtlpTelemetryTest.java @@ -1,6 +1,7 @@ package datadog.trace.api.telemetry; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Collection; @@ -86,6 +87,46 @@ void metricsExportMetricsUseExpectedNames() { assertEquals(1L, valuesByName.get("otel.metrics_export_failures")); } + @Test + void onProfilesConversionQueuesAvgAndMaxGauges() { + collector.onProfilesConversion(50_000_000L); // 50 ms + collector.onProfilesConversion(150_000_000L); // 150 ms + collector.prepareMetrics(); + + Collection metrics = collector.drain(); + + Map gaugesByName = new HashMap<>(); + for (OtlpTelemetry.OtlpMetric metric : metrics) { + gaugesByName.put(metric.metricName, metric); + } + assertEquals(2, gaugesByName.size()); + + OtlpTelemetry.OtlpMetric avg = gaugesByName.get("otel.profiles_conversion_ms"); + assertEquals("gauge", avg.type); + assertEquals(100.0, avg.value.doubleValue(), 0.001); + assertTrue(avg.tags.contains("protocol:http")); + + OtlpTelemetry.OtlpMetric max = gaugesByName.get("otel.profiles_conversion_max_ms"); + assertEquals("gauge", max.type); + assertEquals(150.0, max.value.doubleValue(), 0.001); + assertTrue(max.tags.contains("protocol:http")); + } + + @Test + void onProfilesConversionOmittedWhenNoConversions() { + // stage a different metric so a non-empty drain proves the gauges were specifically + // omitted rather than nothing being emitted at all + collector.onTracesExportAttempt(); + collector.prepareMetrics(); + + Collection metrics = collector.drain(); + + assertFalse(metrics.isEmpty(), "expected the traces metric to be staged"); + assertTrue( + metrics.stream().noneMatch(m -> m.metricName.startsWith("otel.profiles_conversion")), + "no profiles conversion metrics expected without conversions"); + } + @Test void onLogRecordsSubmittedQueuesCountWithGivenValue() { collector.onLogRecordsSubmitted(5); diff --git a/internal-api/src/test/java/datadog/trace/bootstrap/config/provider/OtelProfilesEnvironmentConfigSourceTest.java b/internal-api/src/test/java/datadog/trace/bootstrap/config/provider/OtelProfilesEnvironmentConfigSourceTest.java new file mode 100644 index 00000000000..93afff7978d --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/bootstrap/config/provider/OtelProfilesEnvironmentConfigSourceTest.java @@ -0,0 +1,116 @@ +package datadog.trace.bootstrap.config.provider; + +import static datadog.trace.api.config.GeneralConfig.RUNTIME_METRICS_ENABLED; +import static datadog.trace.api.config.GeneralConfig.SERVICE_NAME; +import static datadog.trace.api.config.OtlpConfig.OTLP_PROFILES_COMPRESSION; +import static datadog.trace.api.config.OtlpConfig.OTLP_PROFILES_ENDPOINT; +import static datadog.trace.api.config.OtlpConfig.OTLP_PROFILES_HEADERS; +import static datadog.trace.api.config.OtlpConfig.OTLP_PROFILES_PROTOCOL; +import static datadog.trace.api.config.OtlpConfig.OTLP_PROFILES_TIMEOUT; +import static datadog.trace.test.junit.utils.config.WithConfigExtension.injectEnvConfig; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import datadog.trace.test.junit.utils.config.WithConfigExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(WithConfigExtension.class) +class OtelProfilesEnvironmentConfigSourceTest { + + @Test + void profilesKeysAreNotMappedWhenOtlpProfilingIsDisabled() { + injectEnvConfig("OTEL_EXPORTER_OTLP_PROFILES_ENDPOINT", "http://collector:4318/p", false); + + OtelEnvironmentConfigSource source = new OtelEnvironmentConfigSource(); + + assertNull(source.get(OTLP_PROFILES_ENDPOINT)); + } + + @Test + void signalSpecificProfilesKeysAreMapped() { + injectEnvConfig("DD_PROFILING_OTLP_ENABLED", "true", false); + injectEnvConfig("OTEL_EXPORTER_OTLP_PROFILES_ENDPOINT", "http://collector:4318/p", false); + injectEnvConfig("OTEL_EXPORTER_OTLP_PROFILES_HEADERS", "api-key=secret", false); + injectEnvConfig("OTEL_EXPORTER_OTLP_PROFILES_PROTOCOL", "http/protobuf", false); + injectEnvConfig("OTEL_EXPORTER_OTLP_PROFILES_COMPRESSION", "gzip", false); + injectEnvConfig("OTEL_EXPORTER_OTLP_PROFILES_TIMEOUT", "5000", false); + + OtelEnvironmentConfigSource source = new OtelEnvironmentConfigSource(); + + assertEquals("http://collector:4318/p", source.get(OTLP_PROFILES_ENDPOINT)); + assertEquals("api-key=secret", source.get(OTLP_PROFILES_HEADERS)); + assertEquals("http/protobuf", source.get(OTLP_PROFILES_PROTOCOL)); + assertEquals("gzip", source.get(OTLP_PROFILES_COMPRESSION)); + assertEquals("5000", source.get(OTLP_PROFILES_TIMEOUT)); + } + + @Test + void generalEndpointGetsProfilesPathAppended() { + injectEnvConfig("DD_PROFILING_OTLP_ENABLED", "true", false); + injectEnvConfig("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector:4318", false); + injectEnvConfig("OTEL_EXPORTER_OTLP_HEADERS", "api-key=secret", false); + + OtelEnvironmentConfigSource source = new OtelEnvironmentConfigSource(); + + assertEquals( + "http://collector:4318/v1development/profiles", source.get(OTLP_PROFILES_ENDPOINT)); + assertEquals("api-key=secret", source.get(OTLP_PROFILES_HEADERS)); + } + + @Test + void generalEndpointIsKeptAsIsForGrpc() { + injectEnvConfig("DD_PROFILING_OTLP_ENABLED", "true", false); + injectEnvConfig("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc", false); + injectEnvConfig("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector:4317", false); + + OtelEnvironmentConfigSource source = new OtelEnvironmentConfigSource(); + + assertEquals("http://collector:4317", source.get(OTLP_PROFILES_ENDPOINT)); + } + + @Test + void generalEndpointIsKeptAsIsForDatadogGrpcProtocol() { + injectEnvConfig("DD_PROFILING_OTLP_ENABLED", "true", false); + injectEnvConfig("DD_OTLP_PROFILES_PROTOCOL", "grpc", false); + injectEnvConfig("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector:4317", false); + + OtelEnvironmentConfigSource source = new OtelEnvironmentConfigSource(); + + assertEquals("http://collector:4317", source.get(OTLP_PROFILES_ENDPOINT)); + } + + @Test + void otlpProfilingDoesNotMapGeneralOtelEnvironment() { + injectEnvConfig("DD_PROFILING_OTLP_ENABLED", "true", false); + injectEnvConfig("OTEL_SERVICE_NAME", "otel-service", false); + injectEnvConfig("OTEL_METRICS_EXPORTER", "none", false); + + OtelEnvironmentConfigSource source = new OtelEnvironmentConfigSource(); + + assertNull(source.get(SERVICE_NAME)); + assertNull(source.get(RUNTIME_METRICS_ENABLED)); + } + + @Test + void profilesKeysAreMappedWhenOtelSdkIsDisabled() { + injectEnvConfig("DD_PROFILING_OTLP_ENABLED", "true", false); + injectEnvConfig("OTEL_SDK_DISABLED", "true", false); + injectEnvConfig("OTEL_EXPORTER_OTLP_PROFILES_ENDPOINT", "http://collector:4318/p", false); + + OtelEnvironmentConfigSource source = new OtelEnvironmentConfigSource(); + + assertEquals("http://collector:4318/p", source.get(OTLP_PROFILES_ENDPOINT)); + } + + @Test + void datadogSettingWinsOverOtelSetting() { + injectEnvConfig("DD_PROFILING_OTLP_ENABLED", "true", false); + injectEnvConfig("DD_OTLP_PROFILES_ENDPOINT", "http://dd:4318/p", false); + injectEnvConfig("OTEL_EXPORTER_OTLP_PROFILES_ENDPOINT", "http://collector:4318/p", false); + + OtelEnvironmentConfigSource source = new OtelEnvironmentConfigSource(); + + assertNull(source.get(OTLP_PROFILES_ENDPOINT)); + } +} diff --git a/internal-api/src/test/java/datadog/trace/util/TempLocationManagerTest.java b/internal-api/src/test/java/datadog/trace/util/TempLocationManagerTest.java index af22b5fe804..a0ee4c65fcc 100644 --- a/internal-api/src/test/java/datadog/trace/util/TempLocationManagerTest.java +++ b/internal-api/src/test/java/datadog/trace/util/TempLocationManagerTest.java @@ -159,9 +159,9 @@ private void syncPoint(boolean timeout) { withTimeout.compareAndSet(false, timeout); hookReached.countDown(); try { - if (!proceedSignal.await(30, TimeUnit.SECONDS)) { + if (!proceedSignal.await(60, TimeUnit.SECONDS)) { cleanupError.set( - new AssertionError("Cleanup thread: proceed signal timeout after 30s")); + new AssertionError("Cleanup thread: proceed signal timeout after 60s")); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -207,13 +207,13 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc, boolean tim try { assertTrue( - hookReached.await(30, TimeUnit.SECONDS), - "Cleanup thread should reach hook within 30 seconds"); + hookReached.await(60, TimeUnit.SECONDS), + "Cleanup thread should reach hook within 60 seconds"); // The visitor may have already deleted the file by this point; tolerate. Files.deleteIfExists(fakeTempFile); proceedSignal.countDown(); - cleanupThread.join(TimeUnit.SECONDS.toMillis(30)); - assertFalse(cleanupThread.isAlive(), "Cleanup should complete within 30s"); + cleanupThread.join(TimeUnit.SECONDS.toMillis(60)); + assertFalse(cleanupThread.isAlive(), "Cleanup should complete within 60s"); Throwable error = cleanupError.get(); if (error != null) { diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 6bc49c2679c..2ba88d54792 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -12070,6 +12070,47 @@ "aliases": [] } ], + "OTEL_EXPORTER_OTLP_PROFILES_ENDPOINT": [ + { + "version": "A", + "type": "string", + "default": null, + "aliases": [] + } + ], + "OTEL_EXPORTER_OTLP_PROFILES_HEADERS": [ + { + "version": "A", + "type": "string", + "default": null, + "aliases": [], + "sensitive": true + } + ], + "OTEL_EXPORTER_OTLP_PROFILES_PROTOCOL": [ + { + "version": "A", + "type": "string", + "default": "http/protobuf", + "aliases": [] + } + ], + "OTEL_EXPORTER_OTLP_PROFILES_COMPRESSION": [ + { + "version": "B", + "type": "string", + "default": "none", + "aliases": [] + } + ], + "OTEL_EXPORTER_OTLP_PROFILES_TIMEOUT": [ + { + "version": "A", + "type": "int", + "default": "30000", + "aliases": [] + } + ], "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": [ { "version": "A", @@ -12191,6 +12232,63 @@ "default": null, "aliases": [] } + ], + "DD_OTLP_PROFILES_ENDPOINT": [ + { + "version": "A", + "type": "string", + "default": null, + "aliases": [] + } + ], + "DD_OTLP_PROFILES_HEADERS": [ + { + "version": "A", + "type": "map", + "default": null, + "aliases": [], + "sensitive": true + } + ], + "DD_OTLP_PROFILES_PROTOCOL": [ + { + "version": "A", + "type": "string", + "default": "HTTP_PROTOBUF", + "aliases": [] + } + ], + "DD_OTLP_PROFILES_COMPRESSION": [ + { + "version": "A", + "type": "string", + "default": "NONE", + "aliases": [] + } + ], + "DD_OTLP_PROFILES_TIMEOUT": [ + { + "version": "A", + "type": "int", + "default": "30000", + "aliases": [] + } + ], + "DD_PROFILING_OTLP_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "false", + "aliases": [] + } + ], + "DD_PROFILING_OTLP_MODE": [ + { + "version": "B", + "type": "string", + "default": "LIGHT", + "aliases": [] + } ] }, "deprecations": { diff --git a/settings.gradle.kts b/settings.gradle.kts index ad3c1a2012c..1d5a3abce87 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -91,6 +91,7 @@ include( ":dd-java-agent:agent-profiling:profiling-controller-ddprof", ":dd-java-agent:agent-profiling:profiling-controller-openjdk", ":dd-java-agent:agent-profiling:profiling-controller-oracle", + ":dd-java-agent:agent-profiling:profiling-otel", ":dd-java-agent:agent-profiling:profiling-scrubber", ":dd-java-agent:agent-profiling:profiling-testing", ":dd-java-agent:agent-profiling:profiling-uploader", @@ -116,6 +117,7 @@ include( include( ":communication", + ":communication:otlp-exporter", ":components:annotations", ":components:context", ":components:environment", diff --git a/utils/config-utils/src/main/java/datadog/trace/bootstrap/config/provider/OtelEnvironmentConfigSource.java b/utils/config-utils/src/main/java/datadog/trace/bootstrap/config/provider/OtelEnvironmentConfigSource.java index 67dd95eab2d..f57c846ba59 100644 --- a/utils/config-utils/src/main/java/datadog/trace/bootstrap/config/provider/OtelEnvironmentConfigSource.java +++ b/utils/config-utils/src/main/java/datadog/trace/bootstrap/config/provider/OtelEnvironmentConfigSource.java @@ -2,6 +2,7 @@ import static datadog.trace.api.ConfigDefaults.DEFAULT_LOGS_OTEL_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_METRICS_OTEL_ENABLED; +import static datadog.trace.api.ConfigDefaults.DEFAULT_OTLP_HTTP_PROFILES_ENDPOINT; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_OTEL_ENABLED; import static datadog.trace.api.config.GeneralConfig.ENV; import static datadog.trace.api.config.GeneralConfig.LOG_LEVEL; @@ -33,6 +34,11 @@ import static datadog.trace.api.config.OtlpConfig.OTLP_METRICS_PROTOCOL; import static datadog.trace.api.config.OtlpConfig.OTLP_METRICS_TEMPORALITY_PREFERENCE; import static datadog.trace.api.config.OtlpConfig.OTLP_METRICS_TIMEOUT; +import static datadog.trace.api.config.OtlpConfig.OTLP_PROFILES_COMPRESSION; +import static datadog.trace.api.config.OtlpConfig.OTLP_PROFILES_ENDPOINT; +import static datadog.trace.api.config.OtlpConfig.OTLP_PROFILES_HEADERS; +import static datadog.trace.api.config.OtlpConfig.OTLP_PROFILES_PROTOCOL; +import static datadog.trace.api.config.OtlpConfig.OTLP_PROFILES_TIMEOUT; import static datadog.trace.api.config.OtlpConfig.OTLP_TRACES_COMPRESSION; import static datadog.trace.api.config.OtlpConfig.OTLP_TRACES_ENDPOINT; import static datadog.trace.api.config.OtlpConfig.OTLP_TRACES_HEADERS; @@ -40,6 +46,8 @@ import static datadog.trace.api.config.OtlpConfig.OTLP_TRACES_TIMEOUT; import static datadog.trace.api.config.OtlpConfig.TRACE_OTEL_ENABLED; import static datadog.trace.api.config.OtlpConfig.TRACE_OTEL_EXPORTER; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_OTLP_ENABLED; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_OTLP_ENABLED_DEFAULT; import static datadog.trace.api.config.TraceInstrumentationConfig.TRACE_ENABLED; import static datadog.trace.api.config.TraceInstrumentationConfig.TRACE_EXTENSIONS_PATH; import static datadog.trace.api.config.TracerConfig.REQUEST_HEADER_TAGS; @@ -104,11 +112,17 @@ public ConfigOrigin origin() { OtelEnvironmentConfigSource(Properties datadogConfigFile) { this.datadogConfigFile = datadogConfigFile; - this.enabled = traceOtelEnabled() || metricsOtelEnabled() || logsOtelEnabled(); + boolean otelEnabled = traceOtelEnabled() || metricsOtelEnabled() || logsOtelEnabled(); + boolean profilesEnabled = profilingOtlpEnabled(); + this.enabled = otelEnabled || profilesEnabled; - if (enabled) { + if (otelEnabled) { setupOtelEnvironment(); } + // OTLP profiling only maps the profiles exporter keys, not the general OTel environment + if (profilesEnabled) { + setupProfilesOtelEnvironment(); + } } private void setupOtelEnvironment() { @@ -266,6 +280,25 @@ private void setupLogsOtelEnvironment() { } } + // OTLP profile export is enabled by dd.profiling.otlp.enabled, not by an OTel exporter setting + private void setupProfilesOtelEnvironment() { + capture( + OTLP_PROFILES_HEADERS, + getOtelOtlpProperty("profiles", "headers", "dd." + OTLP_PROFILES_HEADERS)); + capture( + OTLP_PROFILES_PROTOCOL, + getOtelOtlpProperty("profiles", "protocol", "dd." + OTLP_PROFILES_PROTOCOL)); + capture( + OTLP_PROFILES_COMPRESSION, + getOtelOtlpProperty("profiles", "compression", "dd." + OTLP_PROFILES_COMPRESSION)); + capture( + OTLP_PROFILES_TIMEOUT, + getOtelOtlpProperty("profiles", "timeout", "dd." + OTLP_PROFILES_TIMEOUT)); + capture( + OTLP_PROFILES_ENDPOINT, + getOtelOtlpProperty("profiles", "endpoint", "dd." + OTLP_PROFILES_ENDPOINT)); + } + private boolean traceOtelEnabled() { String enabled = getDatadogProperty("dd." + TRACE_OTEL_ENABLED); if (null != enabled) { @@ -284,6 +317,15 @@ private boolean metricsOtelEnabled() { } } + private boolean profilingOtlpEnabled() { + String enabled = getDatadogProperty("dd." + PROFILING_OTLP_ENABLED); + if (null != enabled) { + return Boolean.parseBoolean(enabled); + } else { + return PROFILING_OTLP_ENABLED_DEFAULT; + } + } + private boolean logsOtelEnabled() { String enabled = getDatadogProperty("dd." + LOGS_OTEL_ENABLED); if (null != enabled) { @@ -355,6 +397,12 @@ private String getOtelOtlpProperty(String signal, String subkey, String ddSysPro && !"grpc".equalsIgnoreCase(otelEnvironment.get(OTLP_TRACES_PROTOCOL))) { otelValue = otelValue + (otelValue.endsWith("/") ? "v1/traces" : "/v1/traces"); } + if ("profiles".equals(signal) && !"grpc".equalsIgnoreCase(profilesProtocol())) { + otelValue = + otelValue + + (otelValue.endsWith("/") ? "" : "/") + + DEFAULT_OTLP_HTTP_PROFILES_ENDPOINT; + } } } if (null == otelValue) { @@ -371,6 +419,12 @@ private String getOtelOtlpProperty(String signal, String subkey, String ddSysPro return otelValue; } + /** The profiles protocol from the OTel environment, else from the Datadog setting that won. */ + private String profilesProtocol() { + String protocol = otelEnvironment.get(OTLP_PROFILES_PROTOCOL); + return protocol != null ? protocol : getDatadogProperty("dd." + OTLP_PROFILES_PROTOCOL); + } + /** * Gets a Datadog property. *