From cbd622682457282712ff382f2da38ca40419e695 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 24 Feb 2026 10:30:30 +0100 Subject: [PATCH 01/28] llmobs: set model tag even when llmobs disabled --- .../openai_java/ChatCompletionDecorator.java | 18 +++++++------ .../openai_java/CompletionDecorator.java | 27 ++++++++++--------- .../openai_java/EmbeddingDecorator.java | 19 ++++++------- .../openai_java/ResponseDecorator.java | 23 +++++++--------- 4 files changed, 44 insertions(+), 43 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index 6c9e9cad9d9..74f14a9daf2 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -31,11 +31,6 @@ public void withChatCompletionCreateParams( AgentSpan span, ChatCompletionCreateParams params, boolean stream) { span.setResourceName(CHAT_COMPLETIONS_CREATE); span.setTag(CommonTags.OPENAI_REQUEST_ENDPOINT, "/v1/chat/completions"); - if (!llmObsEnabled) { - return; - } - - span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); if (params == null) { return; } @@ -45,6 +40,12 @@ public void withChatCompletionCreateParams( .asString() .ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); + if (!llmObsEnabled) { + return; + } + + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); + span.setTag( CommonTags.INPUT, params.messages().stream() @@ -97,13 +98,14 @@ private static LLMObs.LLMMessage llmMessage(ChatCompletionMessageParam m) { } public void withChatCompletion(AgentSpan span, ChatCompletion completion) { - if (!llmObsEnabled) { - return; - } String modelName = completion.model(); span.setTag(CommonTags.OPENAI_RESPONSE_MODEL, modelName); span.setTag(CommonTags.MODEL_NAME, modelName); + if (!llmObsEnabled) { + return; + } + List output = completion.choices().stream() .map(ChatCompletionDecorator::llmMessage) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java index 2291c860d00..51e7c12ef1c 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java @@ -23,11 +23,6 @@ public class CompletionDecorator { public void withCompletionCreateParams(AgentSpan span, CompletionCreateParams params) { span.setResourceName(COMPLETIONS_CREATE); span.setTag(CommonTags.OPENAI_REQUEST_ENDPOINT, "/v1/completions"); - if (!llmObsEnabled) { - return; - } - - span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); if (params == null) { return; } @@ -37,6 +32,12 @@ public void withCompletionCreateParams(AgentSpan span, CompletionCreateParams pa ._value() .asString() .ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); + + if (!llmObsEnabled) { + return; + } + + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); params .prompt() .flatMap(p -> p.string()) @@ -61,14 +62,14 @@ public void withCompletionCreateParams(AgentSpan span, CompletionCreateParams pa } public void withCompletion(AgentSpan span, Completion completion) { - if (!llmObsEnabled) { - return; - } - String modelName = completion.model(); span.setTag(CommonTags.OPENAI_RESPONSE_MODEL, modelName); span.setTag(CommonTags.MODEL_NAME, modelName); + if (!llmObsEnabled) { + return; + } + List output = completion.choices().stream() .map(v -> LLMObs.LLMMessage.from(null, v.text())) @@ -86,10 +87,6 @@ public void withCompletion(AgentSpan span, Completion completion) { } public void withCompletions(AgentSpan span, List completions) { - if (!llmObsEnabled) { - return; - } - if (completions.isEmpty()) { return; } @@ -99,6 +96,10 @@ public void withCompletions(AgentSpan span, List completions) { span.setTag(CommonTags.OPENAI_RESPONSE_MODEL, modelName); span.setTag(CommonTags.MODEL_NAME, modelName); + if (!llmObsEnabled) { + return; + } + Map textByChoiceIndex = new HashMap<>(); for (Completion completion : completions) { completion diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java index 88098358ebf..02d4588358c 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java @@ -25,11 +25,6 @@ public class EmbeddingDecorator { public void withEmbeddingCreateParams(AgentSpan span, EmbeddingCreateParams params) { span.setResourceName(EMBEDDINGS_CREATE); span.setTag(CommonTags.OPENAI_REQUEST_ENDPOINT, "/v1/embeddings"); - if (!llmObsEnabled) { - return; - } - - span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_EMBEDDING_SPAN_KIND); if (params == null) { return; } @@ -39,6 +34,12 @@ public void withEmbeddingCreateParams(AgentSpan span, EmbeddingCreateParams para .asString() .ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); + if (!llmObsEnabled) { + return; + } + + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_EMBEDDING_SPAN_KIND); + span.setTag(CommonTags.INPUT, embeddingDocuments(params.input())); Map metadata = new HashMap<>(); @@ -59,14 +60,14 @@ private List embeddingDocuments(EmbeddingCreateParams.Input inp } public void withCreateEmbeddingResponse(AgentSpan span, CreateEmbeddingResponse response) { - if (!llmObsEnabled) { - return; - } - String modelName = response.model(); span.setTag(CommonTags.OPENAI_RESPONSE_MODEL, modelName); span.setTag(CommonTags.MODEL_NAME, modelName); + if (!llmObsEnabled) { + return; + } + if (!response.data().isEmpty()) { int embeddingCount = response.data().size(); Embedding firstEmbedding = response.data().get(0); diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 4bc95e32934..db2258785d4 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -37,11 +37,6 @@ public class ResponseDecorator { public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params) { span.setResourceName(RESPONSES_CREATE); span.setTag(CommonTags.OPENAI_REQUEST_ENDPOINT, "/v1/responses"); - if (!llmObsEnabled) { - return; - } - - span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); if (params == null) { return; } @@ -51,6 +46,12 @@ public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params String modelName = extractResponseModel(params._model()); span.setTag(CommonTags.OPENAI_REQUEST_MODEL, modelName); + if (!llmObsEnabled) { + return; + } + + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); + List inputMessages = new ArrayList<>(); params @@ -369,10 +370,6 @@ public void withResponse(AgentSpan span, Response response) { } public void withResponseStreamEvents(AgentSpan span, List events) { - if (!llmObsEnabled) { - return; - } - for (ResponseStreamEvent event : events) { if (event.isCompleted()) { Response response = event.asCompleted().response(); @@ -388,14 +385,14 @@ public void withResponseStreamEvents(AgentSpan span, List e } private void withResponse(AgentSpan span, Response response, boolean stream) { - if (!llmObsEnabled) { - return; - } - String modelName = extractResponseModel(response._model()); span.setTag(CommonTags.OPENAI_RESPONSE_MODEL, modelName); span.setTag(CommonTags.MODEL_NAME, modelName); + if (!llmObsEnabled) { + return; + } + List outputMessages = extractResponseOutputMessages(response.output()); if (!outputMessages.isEmpty()) { span.setTag(CommonTags.OUTPUT, outputMessages); From 4f2767372d0c37dae170c3f964c317295ac93b14 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Mon, 2 Mar 2026 13:30:21 +0100 Subject: [PATCH 02/28] Set metadata.stream tag no matter it's true or false --- .../instrumentation/openai_java/ChatCompletionDecorator.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index 74f14a9daf2..eb918ef3886 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -56,9 +56,7 @@ public void withChatCompletionCreateParams( // maxTokens is deprecated but integration tests missing to provide maxCompletionTokens params.maxTokens().ifPresent(v -> metadata.put("max_tokens", v)); params.temperature().ifPresent(v -> metadata.put("temperature", v)); - if (stream) { - metadata.put("stream", true); - } + metadata.put("stream", stream); params .streamOptions() .ifPresent( From d128d6baddaf647fa34ad6fa11c7be0672547b15 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Mon, 2 Mar 2026 13:46:41 +0100 Subject: [PATCH 03/28] Set chat/completion CACHE_READ_INPUT_TOKENS tag --- .../instrumentation/openai_java/ChatCompletionDecorator.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index eb918ef3886..95c746880cf 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -117,6 +117,10 @@ public void withChatCompletion(AgentSpan span, ChatCompletion completion) { span.setTag(CommonTags.INPUT_TOKENS, usage.promptTokens()); span.setTag(CommonTags.OUTPUT_TOKENS, usage.completionTokens()); span.setTag(CommonTags.TOTAL_TOKENS, usage.totalTokens()); + usage + .promptTokensDetails() + .flatMap(details -> details.cachedTokens()) + .ifPresent(v -> span.setTag(CommonTags.CACHE_READ_INPUT_TOKENS, v)); }); } From 3fc5cebce75f121245d4b850445069741ff23d52 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Mon, 2 Mar 2026 18:31:44 +0100 Subject: [PATCH 04/28] Set error nad error_type tags --- .../trace/instrumentation/openai_java/CommonTags.java | 3 +++ .../trace/instrumentation/openai_java/OpenAiDecorator.java | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java index f779662ac0d..a4335a7edfe 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java @@ -21,6 +21,9 @@ interface CommonTags { String ML_APP = TAG_PREFIX + LLMObsTags.ML_APP; String VERSION = TAG_PREFIX + "version"; + String ERROR = TAG_PREFIX + "error"; + String ERROR_TYPE = TAG_PREFIX + "error_type"; + String ENV = TAG_PREFIX + "env"; String SERVICE = TAG_PREFIX + "service"; String PARENT_ID = TAG_PREFIX + "parent_id"; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java index 331269bad83..2332b578ebf 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java @@ -3,6 +3,7 @@ import com.openai.core.ClientOptions; import com.openai.core.http.Headers; import datadog.trace.api.Config; +import datadog.trace.api.DDTags; import datadog.trace.api.WellKnownTags; import datadog.trace.api.llmobs.LLMObsContext; import datadog.trace.api.telemetry.LLMObsMetricCollector; @@ -12,6 +13,7 @@ import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.bootstrap.instrumentation.decorator.ClientDecorator; +import datadog.trace.core.CoreSpan; import java.util.List; public class OpenAiDecorator extends ClientDecorator { @@ -111,6 +113,9 @@ public AgentSpan afterStart(AgentSpan span) { @Override public AgentSpan beforeFinish(AgentSpan span) { if (llmObsEnabled) { + span.setTag(CommonTags.ERROR, span.isError() ? 1 : 0); + span.setTag(CommonTags.ERROR_TYPE, span.getTag(DDTags.ERROR_TYPE)); + Object spanKindTag = span.getTag(CommonTags.SPAN_KIND); if (spanKindTag != null) { String spanKind = spanKindTag.toString(); From 021a9d1c9bf9a04ac3951120cb1dea26c9c92df5 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Mon, 2 Mar 2026 22:37:22 +0100 Subject: [PATCH 05/28] Use "" instead of null for the role in CompletionDecorator to comply wthTestOpenAiLlmInteractions::test_completion --- .../instrumentation/openai_java/CompletionDecorator.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java index 51e7c12ef1c..1b95491b64b 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java @@ -45,7 +45,7 @@ public void withCompletionCreateParams(AgentSpan span, CompletionCreateParams pa input -> span.setTag( CommonTags.INPUT, - Collections.singletonList(LLMObs.LLMMessage.from(null, input)))); + Collections.singletonList(LLMObs.LLMMessage.from("", input)))); Map metadata = new HashMap<>(); params.maxTokens().ifPresent(v -> metadata.put("max_tokens", v)); @@ -72,7 +72,7 @@ public void withCompletion(AgentSpan span, Completion completion) { List output = completion.choices().stream() - .map(v -> LLMObs.LLMMessage.from(null, v.text())) + .map(v -> LLMObs.LLMMessage.from("", v.text())) .collect(Collectors.toList()); span.setTag(CommonTags.OUTPUT, output); @@ -116,7 +116,7 @@ public void withCompletions(AgentSpan span, List completions) { List output = textByChoiceIndex.entrySet().stream() .sorted(Map.Entry.comparingByKey()) - .map(entry -> LLMObs.LLMMessage.from(null, entry.getValue().toString())) + .map(entry -> LLMObs.LLMMessage.from("", entry.getValue().toString())) .collect(Collectors.toList()); span.setTag(CommonTags.OUTPUT, output); From 0637931c5bb710c3ea29581dcdec36ccbba7d514 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Mon, 2 Mar 2026 23:12:24 +0100 Subject: [PATCH 06/28] Use "" instead of null for the content to comply with TestOpenAiLlmInteractions::test_chat_completion_tool_call --- .../instrumentation/openai_java/ChatCompletionDecorator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index 95c746880cf..c8151c00a2a 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -131,7 +131,7 @@ private static LLMObs.LLMMessage llmMessage(ChatCompletion.Choice choice) { if (roleOpt.isPresent()) { role = String.valueOf(roleOpt.get()); } - String content = msg.content().orElse(null); + String content = msg.content().orElse(""); Optional> toolCallsOpt = msg.toolCalls(); if (toolCallsOpt.isPresent() && !toolCallsOpt.get().isEmpty()) { From 0cb41e1a9522f617afed58a4dcc2e1e5ef966388 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 3 Mar 2026 13:34:57 +0100 Subject: [PATCH 07/28] Add missing metatadata.tool_choice --- .../openai_java/ChatCompletionDecorator.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index c8151c00a2a..618c7982410 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -71,6 +71,24 @@ public void withChatCompletionCreateParams( params.n().ifPresent(v -> metadata.put("n", v)); params.seed().ifPresent(v -> metadata.put("seed", v)); span.setTag(CommonTags.METADATA, metadata); + params + .toolChoice() + .ifPresent( + toolChoice -> { + String choice = null; + if (toolChoice.isAuto()) { + choice = "auto"; + } else if (toolChoice.isAllowedToolChoice()) { + choice = "allowed_tools"; + } else if (toolChoice.isNamedToolChoice()) { + choice = "function"; + } else if (toolChoice.isNamedToolChoiceCustom()) { + choice = "custom"; + } + if (choice != null) { + metadata.put("tool_choice", choice); + } + }); } private static LLMObs.LLMMessage llmMessage(ChatCompletionMessageParam m) { From a42f8aa2e3a34c8bf14e031a502ca8bb9a468e6f Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 3 Mar 2026 21:09:03 +0100 Subject: [PATCH 08/28] Add missing tool_definitions --- .../openai_java/ChatCompletionDecorator.java | 108 ++++++++++++++++++ .../openai_java/CommonTags.java | 2 + .../openai_java/OpenAiDecorator.java | 1 - .../datadog/trace/api/llmobs/LLMObsTags.java | 1 + .../writer/ddintake/LLMObsSpanMapper.java | 1 + 5 files changed, 112 insertions(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index 618c7982410..e5def4f67bb 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -1,12 +1,16 @@ package datadog.trace.instrumentation.openai_java; +import com.openai.core.JsonValue; import com.openai.helpers.ChatCompletionAccumulator; +import com.openai.models.FunctionDefinition; import com.openai.models.chat.completions.ChatCompletion; import com.openai.models.chat.completions.ChatCompletionChunk; import com.openai.models.chat.completions.ChatCompletionCreateParams; +import com.openai.models.chat.completions.ChatCompletionFunctionTool; import com.openai.models.chat.completions.ChatCompletionMessage; import com.openai.models.chat.completions.ChatCompletionMessageParam; import com.openai.models.chat.completions.ChatCompletionMessageToolCall; +import com.openai.models.chat.completions.ChatCompletionTool; import datadog.trace.api.Config; import datadog.trace.api.llmobs.LLMObs; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; @@ -89,6 +93,110 @@ public void withChatCompletionCreateParams( metadata.put("tool_choice", choice); } }); + + List tools = params.tools().orElse(Collections.emptyList()); + if (!tools.isEmpty()) { + span.setTag(CommonTags.TOOL_DEFINITIONS, extractToolDefinitions(tools)); + } + } + + private List> extractToolDefinitions(List tools) { + List> toolDefinitions = new ArrayList<>(); + for (ChatCompletionTool tool : tools) { + if (tool.isFunction()) { + Map toolDef = extractFunctionToolDef(tool.asFunction()); + if (toolDef != null) { + toolDefinitions.add(toolDef); + } + } + } + return toolDefinitions; + } + + private static Map extractFunctionToolDef(ChatCompletionFunctionTool funcTool) { + // Try typed access first (works when built programmatically) + Optional funcDefOpt = funcTool._function().asKnown(); + if (funcDefOpt.isPresent()) { + FunctionDefinition funcDef = funcDefOpt.get(); + Map toolDef = new HashMap<>(); + toolDef.put("name", funcDef.name()); + funcDef.description().ifPresent(desc -> toolDef.put("description", desc)); + funcDef + .parameters() + .ifPresent( + params -> + toolDef.put("schema", jsonValueMapToObject(params._additionalProperties()))); + return toolDef; + } + + // Fall back to raw JSON extraction (when deserialized from HTTP request) + Optional rawOpt = funcTool._function().asUnknown(); + if (!rawOpt.isPresent()) { + return null; + } + Optional> objOpt = rawOpt.get().asObject(); + if (!objOpt.isPresent()) { + return null; + } + Map obj = objOpt.get(); + JsonValue nameValue = obj.get("name"); + if (nameValue == null) { + return null; + } + Optional nameOpt = nameValue.asString(); + if (!nameOpt.isPresent()) { + return null; + } + Map toolDef = new HashMap<>(); + toolDef.put("name", nameOpt.get()); + JsonValue descValue = obj.get("description"); + if (descValue != null) { + descValue.asString().ifPresent(desc -> toolDef.put("description", desc)); + } + JsonValue paramsValue = obj.get("parameters"); + if (paramsValue != null) { + Object schema = jsonValueToObject(paramsValue); + if (schema != null) { + toolDef.put("schema", schema); + } + } + return toolDef; + } + + private static Map jsonValueMapToObject(Map map) { + Map result = new HashMap<>(); + for (Map.Entry entry : map.entrySet()) { + result.put(entry.getKey(), jsonValueToObject(entry.getValue())); + } + return result; + } + + private static Object jsonValueToObject(JsonValue value) { + Optional str = value.asString(); + if (str.isPresent()) { + return str.get(); + } + Optional num = value.asNumber(); + if (num.isPresent()) { + return num.get(); + } + Optional bool = value.asBoolean(); + if (bool.isPresent()) { + return bool.get(); + } + Optional> obj = value.asObject(); + if (obj.isPresent()) { + return jsonValueMapToObject(obj.get()); + } + Optional> arr = value.asArray(); + if (arr.isPresent()) { + List list = new ArrayList<>(); + for (JsonValue item : arr.get()) { + list.add(jsonValueToObject(item)); + } + return list; + } + return null; } private static LLMObs.LLMMessage llmMessage(ChatCompletionMessageParam m) { diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java index a4335a7edfe..ed5b689cd9b 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java @@ -28,6 +28,8 @@ interface CommonTags { String SERVICE = TAG_PREFIX + "service"; String PARENT_ID = TAG_PREFIX + "parent_id"; + String TOOL_DEFINITIONS = TAG_PREFIX + "tool_definitions"; + String METRIC_PREFIX = "_ml_obs_metric."; String INPUT_TOKENS = METRIC_PREFIX + "input_tokens"; String OUTPUT_TOKENS = METRIC_PREFIX + "output_tokens"; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java index 2332b578ebf..6380f981797 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java @@ -13,7 +13,6 @@ import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.bootstrap.instrumentation.decorator.ClientDecorator; -import datadog.trace.core.CoreSpan; import java.util.List; public class OpenAiDecorator extends ClientDecorator { diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java index afa4f2b241e..130cf610dc0 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java @@ -12,4 +12,5 @@ public class LLMObsTags { public static final String MODEL_NAME = "model_name"; public static final String MODEL_VERSION = "model_version"; public static final String MODEL_PROVIDER = "model_provider"; + public static final String TOOL_DEFINITIONS = "tool_definitions"; } diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index 7241d469006..e0cd7db3e02 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -230,6 +230,7 @@ private static final class MetaWriter implements MetadataConsumer { LLMOBS_TAG_PREFIX + LLMObsTags.MODEL_NAME, LLMOBS_TAG_PREFIX + LLMObsTags.MODEL_PROVIDER, LLMOBS_TAG_PREFIX + LLMObsTags.MODEL_VERSION, + LLMOBS_TAG_PREFIX + LLMObsTags.TOOL_DEFINITIONS, LLMOBS_TAG_PREFIX + LLMObsTags.METADATA))); MetaWriter withWritable(Writable writable, Map errorInfo) { From 6e10255898af899792665ac5e0d84a84e204fe14 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 3 Mar 2026 21:51:50 +0100 Subject: [PATCH 09/28] Add source:integration tag --- .../datadog/trace/instrumentation/openai_java/CommonTags.java | 1 + .../trace/instrumentation/openai_java/OpenAiDecorator.java | 1 + 2 files changed, 2 insertions(+) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java index ed5b689cd9b..0e550437026 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java @@ -20,6 +20,7 @@ interface CommonTags { String ML_APP = TAG_PREFIX + LLMObsTags.ML_APP; String VERSION = TAG_PREFIX + "version"; + String SOURCE = TAG_PREFIX + "source"; String ERROR = TAG_PREFIX + "error"; String ERROR_TYPE = TAG_PREFIX + "error_type"; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java index 6380f981797..fae2880c082 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java @@ -98,6 +98,7 @@ public AgentSpan afterStart(AgentSpan span) { span.setTag(CommonTags.VERSION, wellKnownTags.getVersion()); span.setTag(CommonTags.ML_APP, Config.get().getLlmObsMlApp()); + span.setTag(CommonTags.SOURCE, "integration"); AgentSpanContext parent = LLMObsContext.current(); String parentSpanId = LLMObsContext.ROOT_SPAN_ID; From 34f3a07ec396815b6878bcf56024d9d1dac861e8 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 4 Mar 2026 11:31:52 +0100 Subject: [PATCH 10/28] Add missing _dd attribute to the llmobs span event --- .../llmobs/writer/ddintake/LLMObsSpanMapper.java | 16 ++++++++++++++-- .../writer/ddintake/LLMObsSpanMapperTest.groovy | 4 ++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index e0cd7db3e02..1dec4ad62e5 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -52,6 +52,8 @@ public class LLMObsSpanMapper implements RemoteMapper { private static final byte[] SPAN_ID = "span_id".getBytes(StandardCharsets.UTF_8); private static final byte[] TRACE_ID = "trace_id".getBytes(StandardCharsets.UTF_8); + private static final byte[] DD = "_dd".getBytes(StandardCharsets.UTF_8); + private static final byte[] APM_TRACE_ID = "apm_trace_id".getBytes(StandardCharsets.UTF_8); private static final byte[] PARENT_ID = "parent_id".getBytes(StandardCharsets.UTF_8); private static final byte[] NAME = "name".getBytes(StandardCharsets.UTF_8); private static final byte[] DURATION = "duration".getBytes(StandardCharsets.UTF_8); @@ -120,7 +122,7 @@ public void map(List> trace, Writable writable) { } for (CoreSpan span : llmobsSpans) { - writable.startMap(11); + writable.startMap(12); // 1 writable.writeUTF8(SPAN_ID); writable.writeString(String.valueOf(span.getSpanId()), null); @@ -156,7 +158,17 @@ public void map(List> trace, Writable writable) { writable.writeUTF8(STATUS); writable.writeString(errored ? "error" : "ok", null); - /* 9 (metrics), 10 (tags), 11 meta */ + // 9 + writable.writeUTF8(DD); + writable.startMap(3); + writable.writeUTF8(SPAN_ID); + writable.writeString(String.valueOf(span.getSpanId()), null); + writable.writeUTF8(TRACE_ID); + writable.writeString(span.getTraceId().toHexString(), null); + writable.writeUTF8(APM_TRACE_ID); + writable.writeString(span.getTraceId().toHexString(), null); + + /* 10 (metrics), 11 (tags), 12 meta */ span.processTagsAndBaggage(metaWriter.withWritable(writable, getErrorsMap(span))); } diff --git a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy index 20656ba2a1e..74fb39fed90 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy @@ -102,6 +102,10 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { spanData.containsKey("start_ns") spanData.containsKey("duration") spanData["error"] == 0 + spanData.containsKey("_dd") + spanData["_dd"]["span_id"] == spanData["span_id"] + spanData["_dd"]["trace_id"] == spanData["trace_id"] + spanData["_dd"]["apm_trace_id"] == spanData["trace_id"] spanData.containsKey("meta") spanData["meta"]["span.kind"] == "llm" From a0c1139404dcc77a654c3ed30ebd91c2808db995 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 4 Mar 2026 11:49:52 +0100 Subject: [PATCH 11/28] Add missing error tags --- .../writer/ddintake/LLMObsSpanMapper.java | 37 +++++++++++++++---- .../ddintake/LLMObsSpanMapperTest.groovy | 11 +++++- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index 1dec4ad62e5..f4c31e2c6ec 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -60,6 +60,9 @@ public class LLMObsSpanMapper implements RemoteMapper { private static final byte[] START_NS = "start_ns".getBytes(StandardCharsets.UTF_8); private static final byte[] STATUS = "status".getBytes(StandardCharsets.UTF_8); private static final byte[] ERROR = "error".getBytes(StandardCharsets.UTF_8); + private static final byte[] ERROR_MESSAGE = "message".getBytes(StandardCharsets.UTF_8); + private static final byte[] ERROR_TYPE = "type".getBytes(StandardCharsets.UTF_8); + private static final byte[] ERROR_STACK = "stack".getBytes(StandardCharsets.UTF_8); private static final byte[] META = "meta".getBytes(StandardCharsets.UTF_8); private static final byte[] METADATA = "metadata".getBytes(StandardCharsets.UTF_8); @@ -215,15 +218,15 @@ private static Map getErrorsMap(CoreSpan span) { Map errors = new HashMap<>(); String errorMsg = span.getTag(DDTags.ERROR_MSG); if (errorMsg != null && !errorMsg.isEmpty()) { - errors.put(DDTags.ERROR_MSG, errorMsg); + errors.put("message", errorMsg); } String errorType = span.getTag(DDTags.ERROR_TYPE); if (errorType != null && !errorType.isEmpty()) { - errors.put(DDTags.ERROR_TYPE, errorType); + errors.put("type", errorType); } String errorStack = span.getTag(DDTags.ERROR_STACK); if (errorStack != null && !errorStack.isEmpty()) { - errors.put(DDTags.ERROR_STACK, errorStack); + errors.put("stack", errorStack); } return errors; } @@ -306,15 +309,35 @@ public void accept(Metadata metadata) { } // write meta (11) - int metaSize = tagsToRemapToMeta.size() + 1 + (null != errorInfo ? errorInfo.size() : 0); + int metaSize = + tagsToRemapToMeta.size() + + 1 + + (null != errorInfo && !errorInfo.isEmpty() ? 1 : 0); writable.writeUTF8(META); writable.startMap(metaSize); writable.writeUTF8(SPAN_KIND); writable.writeString(spanKind, null); - for (Map.Entry error : errorInfo.entrySet()) { - writable.writeUTF8(error.getKey().getBytes()); - writable.writeString(error.getValue(), null); + if (null != errorInfo && !errorInfo.isEmpty()) { + writable.writeUTF8(ERROR); + writable.startMap(errorInfo.size()); + for (Map.Entry error : errorInfo.entrySet()) { + switch (error.getKey()) { + case "message": + writable.writeUTF8(ERROR_MESSAGE); + break; + case "type": + writable.writeUTF8(ERROR_TYPE); + break; + case "stack": + writable.writeUTF8(ERROR_STACK); + break; + default: + writable.writeString(error.getKey(), null); + break; + } + writable.writeString(error.getValue(), null); + } } for (Map.Entry tag : tagsToRemapToMeta.entrySet()) { diff --git a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy index 74fb39fed90..1923c07470b 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper import datadog.communication.serialization.ByteBufferConsumer import datadog.communication.serialization.FlushingBuffer import datadog.communication.serialization.msgpack.MsgPackWriter +import datadog.trace.api.DDTags import datadog.trace.api.llmobs.LLMObs import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes import datadog.trace.bootstrap.instrumentation.api.Tags @@ -44,6 +45,10 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { llmSpan.setTag("_ml_obs_tag.input", inputMessages) llmSpan.setTag("_ml_obs_tag.output", outputMessages) llmSpan.setTag("_ml_obs_tag.metadata", [temperature: 0.7, max_tokens: 100]) + llmSpan.setError(true) + llmSpan.setTag(DDTags.ERROR_MSG, "boom") + llmSpan.setTag(DDTags.ERROR_TYPE, "java.lang.IllegalStateException") + llmSpan.setTag(DDTags.ERROR_STACK, "stacktrace") llmSpan.finish() @@ -101,7 +106,7 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { spanData.containsKey("trace_id") spanData.containsKey("start_ns") spanData.containsKey("duration") - spanData["error"] == 0 + spanData["error"] == 1 spanData.containsKey("_dd") spanData["_dd"]["span_id"] == spanData["span_id"] spanData["_dd"]["trace_id"] == spanData["trace_id"] @@ -109,6 +114,10 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { spanData.containsKey("meta") spanData["meta"]["span.kind"] == "llm" + spanData["meta"].containsKey("error") + spanData["meta"]["error"]["message"] == "boom" + spanData["meta"]["error"]["type"] == "java.lang.IllegalStateException" + spanData["meta"]["error"]["stack"] == "stacktrace" spanData["meta"].containsKey("input") spanData["meta"]["input"].containsKey("messages") spanData["meta"]["input"]["messages"][0].containsKey("content") From effc34302592382744dfc0dc100b264c788923c0 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 4 Mar 2026 12:24:40 +0100 Subject: [PATCH 12/28] Remove error from the llmobs span event. It must be part of meta block --- .../writer/ddintake/LLMObsSpanMapper.java | 18 +++++------------- .../ddintake/LLMObsSpanMapperTest.groovy | 1 - 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index f4c31e2c6ec..e0b5dce3550 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -125,7 +125,7 @@ public void map(List> trace, Writable writable) { } for (CoreSpan span : llmobsSpans) { - writable.startMap(12); + writable.startMap(11); // 1 writable.writeUTF8(SPAN_ID); writable.writeString(String.valueOf(span.getSpanId()), null); @@ -152,16 +152,10 @@ public void map(List> trace, Writable writable) { writable.writeFloat(span.getDurationNano()); // 7 - writable.writeUTF8(ERROR); - writable.writeInt(span.getError()); - - boolean errored = span.getError() == 1; - - // 8 writable.writeUTF8(STATUS); - writable.writeString(errored ? "error" : "ok", null); + writable.writeString(span.getError() == 0 ? "ok" : "error", null); - // 9 + // 8 writable.writeUTF8(DD); writable.startMap(3); writable.writeUTF8(SPAN_ID); @@ -171,7 +165,7 @@ public void map(List> trace, Writable writable) { writable.writeUTF8(APM_TRACE_ID); writable.writeString(span.getTraceId().toHexString(), null); - /* 10 (metrics), 11 (tags), 12 meta */ + /* 9 (metrics), 10 (tags), 11 meta */ span.processTagsAndBaggage(metaWriter.withWritable(writable, getErrorsMap(span))); } @@ -310,9 +304,7 @@ public void accept(Metadata metadata) { // write meta (11) int metaSize = - tagsToRemapToMeta.size() - + 1 - + (null != errorInfo && !errorInfo.isEmpty() ? 1 : 0); + tagsToRemapToMeta.size() + 1 + (null != errorInfo && !errorInfo.isEmpty() ? 1 : 0); writable.writeUTF8(META); writable.startMap(metaSize); writable.writeUTF8(SPAN_KIND); diff --git a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy index 1923c07470b..6140431b836 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy @@ -106,7 +106,6 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { spanData.containsKey("trace_id") spanData.containsKey("start_ns") spanData.containsKey("duration") - spanData["error"] == 1 spanData.containsKey("_dd") spanData["_dd"]["span_id"] == spanData["span_id"] spanData["_dd"]["trace_id"] == spanData["trace_id"] From c0e38761a4f2515ec1ed43e7173c6be3096f25bd Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 4 Mar 2026 13:06:00 +0100 Subject: [PATCH 13/28] Add missing meta.text.verbosity --- .../openai-java/openai-java-3.0/build.gradle | 2 +- .../openai_java/ResponseDecorator.java | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/build.gradle b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/build.gradle index cd654d03334..ceda41c2804 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/build.gradle +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/build.gradle @@ -1,6 +1,6 @@ apply from: "$rootDir/gradle/java.gradle" -def minVer = '3.0.0' +def minVer = '3.0.1' muzzle { pass { diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index db2258785d4..e7be85b77c6 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -424,6 +424,7 @@ private void withResponse(AgentSpan span, Response response, boolean stream) { (Response.Truncation t) -> metadata.put("truncation", t._value().asString().orElse(null))); + Map textMap = new HashMap<>(); response .text() .ifPresent( @@ -432,7 +433,6 @@ private void withResponse(AgentSpan span, Response response, boolean stream) { .format() .ifPresent( format -> { - Map textMap = new HashMap<>(); Map formatMap = new HashMap<>(); if (format.isText()) { formatMap.put("type", "text"); @@ -442,9 +442,17 @@ private void withResponse(AgentSpan span, Response response, boolean stream) { formatMap.put("type", "json_object"); } textMap.put("format", formatMap); - metadata.put("text", textMap); + }); + textConfig + .verbosity() + .ifPresent( + verbosity -> { + textMap.put("verbosity", verbosity.asString()); }); }); + if (!textMap.isEmpty()) { + metadata.put("text", textMap); + } if (stream) { metadata.put("stream", true); From b00077055aec4e578bab797a76bebd3c070ffb78 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 4 Mar 2026 15:42:47 +0100 Subject: [PATCH 14/28] Add summaryText and encrypted_content --- .../openai_java/ResponseDecorator.java | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index e7be85b77c6..320d82da76f 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -496,12 +496,30 @@ private List extractResponseOutputMessages(List writer.name("encrypted_content").value(v)); - writer.name("id").value(reasoning.id()); + + String id = reasoning.id(); + writer.name("id").value(id == null ? "" : id); + writer.endObject(); + messages.add(LLMObs.LLMMessage.from("reasoning", writer.toString())); } } From 53471a2cb6041f96d77c3f1e1ca86ae10fcaeead Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 4 Mar 2026 16:24:38 +0100 Subject: [PATCH 15/28] Add missing tool_calls and tool_results for responses --- .../openai_java/ResponseDecorator.java | 18 ++++++++ .../openai_java/ToolCallExtractor.java | 42 +++++++++++++++++++ .../java/datadog/trace/api/llmobs/LLMObs.java | 5 +++ .../writer/ddintake/LLMObsSpanMapper.java | 10 +++-- 4 files changed, 72 insertions(+), 3 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 320d82da76f..c61c7cc712f 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -6,6 +6,7 @@ import com.openai.models.ResponsesModel; import com.openai.models.responses.Response; import com.openai.models.responses.ResponseCreateParams; +import com.openai.models.responses.ResponseCustomToolCall; import com.openai.models.responses.ResponseFunctionToolCall; import com.openai.models.responses.ResponseInputContent; import com.openai.models.responses.ResponseInputItem; @@ -486,6 +487,23 @@ private List extractResponseOutputMessages(List toolCalls = Collections.singletonList(toolCall); messages.add(LLMObs.LLMMessage.from("assistant", null, toolCalls)); } + } else if (item.isCustomToolCall()) { + ResponseCustomToolCall customToolCall = item.asCustomToolCall(); + LLMObs.ToolCall toolCall = ToolCallExtractor.getToolCall(customToolCall); + if (toolCall != null) { + messages.add( + LLMObs.LLMMessage.from("assistant", null, Collections.singletonList(toolCall))); + } + } else if (item.isMcpCall()) { + ResponseOutputItem.McpCall mcpCall = item.asMcpCall(); + LLMObs.ToolCall toolCall = ToolCallExtractor.getToolCall(mcpCall); + List toolCalls = + toolCall == null ? null : Collections.singletonList(toolCall); + String outputText = mcpCall.output().orElse(""); + LLMObs.ToolResult toolResult = + LLMObs.ToolResult.from(mcpCall.name(), "mcp_tool_result", mcpCall.id(), outputText); + List toolResults = Collections.singletonList(toolResult); + messages.add(LLMObs.LLMMessage.from("assistant", null, toolCalls, toolResults)); } else if (item.isMessage()) { ResponseOutputMessage message = item.asMessage(); String textContent = extractMessageContent(message); diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java index 357c73de0aa..21d2bf6835d 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java @@ -4,7 +4,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.openai.models.chat.completions.ChatCompletionMessageFunctionToolCall; import com.openai.models.chat.completions.ChatCompletionMessageToolCall; +import com.openai.models.responses.ResponseCustomToolCall; import com.openai.models.responses.ResponseFunctionToolCall; +import com.openai.models.responses.ResponseOutputItem.McpCall; import datadog.trace.api.llmobs.LLMObs; import java.util.Collections; import java.util.Map; @@ -64,6 +66,46 @@ public static LLMObs.ToolCall getToolCall(ResponseFunctionToolCall functionCall) return null; } + public static LLMObs.ToolCall getToolCall(ResponseCustomToolCall customToolCall) { + try { + String name = customToolCall.name(); + String callId = customToolCall.callId(); + String inputJson = customToolCall.input(); + + String type = "custom_tool_call"; + Optional typeOpt = customToolCall._type().asString(); + if (typeOpt.isPresent()) { + type = typeOpt.get(); + } + + Map arguments = parseArguments(inputJson); + return LLMObs.ToolCall.from(name, type, callId, arguments); + } catch (Exception e) { + log.debug("Failed to extract custom tool call information", e); + } + return null; + } + + public static LLMObs.ToolCall getToolCall(McpCall mcpCall) { + try { + String name = mcpCall.name(); + String callId = mcpCall.id(); + String argumentsJson = mcpCall.arguments(); + + String type = "mcp_call"; + Optional typeOpt = mcpCall._type().asString(); + if (typeOpt.isPresent()) { + type = typeOpt.get(); + } + + Map arguments = parseArguments(argumentsJson); + return LLMObs.ToolCall.from(name, type, callId, arguments); + } catch (Exception e) { + log.debug("Failed to extract MCP tool call information", e); + } + return null; + } + private static Map parseArguments(String argumentsJson) { try { return MAPPER.readValue(argumentsJson, MAP_TYPE_REF); diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java index 512a3106ce6..629faa23f5a 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java @@ -217,6 +217,11 @@ public static LLMMessage from(String role, String content, List toolCa return new LLMMessage(role, content, toolCalls, null); } + public static LLMMessage from( + String role, String content, List toolCalls, List toolResults) { + return new LLMMessage(role, content, toolCalls, toolResults); + } + public static LLMMessage from(String role, String content) { return new LLMMessage(role, content, null, null); } diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index e0b5dce3550..b4dba725406 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -354,14 +354,18 @@ public void accept(Metadata metadata) { List toolResults = message.getToolResults(); boolean hasToolCalls = null != toolCalls && !toolCalls.isEmpty(); boolean hasToolResults = null != toolResults && !toolResults.isEmpty(); - int mapSize = 2; // role and content + boolean hasContent = message.getContent() != null; + int mapSize = 1; // role + if (hasContent) mapSize++; if (hasToolCalls) mapSize++; if (hasToolResults) mapSize++; writable.startMap(mapSize); writable.writeUTF8(LLM_MESSAGE_ROLE); writable.writeString(message.getRole(), null); - writable.writeUTF8(LLM_MESSAGE_CONTENT); - writable.writeString(message.getContent(), null); + if (hasContent) { + writable.writeUTF8(LLM_MESSAGE_CONTENT); + writable.writeString(message.getContent(), null); + } if (hasToolCalls) { writable.writeUTF8(LLM_MESSAGE_TOOL_CALLS); writable.startArray(toolCalls.size()); From 2207c46e77a2f1a33948f7a2fc60d566956bd266 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 4 Mar 2026 16:55:53 +0100 Subject: [PATCH 16/28] Always set stream param to produce the same request body to be aligned with python openai instrumentation and system-tests --- .../trace/instrumentation/openai_java/ResponseDecorator.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index c61c7cc712f..dce941af568 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -455,9 +455,7 @@ private void withResponse(AgentSpan span, Response response, boolean stream) { metadata.put("text", textMap); } - if (stream) { - metadata.put("stream", true); - } + metadata.put("stream", stream); span.setTag(CommonTags.METADATA, metadata); From ca6e2d13649115fe897a7c2cca65131e4c11f912 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 5 Mar 2026 13:41:50 +0100 Subject: [PATCH 17/28] Add OpenAI prompt-tracking reconstruction for responses (input.prompt with variables + chat_template, longest-first overlap handling) and support map-based LLM input serialization (messages + prompt) in LLMObs mapper. Also filter empty instruction messages to match system-test expectations. --- .../openai_java/CommonTags.java | 1 + .../openai_java/ResponseDecorator.java | 365 +++++++++++++++++- .../writer/ddintake/LLMObsSpanMapper.java | 151 +++++--- 3 files changed, 451 insertions(+), 66 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java index 0e550437026..228b3d52a81 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java @@ -39,4 +39,5 @@ interface CommonTags { String CACHE_READ_INPUT_TOKENS = METRIC_PREFIX + "cache_read_input_tokens"; String REQUEST_REASONING = "_ml_obs_request.reasoning"; + String REQUEST_PROMPT = "_ml_obs_request.prompt"; } diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index dce941af568..3b7e4b50a2a 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -7,6 +7,7 @@ import com.openai.models.responses.Response; import com.openai.models.responses.ResponseCreateParams; import com.openai.models.responses.ResponseCustomToolCall; +import com.openai.models.responses.EasyInputMessage; import com.openai.models.responses.ResponseFunctionToolCall; import com.openai.models.responses.ResponseInputContent; import com.openai.models.responses.ResponseInputItem; @@ -15,6 +16,7 @@ import com.openai.models.responses.ResponseOutputText; import com.openai.models.responses.ResponseReasoningItem; import com.openai.models.responses.ResponseStreamEvent; +import com.openai.models.responses.ResponsePrompt; import datadog.json.JsonWriter; import datadog.trace.api.Config; import datadog.trace.api.llmobs.LLMObs; @@ -24,6 +26,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -32,6 +35,8 @@ public class ResponseDecorator { public static final ResponseDecorator DECORATE = new ResponseDecorator(); private static final CharSequence RESPONSES_CREATE = UTF8BytesString.create("createResponse"); + private static final String IMAGE_FALLBACK_MARKER = "[image]"; + private static final String FILE_FALLBACK_MARKER = "[file]"; private final boolean llmObsEnabled = Config.get().isLlmObsEnabled(); @@ -111,13 +116,26 @@ public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params extractReasoningFromParams(params) .ifPresent(reasoningMap -> span.setTag(CommonTags.REQUEST_REASONING, reasoningMap)); + + extractPromptFromParams(params).ifPresent(prompt -> span.setTag(CommonTags.REQUEST_PROMPT, prompt)); } private LLMObs.LLMMessage extractInputItemMessage(ResponseInputItem item) { - if (item.isMessage()) { + if (item.isEasyInputMessage()) { + EasyInputMessage message = item.asEasyInputMessage(); + String role = message.role().asString(); + String content = extractEasyInputMessageContent(message); + if (content == null || content.isEmpty()) { + return null; + } + return LLMObs.LLMMessage.from(role, content); + } else if (item.isMessage()) { ResponseInputItem.Message message = item.asMessage(); String role = message.role().asString(); String content = extractInputMessageContent(message); + if (content == null || content.isEmpty()) { + return null; + } return LLMObs.LLMMessage.from(role, content); } else if (item.isFunctionCall()) { // Function call is mapped to assistant message with tool_calls @@ -139,6 +157,26 @@ private LLMObs.LLMMessage extractInputItemMessage(ResponseInputItem item) { return null; } + private String extractEasyInputMessageContent(EasyInputMessage message) { + if (message.content().isTextInput()) { + String content = message.content().asTextInput(); + return content == null || content.isEmpty() ? null : content; + } + + if (message.content().isResponseInputMessageContentList()) { + StringBuilder contentBuilder = new StringBuilder(); + for (ResponseInputContent content : message.content().asResponseInputMessageContentList()) { + String contentPart = extractInputContentText(content); + if (contentPart != null) { + contentBuilder.append(contentPart); + } + } + String result = contentBuilder.toString(); + return result.isEmpty() ? null : result; + } + return null; + } + private LLMObs.LLMMessage extractMessageFromRawJson(JsonValue jsonValue) { Optional> objOpt = jsonValue.asObject(); if (!objOpt.isPresent()) { @@ -324,14 +362,35 @@ private String removeQuotes(String str) { private String extractInputMessageContent(ResponseInputItem.Message message) { StringBuilder contentBuilder = new StringBuilder(); for (ResponseInputContent content : message.content()) { - if (content.isInputText()) { - contentBuilder.append(content.asInputText().text()); + String contentPart = extractInputContentText(content); + if (contentPart != null) { + contentBuilder.append(contentPart); } } String result = contentBuilder.toString(); return result.isEmpty() ? null : result; } + private String extractInputContentText(ResponseInputContent content) { + if (content.isInputText()) { + return content.asInputText().text(); + } + if (content.isInputImage()) { + return content.asInputImage().imageUrl().orElse(content.asInputImage().fileId().orElse("")); + } + if (content.isInputFile()) { + return content + .asInputFile() + .fileUrl() + .orElse( + content + .asInputFile() + .fileId() + .orElse(content.asInputFile().filename().orElse(FILE_FALLBACK_MARKER))); + } + return null; + } + private Optional> extractReasoningFromParams(ResponseCreateParams params) { JsonField reasoningField = params._reasoning(); if (reasoningField.isMissing()) { @@ -399,6 +458,8 @@ private void withResponse(AgentSpan span, Response response, boolean stream) { span.setTag(CommonTags.OUTPUT, outputMessages); } + enrichInputWithPromptTracking(span, response); + Map metadata = new HashMap<>(); Object reasoningTag = span.getTag(CommonTags.REQUEST_REASONING); @@ -474,6 +535,304 @@ private void withResponse(AgentSpan span, Response response, boolean stream) { }); } + private void enrichInputWithPromptTracking(AgentSpan span, Response response) { + Object promptTag = span.getTag(CommonTags.REQUEST_PROMPT); + if (!(promptTag instanceof Map)) { + return; + } + + Map prompt = new LinkedHashMap<>((Map) promptTag); + Map variables = Collections.emptyMap(); + Object variablesTag = prompt.get("variables"); + if (variablesTag instanceof Map) { + variables = (Map) variablesTag; + } + + Map inputMap = new LinkedHashMap<>(); + Object inputTag = span.getTag(CommonTags.INPUT); + if (inputTag instanceof Map) { + inputMap.putAll((Map) inputTag); + } + + List inputMessages = extractInputMessagesForPromptTracking(span, response); + if (!inputMessages.isEmpty()) { + inputMap.put("messages", inputMessages); + } + + List> chatTemplate = extractChatTemplate(inputMessages, variables); + if (!chatTemplate.isEmpty()) { + prompt.put("chat_template", chatTemplate); + } + + inputMap.put("prompt", prompt); + + span.setTag(CommonTags.INPUT, inputMap); + } + + private List> extractChatTemplate( + List messages, Map variables) { + Map valueToPlaceholder = new LinkedHashMap<>(); + for (Map.Entry variable : variables.entrySet()) { + if (variable.getValue() == null) { + continue; + } + String valueStr = String.valueOf(variable.getValue()); + if (valueStr.isEmpty() + || IMAGE_FALLBACK_MARKER.equals(valueStr) + || FILE_FALLBACK_MARKER.equals(valueStr)) { + continue; + } + valueToPlaceholder.put(valueStr, "{{" + variable.getKey() + "}}"); + } + + List sortedValues = new ArrayList<>(valueToPlaceholder.keySet()); + sortedValues.sort((a, b) -> Integer.compare(b.length(), a.length())); + + List> chatTemplate = new ArrayList<>(); + for (LLMObs.LLMMessage message : messages) { + String role = message.getRole(); + String content = message.getContent(); + if (role == null || role.isEmpty() || content == null || content.isEmpty()) { + continue; + } + + String templateContent = content; + for (String value : sortedValues) { + templateContent = templateContent.replace(value, valueToPlaceholder.get(value)); + } + + Map messageMap = new LinkedHashMap<>(); + messageMap.put("role", role); + messageMap.put("content", templateContent); + chatTemplate.add(messageMap); + } + return chatTemplate; + } + + private List extractInputMessagesForPromptTracking( + AgentSpan span, Response response) { + List messages = new ArrayList<>(); + + Object inputTag = span.getTag(CommonTags.INPUT); + if (inputTag instanceof List) { + for (Object messageObj : (List) inputTag) { + if (messageObj instanceof LLMObs.LLMMessage) { + messages.add((LLMObs.LLMMessage) messageObj); + } + } + } else if (inputTag instanceof Map) { + Object messagesObj = ((Map) inputTag).get("messages"); + if (messagesObj instanceof List) { + for (Object messageObj : (List) messagesObj) { + if (messageObj instanceof LLMObs.LLMMessage) { + messages.add((LLMObs.LLMMessage) messageObj); + } + } + } + } + + if (!messages.isEmpty()) { + return messages; + } + + response + .instructions() + .ifPresent( + instructions -> { + if (instructions.isInputItemList()) { + for (ResponseInputItem item : instructions.asInputItemList()) { + LLMObs.LLMMessage message = extractInputItemMessage(item); + if (message != null) { + messages.add(message); + } + } + } else if (instructions.isString()) { + String text = instructions.asString(); + if (text != null && !text.isEmpty()) { + messages.add(LLMObs.LLMMessage.from("user", text)); + } + } + }); + + if (!messages.isEmpty()) { + return messages; + } + + // Fallback for SDK union parsing mismatches: parse raw instructions payload. + Optional rawInstructions = response._instructions().asUnknown(); + if (rawInstructions.isPresent()) { + Optional> rawList = rawInstructions.get().asArray(); + if (rawList.isPresent()) { + for (JsonValue item : rawList.get()) { + LLMObs.LLMMessage message = extractMessageFromRawInstruction(item); + if (message != null) { + messages.add(message); + } + } + } + } + + return messages; + } + + private LLMObs.LLMMessage extractMessageFromRawInstruction(JsonValue instructionValue) { + Optional> objOpt = instructionValue.asObject(); + if (!objOpt.isPresent()) { + return null; + } + Map obj = objOpt.get(); + String role = getJsonString(obj.get("role")); + if (role == null || role.isEmpty()) { + return null; + } + + JsonValue contentValue = obj.get("content"); + if (contentValue == null) { + return null; + } + Optional> contentList = contentValue.asArray(); + if (!contentList.isPresent()) { + return null; + } + + StringBuilder contentBuilder = new StringBuilder(); + for (JsonValue contentItem : contentList.get()) { + Optional> contentObjOpt = contentItem.asObject(); + if (!contentObjOpt.isPresent()) { + continue; + } + Map contentObj = contentObjOpt.get(); + String type = getJsonString(contentObj.get("type")); + if ("input_text".equals(type)) { + String text = getJsonString(contentObj.get("text")); + if (text != null) { + contentBuilder.append(text); + } + } else if ("input_image".equals(type)) { + String imageUrl = getJsonString(contentObj.get("image_url")); + if (imageUrl != null && !imageUrl.isEmpty()) { + contentBuilder.append(imageUrl); + } else { + String fileId = getJsonString(contentObj.get("file_id")); + contentBuilder.append(fileId == null || fileId.isEmpty() ? IMAGE_FALLBACK_MARKER : fileId); + } + } else if ("input_file".equals(type)) { + String fileUrl = getJsonString(contentObj.get("file_url")); + if (fileUrl != null && !fileUrl.isEmpty()) { + contentBuilder.append(fileUrl); + } else { + String fileId = getJsonString(contentObj.get("file_id")); + if (fileId != null && !fileId.isEmpty()) { + contentBuilder.append(fileId); + } else { + String filename = getJsonString(contentObj.get("filename")); + contentBuilder.append( + filename == null || filename.isEmpty() ? FILE_FALLBACK_MARKER : filename); + } + } + } + } + + String content = contentBuilder.toString(); + if (content.isEmpty()) { + return null; + } + return LLMObs.LLMMessage.from(role, content); + } + + private Optional> extractPromptFromParams(ResponseCreateParams params) { + Optional promptOpt = params.prompt(); + if (!promptOpt.isPresent()) { + return Optional.empty(); + } + + ResponsePrompt prompt = promptOpt.get(); + Map promptMap = new LinkedHashMap<>(); + + String id = prompt.id(); + if (id != null && !id.isEmpty()) { + promptMap.put("id", id); + } + prompt.version().ifPresent(version -> promptMap.put("version", version)); + prompt + .variables() + .ifPresent( + variables -> { + Map normalized = normalizePromptVariables(variables); + if (!normalized.isEmpty()) { + promptMap.put("variables", normalized); + } + }); + + return promptMap.isEmpty() ? Optional.empty() : Optional.of(promptMap); + } + + private Map normalizePromptVariables(ResponsePrompt.Variables variables) { + Map normalized = new LinkedHashMap<>(); + for (Map.Entry entry : variables._additionalProperties().entrySet()) { + Object value = normalizePromptVariable(entry.getValue()); + if (value != null) { + normalized.put(entry.getKey(), value); + } + } + return normalized; + } + + private Object normalizePromptVariable(JsonValue value) { + if (value == null) { + return null; + } + + Optional asString = value.asString(); + if (asString.isPresent()) { + return asString.get(); + } + + Optional> asObject = value.asObject(); + if (!asObject.isPresent()) { + return value.toString(); + } + + Map obj = asObject.get(); + String type = getJsonString(obj.get("type")); + String text = getJsonString(obj.get("text")); + if (text != null && !text.isEmpty()) { + return text; + } + + if ("input_image".equals(type)) { + String imageUrl = getJsonString(obj.get("image_url")); + if (imageUrl != null && !imageUrl.isEmpty()) { + return imageUrl; + } + String fileId = getJsonString(obj.get("file_id")); + return fileId == null || fileId.isEmpty() ? IMAGE_FALLBACK_MARKER : fileId; + } + + if ("input_file".equals(type)) { + String fileUrl = getJsonString(obj.get("file_url")); + if (fileUrl != null && !fileUrl.isEmpty()) { + return fileUrl; + } + String fileId = getJsonString(obj.get("file_id")); + if (fileId != null && !fileId.isEmpty()) { + return fileId; + } + String filename = getJsonString(obj.get("filename")); + return filename == null || filename.isEmpty() ? FILE_FALLBACK_MARKER : filename; + } + + return value.toString(); + } + + private String getJsonString(JsonValue value) { + if (value == null) { + return null; + } + Optional asString = value.asString(); + return asString.orElse(null); + } + private List extractResponseOutputMessages(List output) { List messages = new ArrayList<>(); diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index b4dba725406..7ff16cbed4e 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -337,74 +337,19 @@ public void accept(Metadata metadata) { Object val = tag.getValue(); if (key.equals(INPUT) || key.equals(OUTPUT)) { if (spanKind.equals(Tags.LLMOBS_LLM_SPAN_KIND)) { - if (!(val instanceof List)) { + writable.writeString(key, null); + if (val instanceof List) { + writable.startMap(1); + writable.writeString("messages", null); + writeLlmMessages((List) val); + } else if (key.equals(INPUT) && val instanceof Map) { + writeLlmInputMap((Map) val); + } else { LOGGER.warn( "unexpectedly found incorrect type for LLM span IO {}, expecting list", val.getClass().getName()); continue; } - writable.writeString(key, null); - writable.startMap(1); - // llm span kind must have llm objects - List messages = (List) val; - writable.writeString("messages", null); - writable.startArray(messages.size()); - for (LLMObs.LLMMessage message : messages) { - List toolCalls = message.getToolCalls(); - List toolResults = message.getToolResults(); - boolean hasToolCalls = null != toolCalls && !toolCalls.isEmpty(); - boolean hasToolResults = null != toolResults && !toolResults.isEmpty(); - boolean hasContent = message.getContent() != null; - int mapSize = 1; // role - if (hasContent) mapSize++; - if (hasToolCalls) mapSize++; - if (hasToolResults) mapSize++; - writable.startMap(mapSize); - writable.writeUTF8(LLM_MESSAGE_ROLE); - writable.writeString(message.getRole(), null); - if (hasContent) { - writable.writeUTF8(LLM_MESSAGE_CONTENT); - writable.writeString(message.getContent(), null); - } - if (hasToolCalls) { - writable.writeUTF8(LLM_MESSAGE_TOOL_CALLS); - writable.startArray(toolCalls.size()); - for (LLMObs.ToolCall toolCall : toolCalls) { - Map arguments = toolCall.getArguments(); - boolean hasArguments = null != arguments && !arguments.isEmpty(); - writable.startMap(hasArguments ? 4 : 3); - writable.writeUTF8(LLM_TOOL_CALL_NAME); - writable.writeString(toolCall.getName(), null); - writable.writeUTF8(LLM_TOOL_CALL_TYPE); - writable.writeString(toolCall.getType(), null); - writable.writeUTF8(LLM_TOOL_CALL_TOOL_ID); - writable.writeString(toolCall.getToolId(), null); - if (hasArguments) { - writable.writeUTF8(LLM_TOOL_CALL_ARGUMENTS); - writable.startMap(arguments.size()); - for (Map.Entry argument : arguments.entrySet()) { - writable.writeString(argument.getKey(), null); - writable.writeObject(argument.getValue(), null); - } - } - } - } - if (hasToolResults) { - writable.writeUTF8(LLM_MESSAGE_TOOL_RESULTS); - writable.startArray(toolResults.size()); - for (LLMObs.ToolResult toolResult : toolResults) { - writable.startMap(4); - writable.writeUTF8(LLM_TOOL_CALL_NAME); - writable.writeString(toolResult.getName(), null); - writable.writeUTF8(LLM_TOOL_CALL_TYPE); - writable.writeString(toolResult.getType(), null); - writable.writeUTF8(LLM_TOOL_CALL_TOOL_ID); - writable.writeString(toolResult.getToolId(), null); - writable.writeUTF8(LLM_TOOL_RESULT_RESULT); - writable.writeString(toolResult.getResult(), null); - } - } - } } else if (spanKind.equals(Tags.LLMOBS_EMBEDDING_SPAN_KIND) && key.equals(INPUT)) { if (!(val instanceof List)) { LOGGER.warn( @@ -442,6 +387,86 @@ public void accept(Metadata metadata) { } } } + + private void writeLlmInputMap(Map inputMap) { + writable.startMap(inputMap.size()); + for (Map.Entry entry : inputMap.entrySet()) { + String inputKey = String.valueOf(entry.getKey()); + Object inputValue = entry.getValue(); + writable.writeString(inputKey, null); + if ("messages".equals(inputKey) && inputValue instanceof List) { + writeLlmMessages((List) inputValue); + } else { + writable.writeObject(inputValue, null); + } + } + } + + private void writeLlmMessages(List messages) { + writable.startArray(messages.size()); + for (Object messageObj : messages) { + if (!(messageObj instanceof LLMObs.LLMMessage)) { + writable.writeObject(messageObj, null); + continue; + } + + LLMObs.LLMMessage message = (LLMObs.LLMMessage) messageObj; + List toolCalls = message.getToolCalls(); + List toolResults = message.getToolResults(); + boolean hasToolCalls = null != toolCalls && !toolCalls.isEmpty(); + boolean hasToolResults = null != toolResults && !toolResults.isEmpty(); + boolean hasContent = message.getContent() != null; + int mapSize = 1; + if (hasContent) mapSize++; + if (hasToolCalls) mapSize++; + if (hasToolResults) mapSize++; + writable.startMap(mapSize); + writable.writeUTF8(LLM_MESSAGE_ROLE); + writable.writeString(message.getRole(), null); + if (hasContent) { + writable.writeUTF8(LLM_MESSAGE_CONTENT); + writable.writeString(message.getContent(), null); + } + if (hasToolCalls) { + writable.writeUTF8(LLM_MESSAGE_TOOL_CALLS); + writable.startArray(toolCalls.size()); + for (LLMObs.ToolCall toolCall : toolCalls) { + Map arguments = toolCall.getArguments(); + boolean hasArguments = null != arguments && !arguments.isEmpty(); + writable.startMap(hasArguments ? 4 : 3); + writable.writeUTF8(LLM_TOOL_CALL_NAME); + writable.writeString(toolCall.getName(), null); + writable.writeUTF8(LLM_TOOL_CALL_TYPE); + writable.writeString(toolCall.getType(), null); + writable.writeUTF8(LLM_TOOL_CALL_TOOL_ID); + writable.writeString(toolCall.getToolId(), null); + if (hasArguments) { + writable.writeUTF8(LLM_TOOL_CALL_ARGUMENTS); + writable.startMap(arguments.size()); + for (Map.Entry argument : arguments.entrySet()) { + writable.writeString(argument.getKey(), null); + writable.writeObject(argument.getValue(), null); + } + } + } + } + if (hasToolResults) { + writable.writeUTF8(LLM_MESSAGE_TOOL_RESULTS); + writable.startArray(toolResults.size()); + for (LLMObs.ToolResult toolResult : toolResults) { + writable.startMap(4); + writable.writeUTF8(LLM_TOOL_CALL_NAME); + writable.writeString(toolResult.getName(), null); + writable.writeUTF8(LLM_TOOL_CALL_TYPE); + writable.writeString(toolResult.getType(), null); + writable.writeUTF8(LLM_TOOL_CALL_TOOL_ID); + writable.writeString(toolResult.getToolId(), null); + writable.writeUTF8(LLM_TOOL_RESULT_RESULT); + writable.writeString(toolResult.getResult(), null); + } + } + } + } } private static class PayloadV1 extends Payload { From 7d683b6209d4c2e90e630cf9215a237b01fd7e62 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 5 Mar 2026 14:03:03 +0100 Subject: [PATCH 18/28] Fix OpenAI Responses prompt tracking to use response instructions first and return [image] (not empty) when stripped input_image URLs are missing, aligning mixed-input chat_template output with expected behavior. --- .../openai_java/ResponseDecorator.java | 55 ++++++++++--------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 3b7e4b50a2a..aa334485730 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -4,19 +4,19 @@ import com.openai.core.JsonValue; import com.openai.models.Reasoning; import com.openai.models.ResponsesModel; +import com.openai.models.responses.EasyInputMessage; import com.openai.models.responses.Response; import com.openai.models.responses.ResponseCreateParams; import com.openai.models.responses.ResponseCustomToolCall; -import com.openai.models.responses.EasyInputMessage; import com.openai.models.responses.ResponseFunctionToolCall; import com.openai.models.responses.ResponseInputContent; import com.openai.models.responses.ResponseInputItem; import com.openai.models.responses.ResponseOutputItem; import com.openai.models.responses.ResponseOutputMessage; import com.openai.models.responses.ResponseOutputText; +import com.openai.models.responses.ResponsePrompt; import com.openai.models.responses.ResponseReasoningItem; import com.openai.models.responses.ResponseStreamEvent; -import com.openai.models.responses.ResponsePrompt; import datadog.json.JsonWriter; import datadog.trace.api.Config; import datadog.trace.api.llmobs.LLMObs; @@ -117,7 +117,8 @@ public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params extractReasoningFromParams(params) .ifPresent(reasoningMap -> span.setTag(CommonTags.REQUEST_REASONING, reasoningMap)); - extractPromptFromParams(params).ifPresent(prompt -> span.setTag(CommonTags.REQUEST_PROMPT, prompt)); + extractPromptFromParams(params) + .ifPresent(prompt -> span.setTag(CommonTags.REQUEST_PROMPT, prompt)); } private LLMObs.LLMMessage extractInputItemMessage(ResponseInputItem item) { @@ -376,7 +377,10 @@ private String extractInputContentText(ResponseInputContent content) { return content.asInputText().text(); } if (content.isInputImage()) { - return content.asInputImage().imageUrl().orElse(content.asInputImage().fileId().orElse("")); + return content + .asInputImage() + .imageUrl() + .orElse(content.asInputImage().fileId().orElse(IMAGE_FALLBACK_MARKER)); } if (content.isInputFile()) { return content @@ -613,28 +617,6 @@ private List extractInputMessagesForPromptTracking( AgentSpan span, Response response) { List messages = new ArrayList<>(); - Object inputTag = span.getTag(CommonTags.INPUT); - if (inputTag instanceof List) { - for (Object messageObj : (List) inputTag) { - if (messageObj instanceof LLMObs.LLMMessage) { - messages.add((LLMObs.LLMMessage) messageObj); - } - } - } else if (inputTag instanceof Map) { - Object messagesObj = ((Map) inputTag).get("messages"); - if (messagesObj instanceof List) { - for (Object messageObj : (List) messagesObj) { - if (messageObj instanceof LLMObs.LLMMessage) { - messages.add((LLMObs.LLMMessage) messageObj); - } - } - } - } - - if (!messages.isEmpty()) { - return messages; - } - response .instructions() .ifPresent( @@ -672,6 +654,24 @@ private List extractInputMessagesForPromptTracking( } } + Object inputTag = span.getTag(CommonTags.INPUT); + if (inputTag instanceof List) { + for (Object messageObj : (List) inputTag) { + if (messageObj instanceof LLMObs.LLMMessage) { + messages.add((LLMObs.LLMMessage) messageObj); + } + } + } else if (inputTag instanceof Map) { + Object messagesObj = ((Map) inputTag).get("messages"); + if (messagesObj instanceof List) { + for (Object messageObj : (List) messagesObj) { + if (messageObj instanceof LLMObs.LLMMessage) { + messages.add((LLMObs.LLMMessage) messageObj); + } + } + } + } + return messages; } @@ -714,7 +714,8 @@ private LLMObs.LLMMessage extractMessageFromRawInstruction(JsonValue instruction contentBuilder.append(imageUrl); } else { String fileId = getJsonString(contentObj.get("file_id")); - contentBuilder.append(fileId == null || fileId.isEmpty() ? IMAGE_FALLBACK_MARKER : fileId); + contentBuilder.append( + fileId == null || fileId.isEmpty() ? IMAGE_FALLBACK_MARKER : fileId); } } else if ("input_file".equals(type)) { String fileUrl = getJsonString(contentObj.get("file_url")); From 2c17ddc1a038595f7028142b214388a3d8092eeb Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 5 Mar 2026 14:24:04 +0100 Subject: [PATCH 19/28] Set LLMObs error-path defaults in Java to always emit model_name and output.messages from request params so existing error-span tests pass. --- .../openai_java/ChatCompletionDecorator.java | 14 +++++++++----- .../openai_java/CompletionDecorator.java | 15 ++++++++++----- .../openai_java/EmbeddingDecorator.java | 10 +++++----- .../openai_java/ResponseDecorator.java | 7 +++++++ 4 files changed, 31 insertions(+), 15 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index e5def4f67bb..d5dfde5bd6b 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -38,16 +38,20 @@ public void withChatCompletionCreateParams( if (params == null) { return; } - params - .model() - ._value() - .asString() - .ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); + Optional modelName = params.model()._value().asString(); + modelName.ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); if (!llmObsEnabled) { return; } + // Keep model_name and output shape stable on error paths where no response is available. + modelName.ifPresent( + str -> { + span.setTag(CommonTags.MODEL_NAME, str); + span.setTag(CommonTags.OUTPUT, Collections.singletonList(LLMObs.LLMMessage.from("", ""))); + }); + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); span.setTag( diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java index 1b95491b64b..f0f29386582 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java @@ -11,6 +11,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; public class CompletionDecorator { @@ -27,16 +28,20 @@ public void withCompletionCreateParams(AgentSpan span, CompletionCreateParams pa return; } - params - .model() - ._value() - .asString() - .ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); + Optional modelName = params.model()._value().asString(); + modelName.ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); if (!llmObsEnabled) { return; } + // Keep model_name and output shape stable on error paths where no response is available. + modelName.ifPresent( + str -> { + span.setTag(CommonTags.MODEL_NAME, str); + span.setTag(CommonTags.OUTPUT, Collections.singletonList(LLMObs.LLMMessage.from("", ""))); + }); + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); params .prompt() diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java index 02d4588358c..4e986603d93 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java @@ -28,16 +28,16 @@ public void withEmbeddingCreateParams(AgentSpan span, EmbeddingCreateParams para if (params == null) { return; } - params - .model() - ._value() - .asString() - .ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); + Optional modelName = params.model()._value().asString(); + modelName.ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); if (!llmObsEnabled) { return; } + // Keep model_name stable on error paths where no response is available. + modelName.ifPresent(str -> span.setTag(CommonTags.MODEL_NAME, str)); + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_EMBEDDING_SPAN_KIND); span.setTag(CommonTags.INPUT, embeddingDocuments(params.input())); diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index aa334485730..b91ccf6c954 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -56,6 +56,13 @@ public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params return; } + // Keep model_name/output/metadata shape stable on error paths where no response is available. + if (modelName != null && !modelName.isEmpty()) { + span.setTag(CommonTags.MODEL_NAME, modelName); + } + span.setTag(CommonTags.OUTPUT, Collections.singletonList(LLMObs.LLMMessage.from("", ""))); + span.setTag(CommonTags.METADATA, new HashMap()); + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); List inputMessages = new ArrayList<>(); From ad3b782f56e40eb3ae8e4a99e42e54d7b2aca510 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 5 Mar 2026 16:05:00 +0100 Subject: [PATCH 20/28] Add OpenAI Responses tool definition extraction to populate LLMObs tool_definitions tags --- .../openai_java/ResponseDecorator.java | 158 ++++++++++++++++++ 1 file changed, 158 insertions(+) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index b91ccf6c954..3ca06d74f10 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -17,6 +17,8 @@ import com.openai.models.responses.ResponsePrompt; import com.openai.models.responses.ResponseReasoningItem; import com.openai.models.responses.ResponseStreamEvent; +import com.openai.models.responses.FunctionTool; +import com.openai.models.responses.Tool; import datadog.json.JsonWriter; import datadog.trace.api.Config; import datadog.trace.api.llmobs.LLMObs; @@ -126,6 +128,123 @@ public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params extractPromptFromParams(params) .ifPresent(prompt -> span.setTag(CommonTags.REQUEST_PROMPT, prompt)); + + List> toolDefinitions = extractToolDefinitionsFromParams(params); + if (!toolDefinitions.isEmpty()) { + span.setTag(CommonTags.TOOL_DEFINITIONS, toolDefinitions); + } + } + + private List> extractToolDefinitionsFromParams(ResponseCreateParams params) { + try { + Optional> toolsOpt = params.tools(); + if (toolsOpt.isPresent()) { + List> toolDefinitions = new ArrayList<>(); + for (Tool tool : toolsOpt.get()) { + if (!tool.isFunction()) { + continue; + } + Map toolDef = extractFunctionToolDefinition(tool.asFunction()); + if (toolDef != null) { + toolDefinitions.add(toolDef); + } + } + if (!toolDefinitions.isEmpty()) { + return toolDefinitions; + } + } + } catch (Throwable ignored) { + // fall back to raw JSON if typed extraction is unavailable or fails + } + + try { + Optional rawToolsOpt = params._tools().asUnknown(); + if (!rawToolsOpt.isPresent()) { + return Collections.emptyList(); + } + Optional> rawToolListOpt = rawToolsOpt.get().asArray(); + if (!rawToolListOpt.isPresent()) { + return Collections.emptyList(); + } + + List> toolDefinitions = new ArrayList<>(); + for (JsonValue rawTool : rawToolListOpt.get()) { + Map toolDef = extractFunctionToolDefinition(rawTool); + if (toolDef != null) { + toolDefinitions.add(toolDef); + } + } + return toolDefinitions; + } catch (Throwable ignored) { + return Collections.emptyList(); + } + } + + private Map extractFunctionToolDefinition(FunctionTool functionTool) { + String name = functionTool.name(); + if (name == null || name.isEmpty()) { + return null; + } + + Map toolDef = new HashMap<>(); + toolDef.put("name", name); + functionTool.description().ifPresent(desc -> toolDef.put("description", desc)); + functionTool + .parameters() + .ifPresent(parameters -> toolDef.put("schema", jsonValueMapToObject(parameters._additionalProperties()))); + return toolDef; + } + + private Map extractFunctionToolDefinition(JsonValue rawTool) { + Optional> toolObjOpt = rawTool.asObject(); + if (!toolObjOpt.isPresent()) { + return null; + } + + Map toolObj = toolObjOpt.get(); + String type = getJsonString(toolObj.get("type")); + if (!"function".equals(type)) { + return null; + } + + JsonValue functionObjValue = toolObj.get("function"); + Map functionObj = null; + if (functionObjValue != null) { + Optional> nestedFunctionOpt = functionObjValue.asObject(); + if (nestedFunctionOpt.isPresent()) { + functionObj = nestedFunctionOpt.get(); + } + } + + String name = + functionObj == null + ? getJsonString(toolObj.get("name")) + : getJsonString(functionObj.get("name")); + if (name == null || name.isEmpty()) { + return null; + } + + Map toolDef = new HashMap<>(); + toolDef.put("name", name); + + String description = + functionObj == null + ? getJsonString(toolObj.get("description")) + : getJsonString(functionObj.get("description")); + if (description != null) { + toolDef.put("description", description); + } + + JsonValue parameters = + functionObj == null ? toolObj.get("parameters") : functionObj.get("parameters"); + if (parameters != null) { + Object schema = jsonValueToObject(parameters); + if (schema != null) { + toolDef.put("schema", schema); + } + } + + return toolDef; } private LLMObs.LLMMessage extractInputItemMessage(ResponseInputItem item) { @@ -841,6 +960,45 @@ private String getJsonString(JsonValue value) { return asString.orElse(null); } + private Map jsonValueMapToObject(Map map) { + Map result = new HashMap<>(); + for (Map.Entry entry : map.entrySet()) { + result.put(entry.getKey(), jsonValueToObject(entry.getValue())); + } + return result; + } + + private Object jsonValueToObject(JsonValue value) { + if (value == null) { + return null; + } + Optional str = value.asString(); + if (str.isPresent()) { + return str.get(); + } + Optional num = value.asNumber(); + if (num.isPresent()) { + return num.get(); + } + Optional bool = value.asBoolean(); + if (bool.isPresent()) { + return bool.get(); + } + Optional> obj = value.asObject(); + if (obj.isPresent()) { + return jsonValueMapToObject(obj.get()); + } + Optional> arr = value.asArray(); + if (arr.isPresent()) { + List list = new ArrayList<>(); + for (JsonValue item : arr.get()) { + list.add(jsonValueToObject(item)); + } + return list; + } + return null; + } + private List extractResponseOutputMessages(List output) { List messages = new ArrayList<>(); From 1810327aef053000f2a6ffefc54b2cbf39a8aee5 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 5 Mar 2026 16:13:58 +0100 Subject: [PATCH 21/28] Fix ChatCompletionServiceTest --- .../src/test/groovy/ChatCompletionServiceTest.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy index 22e7fbbb579..580c0ecd262 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy @@ -6,7 +6,6 @@ import com.openai.core.http.HttpResponseFor import com.openai.core.http.StreamResponse import com.openai.models.chat.completions.ChatCompletion import com.openai.models.chat.completions.ChatCompletionChunk -import com.openai.models.completions.Completion import datadog.trace.api.DDSpanTypes import datadog.trace.api.llmobs.LLMObs import datadog.trace.bootstrap.instrumentation.api.Tags @@ -94,7 +93,7 @@ class ChatCompletionServiceTest extends OpenAiTest { } def "create async chat/completion test withRawResponse"() { - CompletableFuture> completionFuture = runUnderTrace("parent") { + CompletableFuture> completionFuture = runUnderTrace("parent") { openAiClient.async().chat().completions().withRawResponse().create(params) } From 46221e411d13675090941c7fb15c38d74401eda1 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 5 Mar 2026 16:25:04 +0100 Subject: [PATCH 22/28] Extract JsonValueUtils --- .../openai_java/ChatCompletionDecorator.java | 39 ++------------ .../openai_java/ChatCompletionModule.java | 1 + .../openai_java/JsonValueUtils.java | 51 +++++++++++++++++++ .../openai_java/ResponseDecorator.java | 42 ++------------- .../openai_java/ResponseModule.java | 1 + 5 files changed, 59 insertions(+), 75 deletions(-) create mode 100644 dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/JsonValueUtils.java diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index d5dfde5bd6b..f35e485c90e 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -1,5 +1,8 @@ package datadog.trace.instrumentation.openai_java; +import static datadog.trace.instrumentation.openai_java.JsonValueUtils.jsonValueMapToObject; +import static datadog.trace.instrumentation.openai_java.JsonValueUtils.jsonValueToObject; + import com.openai.core.JsonValue; import com.openai.helpers.ChatCompletionAccumulator; import com.openai.models.FunctionDefinition; @@ -167,42 +170,6 @@ private static Map extractFunctionToolDef(ChatCompletionFunction return toolDef; } - private static Map jsonValueMapToObject(Map map) { - Map result = new HashMap<>(); - for (Map.Entry entry : map.entrySet()) { - result.put(entry.getKey(), jsonValueToObject(entry.getValue())); - } - return result; - } - - private static Object jsonValueToObject(JsonValue value) { - Optional str = value.asString(); - if (str.isPresent()) { - return str.get(); - } - Optional num = value.asNumber(); - if (num.isPresent()) { - return num.get(); - } - Optional bool = value.asBoolean(); - if (bool.isPresent()) { - return bool.get(); - } - Optional> obj = value.asObject(); - if (obj.isPresent()) { - return jsonValueMapToObject(obj.get()); - } - Optional> arr = value.asArray(); - if (arr.isPresent()) { - List list = new ArrayList<>(); - for (JsonValue item : arr.get()) { - list.add(jsonValueToObject(item)); - } - return list; - } - return null; - } - private static LLMObs.LLMMessage llmMessage(ChatCompletionMessageParam m) { String role = "unknown"; String content = null; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionModule.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionModule.java index f30d02f9570..2c89bf339ad 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionModule.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionModule.java @@ -18,6 +18,7 @@ public String[] helperClassNames() { packageName + ".CommonTags", packageName + ".ChatCompletionDecorator", packageName + ".OpenAiDecorator", + packageName + ".JsonValueUtils", packageName + ".HttpResponseWrapper", packageName + ".HttpStreamResponseWrapper", packageName + ".HttpStreamResponseStreamWrapper", diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/JsonValueUtils.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/JsonValueUtils.java new file mode 100644 index 00000000000..c251f5c17fd --- /dev/null +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/JsonValueUtils.java @@ -0,0 +1,51 @@ +package datadog.trace.instrumentation.openai_java; + +import com.openai.core.JsonValue; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +public final class JsonValueUtils { + private JsonValueUtils() {} + + public static Map jsonValueMapToObject(Map map) { + Map result = new HashMap<>(); + for (Map.Entry entry : map.entrySet()) { + result.put(entry.getKey(), jsonValueToObject(entry.getValue())); + } + return result; + } + + public static Object jsonValueToObject(JsonValue value) { + if (value == null) { + return null; + } + Optional str = value.asString(); + if (str.isPresent()) { + return str.get(); + } + Optional num = value.asNumber(); + if (num.isPresent()) { + return num.get(); + } + Optional bool = value.asBoolean(); + if (bool.isPresent()) { + return bool.get(); + } + Optional> obj = value.asObject(); + if (obj.isPresent()) { + return jsonValueMapToObject(obj.get()); + } + Optional> arr = value.asArray(); + if (arr.isPresent()) { + List list = new ArrayList<>(); + for (JsonValue item : arr.get()) { + list.add(jsonValueToObject(item)); + } + return list; + } + return null; + } +} diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 3ca06d74f10..631d5d213e1 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -1,5 +1,8 @@ package datadog.trace.instrumentation.openai_java; +import static datadog.trace.instrumentation.openai_java.JsonValueUtils.jsonValueMapToObject; +import static datadog.trace.instrumentation.openai_java.JsonValueUtils.jsonValueToObject; + import com.openai.core.JsonField; import com.openai.core.JsonValue; import com.openai.models.Reasoning; @@ -960,45 +963,6 @@ private String getJsonString(JsonValue value) { return asString.orElse(null); } - private Map jsonValueMapToObject(Map map) { - Map result = new HashMap<>(); - for (Map.Entry entry : map.entrySet()) { - result.put(entry.getKey(), jsonValueToObject(entry.getValue())); - } - return result; - } - - private Object jsonValueToObject(JsonValue value) { - if (value == null) { - return null; - } - Optional str = value.asString(); - if (str.isPresent()) { - return str.get(); - } - Optional num = value.asNumber(); - if (num.isPresent()) { - return num.get(); - } - Optional bool = value.asBoolean(); - if (bool.isPresent()) { - return bool.get(); - } - Optional> obj = value.asObject(); - if (obj.isPresent()) { - return jsonValueMapToObject(obj.get()); - } - Optional> arr = value.asArray(); - if (arr.isPresent()) { - List list = new ArrayList<>(); - for (JsonValue item : arr.get()) { - list.add(jsonValueToObject(item)); - } - return list; - } - return null; - } - private List extractResponseOutputMessages(List output) { List messages = new ArrayList<>(); diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseModule.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseModule.java index 25266504f53..b87b3910490 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseModule.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseModule.java @@ -19,6 +19,7 @@ public String[] helperClassNames() { packageName + ".ResponseDecorator", packageName + ".FunctionCallOutputExtractor", packageName + ".OpenAiDecorator", + packageName + ".JsonValueUtils", packageName + ".HttpResponseWrapper", packageName + ".HttpStreamResponseWrapper", packageName + ".HttpStreamResponseStreamWrapper", From 61ad6678b8fe9d52ad56990195f56b1c0bd1243a Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 5 Mar 2026 16:48:39 +0100 Subject: [PATCH 23/28] Refactor OpenAI responses instrumentation to reuse ToolCallExtractor JSON argument parsing and remove duplicate manual parsing logic from ResponseDecorator. --- .../openai_java/ResponseDecorator.java | 77 +------------------ .../openai_java/ToolCallExtractor.java | 2 +- 2 files changed, 2 insertions(+), 77 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 631d5d213e1..810e8419176 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -352,7 +352,7 @@ private LLMObs.LLMMessage extractMessageFromRawJson(JsonValue jsonValue) { } if (callId != null && name != null && argumentsStr != null) { - Map arguments = parseJsonString(argumentsStr); + Map arguments = ToolCallExtractor.parseArguments(argumentsStr); LLMObs.ToolCall toolCall = LLMObs.ToolCall.from(name, "function_call", callId, arguments); return LLMObs.LLMMessage.from("assistant", null, Collections.singletonList(toolCall)); @@ -414,81 +414,6 @@ private LLMObs.LLMMessage extractMessageFromRawJson(JsonValue jsonValue) { return null; } - private Map parseJsonString(String jsonStr) { - if (jsonStr == null || jsonStr.isEmpty()) { - return Collections.emptyMap(); - } - try { - jsonStr = jsonStr.trim(); - if (!jsonStr.startsWith("{") || !jsonStr.endsWith("}")) { - return Collections.emptyMap(); - } - - Map result = new HashMap<>(); - String content = jsonStr.substring(1, jsonStr.length() - 1).trim(); - - if (content.isEmpty()) { - return result; - } - - // Parse JSON manually, respecting quoted strings - List pairs = splitByCommaRespectingQuotes(content); - - for (String pair : pairs) { - int colonIdx = pair.indexOf(':'); - if (colonIdx > 0) { - String key = pair.substring(0, colonIdx).trim(); - String value = pair.substring(colonIdx + 1).trim(); - - // Remove quotes from key - key = removeQuotes(key); - // Remove quotes from value - value = removeQuotes(value); - - result.put(key, value); - } - } - - return result; - } catch (Exception e) { - return Collections.emptyMap(); - } - } - - private List splitByCommaRespectingQuotes(String str) { - List result = new ArrayList<>(); - StringBuilder current = new StringBuilder(); - boolean inQuotes = false; - - for (int i = 0; i < str.length(); i++) { - char c = str.charAt(i); - - if (c == '"') { - inQuotes = !inQuotes; - current.append(c); - } else if (c == ',' && !inQuotes) { - result.add(current.toString()); - current = new StringBuilder(); - } else { - current.append(c); - } - } - - if (current.length() > 0) { - result.add(current.toString()); - } - - return result; - } - - private String removeQuotes(String str) { - str = str.trim(); - if (str.startsWith("\"") && str.endsWith("\"") && str.length() >= 2) { - return str.substring(1, str.length() - 1); - } - return str; - } - private String extractInputMessageContent(ResponseInputItem.Message message) { StringBuilder contentBuilder = new StringBuilder(); for (ResponseInputContent content : message.content()) { diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java index 21d2bf6835d..ffeca857a20 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java @@ -106,7 +106,7 @@ public static LLMObs.ToolCall getToolCall(McpCall mcpCall) { return null; } - private static Map parseArguments(String argumentsJson) { + static Map parseArguments(String argumentsJson) { try { return MAPPER.readValue(argumentsJson, MAP_TYPE_REF); } catch (Exception e) { From f0957b79844420163b73cf302fd1faa07afa6f53 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 5 Mar 2026 19:04:37 +0100 Subject: [PATCH 24/28] Fix test assertions --- .../openai_java/ResponseDecorator.java | 6 ++++-- .../groovy/ChatCompletionServiceTest.groovy | 19 ++++++++++++++++--- .../test/groovy/CompletionServiceTest.groovy | 2 ++ .../test/groovy/EmbeddingServiceTest.groovy | 2 ++ .../test/groovy/ResponseServiceTest.groovy | 2 ++ 5 files changed, 26 insertions(+), 5 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 810e8419176..07359a9b326 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -8,6 +8,7 @@ import com.openai.models.Reasoning; import com.openai.models.ResponsesModel; import com.openai.models.responses.EasyInputMessage; +import com.openai.models.responses.FunctionTool; import com.openai.models.responses.Response; import com.openai.models.responses.ResponseCreateParams; import com.openai.models.responses.ResponseCustomToolCall; @@ -20,7 +21,6 @@ import com.openai.models.responses.ResponsePrompt; import com.openai.models.responses.ResponseReasoningItem; import com.openai.models.responses.ResponseStreamEvent; -import com.openai.models.responses.FunctionTool; import com.openai.models.responses.Tool; import datadog.json.JsonWriter; import datadog.trace.api.Config; @@ -194,7 +194,9 @@ private Map extractFunctionToolDefinition(FunctionTool functionT functionTool.description().ifPresent(desc -> toolDef.put("description", desc)); functionTool .parameters() - .ifPresent(parameters -> toolDef.put("schema", jsonValueMapToObject(parameters._additionalProperties()))); + .ifPresent( + parameters -> + toolDef.put("schema", jsonValueMapToObject(parameters._additionalProperties()))); return toolDef; } diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy index 580c0ecd262..fad254f69c5 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy @@ -147,7 +147,7 @@ class ChatCompletionServiceTest extends OpenAiTest { expect: List outputTag = [] - assertChatCompletionTrace(false, outputTag, [:]) + assertChatCompletionTrace(false, outputTag, [:], true) and: outputTag.size() == 1 LLMObs.LLMMessage outputMsg = outputTag.get(0) @@ -178,7 +178,7 @@ class ChatCompletionServiceTest extends OpenAiTest { expect: List outputTag = [] - assertChatCompletionTrace(true, outputTag, [stream: true]) + assertChatCompletionTrace(true, outputTag, [stream: true], true) and: outputTag.size() == 1 LLMObs.LLMMessage outputMsg = outputTag.get(0) @@ -294,6 +294,13 @@ class ChatCompletionServiceTest extends OpenAiTest { } private void assertChatCompletionTrace(boolean isStreaming, List outputTagsOut, Map metadata) { + assertChatCompletionTrace(isStreaming, outputTagsOut, metadata, false) + } + + private void assertChatCompletionTrace(boolean isStreaming, List outputTagsOut, Map metadata, boolean expectToolDefinitions) { + def expectedMetadata = new LinkedHashMap(metadata) + expectedMetadata.putIfAbsent("stream", isStreaming) + assertTraces(1) { trace(3) { sortSpansByStart() @@ -312,7 +319,7 @@ class ChatCompletionServiceTest extends OpenAiTest { "_ml_obs_tag.span.kind" "llm" "_ml_obs_tag.model_provider" "openai" "_ml_obs_tag.model_name" String - "_ml_obs_tag.metadata" metadata + "_ml_obs_tag.metadata" expectedMetadata "_ml_obs_tag.input" List "_ml_obs_tag.output" List def outputTags = tag("_ml_obs_tag.output") @@ -324,10 +331,16 @@ class ChatCompletionServiceTest extends OpenAiTest { "_ml_obs_metric.input_tokens" Long "_ml_obs_metric.output_tokens" Long "_ml_obs_metric.total_tokens" Long + "_ml_obs_metric.cache_read_input_tokens" Long } "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String + if (expectToolDefinitions) { + "$CommonTags.TOOL_DEFINITIONS" List + } + "$CommonTags.SOURCE" "integration" + "$CommonTags.ERROR" 0 "openai.request.method" "POST" "openai.request.endpoint" "/v1/chat/completions" "openai.api_base" openAiBaseApi diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy index 43f5e247c55..72ffaee8052 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy @@ -165,6 +165,8 @@ class CompletionServiceTest extends OpenAiTest { "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String + "$CommonTags.SOURCE" "integration" + "$CommonTags.ERROR" 0 "openai.request.method" "POST" "openai.request.endpoint" "/v1/completions" "openai.api_base" openAiBaseApi diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy index eb14f2999de..44999641fc2 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy @@ -64,6 +64,8 @@ class EmbeddingServiceTest extends OpenAiTest { "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String + "$CommonTags.SOURCE" "integration" + "$CommonTags.ERROR" 0 "_ml_obs_metric.input_tokens" Long "_ml_obs_metric.total_tokens" Long "openai.request.method" "POST" diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index f747005cff4..3b205d8496f 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -220,6 +220,8 @@ class ResponseServiceTest extends OpenAiTest { "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String + "$CommonTags.SOURCE" "integration" + "$CommonTags.ERROR" 0 if (reasoning != null) { "_ml_obs_request.reasoning" reasoning } From f3f1f75ec26f06c157090a87f5a64f45df3b720e Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Fri, 6 Mar 2026 10:35:37 +0100 Subject: [PATCH 25/28] Add integration tag --- .../trace/instrumentation/openai_java/CommonTags.java | 1 + .../trace/instrumentation/openai_java/OpenAiDecorator.java | 1 + .../src/test/groovy/ChatCompletionServiceTest.groovy | 1 + .../src/test/groovy/CompletionServiceTest.groovy | 1 + .../src/test/groovy/EmbeddingServiceTest.groovy | 1 + .../src/test/groovy/ResponseServiceTest.groovy | 1 + .../trace/llmobs/writer/ddintake/LLMObsSpanMapper.java | 3 ++- .../llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy | 6 ++++-- 8 files changed, 12 insertions(+), 3 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java index 228b3d52a81..c9917332e7e 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java @@ -19,6 +19,7 @@ interface CommonTags { String MODEL_PROVIDER = TAG_PREFIX + LLMObsTags.MODEL_PROVIDER; String ML_APP = TAG_PREFIX + LLMObsTags.ML_APP; + String INTEGRATION = TAG_PREFIX + "integration"; String VERSION = TAG_PREFIX + "version"; String SOURCE = TAG_PREFIX + "source"; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java index fae2880c082..033381d60bb 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java @@ -99,6 +99,7 @@ public AgentSpan afterStart(AgentSpan span) { span.setTag(CommonTags.ML_APP, Config.get().getLlmObsMlApp()); span.setTag(CommonTags.SOURCE, "integration"); + span.setTag(CommonTags.INTEGRATION, INTEGRATION); AgentSpanContext parent = LLMObsContext.current(); String parentSpanId = LLMObsContext.ROOT_SPAN_ID; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy index fad254f69c5..c4a066d1895 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy @@ -340,6 +340,7 @@ class ChatCompletionServiceTest extends OpenAiTest { "$CommonTags.TOOL_DEFINITIONS" List } "$CommonTags.SOURCE" "integration" + "$CommonTags.INTEGRATION" "openai" "$CommonTags.ERROR" 0 "openai.request.method" "POST" "openai.request.endpoint" "/v1/chat/completions" diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy index 72ffaee8052..de9af838086 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy @@ -166,6 +166,7 @@ class CompletionServiceTest extends OpenAiTest { "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String "$CommonTags.SOURCE" "integration" + "$CommonTags.INTEGRATION" "openai" "$CommonTags.ERROR" 0 "openai.request.method" "POST" "openai.request.endpoint" "/v1/completions" diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy index 44999641fc2..112b649a856 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy @@ -65,6 +65,7 @@ class EmbeddingServiceTest extends OpenAiTest { "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String "$CommonTags.SOURCE" "integration" + "$CommonTags.INTEGRATION" "openai" "$CommonTags.ERROR" 0 "_ml_obs_metric.input_tokens" Long "_ml_obs_metric.total_tokens" Long diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index 3b205d8496f..030f61009ac 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -219,6 +219,7 @@ class ResponseServiceTest extends OpenAiTest { "_ml_obs_metric.cache_read_input_tokens" Long "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String + "$CommonTags.INTEGRATION" "openai" "_ml_obs_tag.service" String "$CommonTags.SOURCE" "integration" "$CommonTags.ERROR" 0 diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index 7ff16cbed4e..cb8ebd3d8a1 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -70,6 +70,7 @@ public class LLMObsSpanMapper implements RemoteMapper { private static final byte[] SPANS = "spans".getBytes(StandardCharsets.UTF_8); private static final byte[] METRICS = "metrics".getBytes(StandardCharsets.UTF_8); private static final byte[] TAGS = "tags".getBytes(StandardCharsets.UTF_8); + private static final String LLMOBS_LANGUAGE_TAG = "language:jvm"; private static final byte[] LLM_MESSAGE_ROLE = "role".getBytes(StandardCharsets.UTF_8); private static final byte[] LLM_MESSAGE_CONTENT = "content".getBytes(StandardCharsets.UTF_8); @@ -293,7 +294,7 @@ public void accept(Metadata metadata) { // write tags (10) writable.writeUTF8(TAGS); writable.startArray(tagsSize + 1); - writable.writeString("language:jvm", null); + writable.writeString(LLMOBS_LANGUAGE_TAG, null); for (Map.Entry tag : metadata.getTags().entrySet()) { String key = tag.getKey(); Object value = tag.getValue(); diff --git a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy index 6140431b836..3cf52ae0150 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy @@ -160,7 +160,8 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { def trace = [regularSpan1, regularSpan2] CapturingByteBufferConsumer sink = new CapturingByteBufferConsumer() - MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(1024, sink)) + // Keep all formatted spans in a single flush for this assertion. + MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(16 * 1024, sink)) when: packer.format(trace, mapper) @@ -204,7 +205,8 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { def trace1 = [llmSpan1, llmSpan2] def trace2 = [llmSpan3] CapturingByteBufferConsumer sink = new CapturingByteBufferConsumer() - MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(1024, sink)) + // Keep all formatted spans in a single flush for this assertion. + MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(16 * 1024, sink)) when: packer.format(trace1, mapper) From 668e955e352ee8306d2f9ea1ce69570e15fe837f Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Fri, 6 Mar 2026 11:01:25 +0100 Subject: [PATCH 26/28] Add ddtrace.verion --- .../java/datadog/trace/llmobs/domain/DDLLMObsSpan.java | 3 +++ .../datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy | 9 +++++++++ .../trace/instrumentation/openai_java/CommonTags.java | 1 + .../instrumentation/openai_java/OpenAiDecorator.java | 2 ++ .../src/test/groovy/ChatCompletionServiceTest.groovy | 1 + .../src/test/groovy/CompletionServiceTest.groovy | 1 + .../src/test/groovy/EmbeddingServiceTest.groovy | 1 + .../src/test/groovy/ResponseServiceTest.groovy | 1 + 8 files changed, 19 insertions(+) diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java index 56bc1f69c88..c7636f9cd7a 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java @@ -2,6 +2,7 @@ import datadog.context.ContextScope; import datadog.trace.api.DDSpanTypes; +import datadog.trace.api.DDTraceApiInfo; import datadog.trace.api.DDTraceId; import datadog.trace.api.WellKnownTags; import datadog.trace.api.llmobs.LLMObs; @@ -38,6 +39,7 @@ public class DDLLMObsSpan implements LLMObsSpan { private static final String SERVICE = LLMOBS_TAG_PREFIX + "service"; private static final String VERSION = LLMOBS_TAG_PREFIX + "version"; + private static final String DDTRACE_VERSION = LLMOBS_TAG_PREFIX + "ddtrace.version"; private static final String ENV = LLMOBS_TAG_PREFIX + "env"; private static final String LLM_OBS_INSTRUMENTATION_NAME = "llmobs"; @@ -74,6 +76,7 @@ public DDLLMObsSpan( this.span.setTag(ENV, wellKnownTags.getEnv()); this.span.setTag(SERVICE, wellKnownTags.getService()); this.span.setTag(VERSION, wellKnownTags.getVersion()); + this.span.setTag(DDTRACE_VERSION, DDTraceApiInfo.VERSION); this.span.setTag(SPAN_KIND, kind); this.spanKind = kind; diff --git a/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy b/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy index 7595b51e82a..87123f9f473 100644 --- a/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy +++ b/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy @@ -2,6 +2,7 @@ package datadog.trace.llmobs.domain import datadog.trace.agent.tooling.TracerInstaller import datadog.trace.api.DDTags +import datadog.trace.api.DDTraceApiInfo import datadog.trace.api.IdGenerationStrategy import datadog.trace.api.WellKnownTags import datadog.trace.api.llmobs.LLMObs @@ -131,6 +132,8 @@ class DDLLMObsSpanTest extends DDSpecification{ def tagVersion = innerSpan.getTag(LLMOBS_TAG_PREFIX + "version") tagVersion instanceof UTF8BytesString "v1" == tagVersion.toString() + + DDTraceApiInfo.VERSION == innerSpan.getTag(LLMOBS_TAG_PREFIX + "ddtrace.version") } def "test span with overwrites"() { @@ -216,6 +219,8 @@ class DDLLMObsSpanTest extends DDSpecification{ def tagVersion = innerSpan.getTag(LLMOBS_TAG_PREFIX + "version") tagVersion instanceof UTF8BytesString "v1" == tagVersion.toString() + + DDTraceApiInfo.VERSION == innerSpan.getTag(LLMOBS_TAG_PREFIX + "ddtrace.version") } def "test llm span string input formatted to messages"() { @@ -267,6 +272,8 @@ class DDLLMObsSpanTest extends DDSpecification{ def tagVersion = innerSpan.getTag(LLMOBS_TAG_PREFIX + "version") tagVersion instanceof UTF8BytesString "v1" == tagVersion.toString() + + DDTraceApiInfo.VERSION == innerSpan.getTag(LLMOBS_TAG_PREFIX + "ddtrace.version") } def "test llm span with messages"() { @@ -323,6 +330,8 @@ class DDLLMObsSpanTest extends DDSpecification{ def tagVersion = innerSpan.getTag(LLMOBS_TAG_PREFIX + "version") tagVersion instanceof UTF8BytesString "v1" == tagVersion.toString() + + DDTraceApiInfo.VERSION == innerSpan.getTag(LLMOBS_TAG_PREFIX + "ddtrace.version") } private LLMObsSpan givenALLMObsSpan(String kind, name){ diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java index c9917332e7e..a992c85400c 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java @@ -21,6 +21,7 @@ interface CommonTags { String ML_APP = TAG_PREFIX + LLMObsTags.ML_APP; String INTEGRATION = TAG_PREFIX + "integration"; String VERSION = TAG_PREFIX + "version"; + String DDTRACE_VERSION = TAG_PREFIX + "ddtrace.version"; String SOURCE = TAG_PREFIX + "source"; String ERROR = TAG_PREFIX + "error"; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java index 033381d60bb..1bb3530fc93 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java @@ -4,6 +4,7 @@ import com.openai.core.http.Headers; import datadog.trace.api.Config; import datadog.trace.api.DDTags; +import datadog.trace.api.DDTraceApiInfo; import datadog.trace.api.WellKnownTags; import datadog.trace.api.llmobs.LLMObsContext; import datadog.trace.api.telemetry.LLMObsMetricCollector; @@ -96,6 +97,7 @@ public AgentSpan afterStart(AgentSpan span) { span.setTag(CommonTags.ENV, wellKnownTags.getEnv()); span.setTag(CommonTags.SERVICE, wellKnownTags.getService()); span.setTag(CommonTags.VERSION, wellKnownTags.getVersion()); + span.setTag(CommonTags.DDTRACE_VERSION, DDTraceApiInfo.VERSION); span.setTag(CommonTags.ML_APP, Config.get().getLlmObsMlApp()); span.setTag(CommonTags.SOURCE, "integration"); diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy index c4a066d1895..28430fcccac 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy @@ -336,6 +336,7 @@ class ChatCompletionServiceTest extends OpenAiTest { "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String + "$CommonTags.DDTRACE_VERSION" String if (expectToolDefinitions) { "$CommonTags.TOOL_DEFINITIONS" List } diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy index de9af838086..dcf537df854 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy @@ -165,6 +165,7 @@ class CompletionServiceTest extends OpenAiTest { "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String + "$CommonTags.DDTRACE_VERSION" String "$CommonTags.SOURCE" "integration" "$CommonTags.INTEGRATION" "openai" "$CommonTags.ERROR" 0 diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy index 112b649a856..32468dd00df 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy @@ -64,6 +64,7 @@ class EmbeddingServiceTest extends OpenAiTest { "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String + "$CommonTags.DDTRACE_VERSION" String "$CommonTags.SOURCE" "integration" "$CommonTags.INTEGRATION" "openai" "$CommonTags.ERROR" 0 diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index 030f61009ac..62f64b04223 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -221,6 +221,7 @@ class ResponseServiceTest extends OpenAiTest { "_ml_obs_tag.ml_app" String "$CommonTags.INTEGRATION" "openai" "_ml_obs_tag.service" String + "$CommonTags.DDTRACE_VERSION" String "$CommonTags.SOURCE" "integration" "$CommonTags.ERROR" 0 if (reasoning != null) { From d57402ef743e1a5cd566cf654bc0d66b8023d548 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Fri, 6 Mar 2026 12:08:53 +0100 Subject: [PATCH 27/28] Improve test assertions --- .../groovy/ChatCompletionServiceTest.groovy | 28 ++++++- .../test/groovy/CompletionServiceTest.groovy | 46 ++++++++--- .../test/groovy/EmbeddingServiceTest.groovy | 15 ++++ .../test/groovy/ResponseServiceTest.groovy | 76 ++++++++++++++++--- .../ddintake/LLMObsSpanMapperTest.groovy | 39 +++++++++- 5 files changed, 177 insertions(+), 27 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy index 28430fcccac..7376f73f92e 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy @@ -147,7 +147,18 @@ class ChatCompletionServiceTest extends OpenAiTest { expect: List outputTag = [] - assertChatCompletionTrace(false, outputTag, [:], true) + List> toolDefinitions = [] + assertChatCompletionTrace(false, outputTag, [:], true, toolDefinitions) + and: + toolDefinitions.size() == 1 + toolDefinitions[0].name == "extract_student_info" + toolDefinitions[0].description == "Get the student information from the body of the input text" + toolDefinitions[0].schema.type == "object" + (toolDefinitions[0].schema.properties as Map).containsKey("name") + (toolDefinitions[0].schema.properties as Map).containsKey("major") + (toolDefinitions[0].schema.properties as Map).containsKey("school") + (toolDefinitions[0].schema.properties as Map).containsKey("grades") + (toolDefinitions[0].schema.properties as Map).containsKey("clubs") and: outputTag.size() == 1 LLMObs.LLMMessage outputMsg = outputTag.get(0) @@ -178,7 +189,12 @@ class ChatCompletionServiceTest extends OpenAiTest { expect: List outputTag = [] - assertChatCompletionTrace(true, outputTag, [stream: true], true) + List> toolDefinitions = [] + assertChatCompletionTrace(true, outputTag, [stream: true], true, toolDefinitions) + and: + toolDefinitions.size() == 1 + toolDefinitions[0].name == "extract_student_info" + toolDefinitions[0].description == "Get the student information from the body of the input text" and: outputTag.size() == 1 LLMObs.LLMMessage outputMsg = outputTag.get(0) @@ -294,10 +310,10 @@ class ChatCompletionServiceTest extends OpenAiTest { } private void assertChatCompletionTrace(boolean isStreaming, List outputTagsOut, Map metadata) { - assertChatCompletionTrace(isStreaming, outputTagsOut, metadata, false) + assertChatCompletionTrace(isStreaming, outputTagsOut, metadata, false, null) } - private void assertChatCompletionTrace(boolean isStreaming, List outputTagsOut, Map metadata, boolean expectToolDefinitions) { + private void assertChatCompletionTrace(boolean isStreaming, List outputTagsOut, Map metadata, boolean expectToolDefinitions, List> toolDefinitionsOut) { def expectedMetadata = new LinkedHashMap(metadata) expectedMetadata.putIfAbsent("stream", isStreaming) @@ -339,6 +355,10 @@ class ChatCompletionServiceTest extends OpenAiTest { "$CommonTags.DDTRACE_VERSION" String if (expectToolDefinitions) { "$CommonTags.TOOL_DEFINITIONS" List + def toolDefinitions = tag("$CommonTags.TOOL_DEFINITIONS") + if (toolDefinitionsOut != null && toolDefinitions != null) { + toolDefinitionsOut.addAll(toolDefinitions) + } } "$CommonTags.SOURCE" "integration" "$CommonTags.INTEGRATION" "openai" diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy index dcf537df854..8ca98ca3677 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy @@ -8,6 +8,7 @@ import com.openai.core.http.HttpResponseFor import com.openai.core.http.StreamResponse import com.openai.models.completions.Completion import datadog.trace.api.DDSpanTypes +import datadog.trace.api.llmobs.LLMObs import datadog.trace.bootstrap.instrumentation.api.Tags import java.util.concurrent.CompletableFuture import java.util.stream.Stream @@ -20,7 +21,7 @@ class CompletionServiceTest extends OpenAiTest { } expect: - assertCompletionTrace() + assertCompletionTrace(false) where: params << [completionCreateParams(true), completionCreateParams(false)] @@ -35,7 +36,7 @@ class CompletionServiceTest extends OpenAiTest { resp.statusCode() == 200 resp.parse().valid // force response parsing, so it sets all the tags and: - assertCompletionTrace() + assertCompletionTrace(false) where: params << [completionCreateParams(true), completionCreateParams(false)] @@ -52,7 +53,7 @@ class CompletionServiceTest extends OpenAiTest { } expect: - assertCompletionTrace() + assertCompletionTrace(true) where: params << [completionCreateStreamedParams(true), completionCreateStreamedParams(false)] @@ -69,7 +70,7 @@ class CompletionServiceTest extends OpenAiTest { } expect: - assertCompletionTrace() + assertCompletionTrace(true) where: params << [completionCreateStreamedParams(true), completionCreateStreamedParams(false)] @@ -83,7 +84,7 @@ class CompletionServiceTest extends OpenAiTest { completionFuture.get() expect: - assertCompletionTrace() + assertCompletionTrace(false) where: params << [completionCreateParams(true), completionCreateParams(false)] @@ -98,7 +99,7 @@ class CompletionServiceTest extends OpenAiTest { resp.parse().valid // force response parsing, so it sets all the tags expect: - assertCompletionTrace() + assertCompletionTrace(false) where: params << [completionCreateParams(true), completionCreateParams(false)] @@ -113,7 +114,7 @@ class CompletionServiceTest extends OpenAiTest { } asyncResp.onCompleteFuture().get() expect: - assertCompletionTrace() + assertCompletionTrace(true) where: params << [completionCreateStreamedParams(true), completionCreateStreamedParams(false)] @@ -131,13 +132,17 @@ class CompletionServiceTest extends OpenAiTest { } expect: resp.statusCode() == 200 - assertCompletionTrace() + assertCompletionTrace(true) where: params << [completionCreateStreamedParams(true), completionCreateStreamedParams(false)] } - private void assertCompletionTrace() { + private void assertCompletionTrace(boolean streamRequest) { + List inputTagsOut = [] + List outputTagsOut = [] + Map metadataOut = [:] + assertTraces(1) { trace(3) { sortSpansByStart() @@ -157,8 +162,20 @@ class CompletionServiceTest extends OpenAiTest { "_ml_obs_tag.model_provider" "openai" "_ml_obs_tag.model_name" String "_ml_obs_tag.metadata" Map + def metadata = tag("_ml_obs_tag.metadata") + if (metadata != null) { + metadataOut.putAll(metadata) + } "_ml_obs_tag.input" List + def inputTags = tag("_ml_obs_tag.input") + if (inputTags != null) { + inputTagsOut.addAll(inputTags) + } "_ml_obs_tag.output" List + def outputTags = tag("_ml_obs_tag.output") + if (outputTags != null) { + outputTagsOut.addAll(outputTags) + } "_ml_obs_metric.input_tokens" Long "_ml_obs_metric.output_tokens" Long "_ml_obs_metric.total_tokens" Long @@ -193,5 +210,16 @@ class CompletionServiceTest extends OpenAiTest { } } } + + assert inputTagsOut.size() == 1 + assert inputTagsOut[0].role == "" + assert inputTagsOut[0].content == "Tell me a story about building the best SDK!" + assert outputTagsOut.size() >= 1 + assert outputTagsOut.every { it.role == "" } + if (streamRequest) { + assert metadataOut.stream_options == [include_usage: true] + } else { + assert !metadataOut.containsKey("stream_options") + } } } diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy index 32468dd00df..0a4f76ff47a 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy @@ -40,6 +40,9 @@ class EmbeddingServiceTest extends OpenAiTest { } private void assertEmbeddingTrace() { + List inputTagsOut = [] + Map metadataOut = [:] + assertTraces(1) { trace(3) { sortSpansByStart() @@ -59,7 +62,15 @@ class EmbeddingServiceTest extends OpenAiTest { "_ml_obs_tag.model_provider" "openai" "_ml_obs_tag.model_name" "text-embedding-ada-002-v2" "_ml_obs_tag.input" List + def inputTags = tag("_ml_obs_tag.input") + if (inputTags != null) { + inputTagsOut.addAll(inputTags) + } "_ml_obs_tag.metadata" Map + def metadata = tag("_ml_obs_tag.metadata") + if (metadata != null) { + metadataOut.putAll(metadata) + } "_ml_obs_tag.output" "[1 embedding(s) returned with size 1536]" "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String @@ -94,5 +105,9 @@ class EmbeddingServiceTest extends OpenAiTest { } } } + + assert inputTagsOut.size() == 1 + assert inputTagsOut[0].text == "hello world" + assert metadataOut == [encoding_format: "base64"] } } diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index 62f64b04223..4d26098050c 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -23,7 +23,10 @@ class ResponseServiceTest extends OpenAiTest { expect: resp != null and: - assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == false where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -38,7 +41,10 @@ class ResponseServiceTest extends OpenAiTest { resp.statusCode() == 200 resp.parse().valid // force response parsing, so it sets all the tags and: - assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == false where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -55,7 +61,10 @@ class ResponseServiceTest extends OpenAiTest { } expect: - assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == true where: scenario | params @@ -76,7 +85,11 @@ class ResponseServiceTest extends OpenAiTest { } expect: - assertResponseTrace(true, "o4-mini", "o4-mini-2025-04-16", [effort: "medium", summary: "detailed"]) + Map metadata = [:] + assertResponseTrace(true, "o4-mini", "o4-mini-2025-04-16", [effort: "medium", summary: "detailed"], null, null, metadata) + and: + metadata.stream == true + metadata.reasoning == [effort: "medium", summary: "detailed"] where: responseCreateParams << [responseCreateParamsWithReasoning(false), responseCreateParamsWithReasoning(true)] @@ -93,7 +106,10 @@ class ResponseServiceTest extends OpenAiTest { } expect: - assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == true where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -107,7 +123,10 @@ class ResponseServiceTest extends OpenAiTest { responseFuture.get() expect: - assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == false where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -122,7 +141,10 @@ class ResponseServiceTest extends OpenAiTest { resp.parse().valid // force response parsing, so it sets all the tags expect: - assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == false where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -137,7 +159,10 @@ class ResponseServiceTest extends OpenAiTest { } asyncResp.onCompleteFuture().get() expect: - assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == true where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -155,7 +180,10 @@ class ResponseServiceTest extends OpenAiTest { } expect: resp.statusCode() == 200 - assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == true where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -173,8 +201,17 @@ class ResponseServiceTest extends OpenAiTest { expect: List inputTags = [] - assertResponseTrace(true, "gpt-4.1", "gpt-4.1-2025-04-14", null, inputTags) + Map metadata = [:] + assertResponseTrace(true, "gpt-4.1", "gpt-4.1-2025-04-14", null, inputTags, null, metadata) and: + metadata.stream == true + inputTags.size() == 3 + inputTags[1].toolCalls.size() == 1 + inputTags[1].toolCalls[0].name == "get_weather" + inputTags[1].toolCalls[0].type == "function_call" + inputTags[1].toolCalls[0].arguments == [location: "San Francisco, CA"] + inputTags[2].toolResults.size() == 1 + inputTags[2].toolResults[0].type == "function_call_output" !inputTags.isEmpty() inputTags[2].toolResults[0].result == '{"temperature": "72°F", "conditions": "sunny", "humidity": "65%"}' @@ -183,10 +220,17 @@ class ResponseServiceTest extends OpenAiTest { } private void assertResponseTrace(boolean isStreaming, String reqModel, String respModel, Map reasoning) { - assertResponseTrace(isStreaming, reqModel, respModel, reasoning, null) + assertResponseTrace(isStreaming, reqModel, respModel, reasoning, null, null, null) } - private void assertResponseTrace(boolean isStreaming, String reqModel, String respModel, Map reasoning, List inputTagsOut) { + private void assertResponseTrace( + boolean isStreaming, + String reqModel, + String respModel, + Map reasoning, + List inputTagsOut, + List outputTagsOut, + Map metadataOut) { assertTraces(1) { trace(3) { sortSpansByStart() @@ -206,12 +250,20 @@ class ResponseServiceTest extends OpenAiTest { "_ml_obs_tag.model_provider" "openai" "_ml_obs_tag.model_name" String "_ml_obs_tag.metadata" Map + def metadata = tag("_ml_obs_tag.metadata") + if (metadataOut != null && metadata != null) { + metadataOut.putAll(metadata) + } "_ml_obs_tag.input" List def inputTags = tag("_ml_obs_tag.input") if (inputTagsOut != null && inputTags != null) { inputTagsOut.addAll(inputTags) } "_ml_obs_tag.output" List + def outputTags = tag("_ml_obs_tag.output") + if (outputTagsOut != null && outputTags != null) { + outputTagsOut.addAll(outputTags) + } "_ml_obs_metric.input_tokens" Long "_ml_obs_metric.output_tokens" Long "_ml_obs_metric.total_tokens" Long diff --git a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy index 3cf52ae0150..6df08aa39ce 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy @@ -40,11 +40,29 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { llmSpan.setSpanType(InternalSpanTypes.LLMOBS) - def inputMessages = [LLMObs.LLMMessage.from("user", "Hello, what's the weather like?")] + def toolCall = LLMObs.ToolCall.from("get_weather", "function_call", "call_123", [location: "San Francisco"]) + def toolResult = LLMObs.ToolResult.from("get_weather", "function_call_output", "call_123", '{"temperature":"72F"}') + def inputMessages = [ + LLMObs.LLMMessage.from("user", "Hello, what's the weather like?"), + LLMObs.LLMMessage.from("assistant", null, [toolCall], [toolResult]) + ] def outputMessages = [LLMObs.LLMMessage.from("assistant", "I'll help you check the weather.")] - llmSpan.setTag("_ml_obs_tag.input", inputMessages) + llmSpan.setTag("_ml_obs_tag.input", [ + messages: inputMessages, + prompt: [ + id: "prompt_123", + version: "1", + variables: [city: "San Francisco"], + chat_template: [[role: "user", content: "Hello, what's the weather like in {{city}}?"]] + ] + ]) llmSpan.setTag("_ml_obs_tag.output", outputMessages) llmSpan.setTag("_ml_obs_tag.metadata", [temperature: 0.7, max_tokens: 100]) + llmSpan.setTag("_ml_obs_tag.tool_definitions", [[ + name: "get_weather", + description: "Get weather by city", + schema: [type: "object", properties: [city: [type: "string"]]] + ]]) llmSpan.setError(true) llmSpan.setTag(DDTags.ERROR_MSG, "boom") llmSpan.setTag(DDTags.ERROR_TYPE, "java.lang.IllegalStateException") @@ -123,12 +141,29 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { spanData["meta"]["input"]["messages"][0]["content"] == "Hello, what's the weather like?" spanData["meta"]["input"]["messages"][0].containsKey("role") spanData["meta"]["input"]["messages"][0]["role"] == "user" + spanData["meta"]["input"]["messages"][1]["role"] == "assistant" + !spanData["meta"]["input"]["messages"][1].containsKey("content") + spanData["meta"]["input"]["messages"][1]["tool_calls"][0]["name"] == "get_weather" + spanData["meta"]["input"]["messages"][1]["tool_calls"][0]["type"] == "function_call" + spanData["meta"]["input"]["messages"][1]["tool_calls"][0]["tool_id"] == "call_123" + spanData["meta"]["input"]["messages"][1]["tool_calls"][0]["arguments"] == [location: "San Francisco"] + spanData["meta"]["input"]["messages"][1]["tool_results"][0]["name"] == "get_weather" + spanData["meta"]["input"]["messages"][1]["tool_results"][0]["type"] == "function_call_output" + spanData["meta"]["input"]["messages"][1]["tool_results"][0]["tool_id"] == "call_123" + spanData["meta"]["input"]["messages"][1]["tool_results"][0]["result"] == '{"temperature":"72F"}' + spanData["meta"]["input"]["prompt"]["id"] == "prompt_123" + spanData["meta"]["input"]["prompt"]["version"] == "1" + spanData["meta"]["input"]["prompt"]["variables"] == [city: "San Francisco"] + spanData["meta"]["input"]["prompt"]["chat_template"] == [[role: "user", content: "Hello, what's the weather like in {{city}}?"]] spanData["meta"].containsKey("output") spanData["meta"]["output"].containsKey("messages") spanData["meta"]["output"]["messages"][0].containsKey("content") spanData["meta"]["output"]["messages"][0]["content"] == "I'll help you check the weather." spanData["meta"]["output"]["messages"][0].containsKey("role") spanData["meta"]["output"]["messages"][0]["role"] == "assistant" + spanData["meta"]["tool_definitions"][0]["name"] == "get_weather" + spanData["meta"]["tool_definitions"][0]["description"] == "Get weather by city" + spanData["meta"]["tool_definitions"][0]["schema"] == [type: "object", properties: [city: [type: "string"]]] spanData["meta"].containsKey("metadata") spanData.containsKey("metrics") From 0c879ba692386cd944b87734bcfce2285beaa37e Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Fri, 6 Mar 2026 12:53:59 +0100 Subject: [PATCH 28/28] Fix format --- .../openai_java/ResponseDecorator.java | 4 ++++ .../src/test/groovy/ResponseServiceTest.groovy | 14 +++++++------- .../writer/ddintake/LLMObsSpanMapperTest.groovy | 12 +++++++----- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 07359a9b326..9488ad0c865 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -710,6 +710,10 @@ private List extractInputMessagesForPromptTracking( } } + if (!messages.isEmpty()) { + return messages; + } + Object inputTag = span.getTag(CommonTags.INPUT); if (inputTag instanceof List) { for (Object messageObj : (List) inputTag) { diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index 4d26098050c..02e28237750 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -224,13 +224,13 @@ class ResponseServiceTest extends OpenAiTest { } private void assertResponseTrace( - boolean isStreaming, - String reqModel, - String respModel, - Map reasoning, - List inputTagsOut, - List outputTagsOut, - Map metadataOut) { + boolean isStreaming, + String reqModel, + String respModel, + Map reasoning, + List inputTagsOut, + List outputTagsOut, + Map metadataOut) { assertTraces(1) { trace(3) { sortSpansByStart() diff --git a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy index 6df08aa39ce..7d7de1180a7 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy @@ -58,11 +58,13 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { ]) llmSpan.setTag("_ml_obs_tag.output", outputMessages) llmSpan.setTag("_ml_obs_tag.metadata", [temperature: 0.7, max_tokens: 100]) - llmSpan.setTag("_ml_obs_tag.tool_definitions", [[ - name: "get_weather", - description: "Get weather by city", - schema: [type: "object", properties: [city: [type: "string"]]] - ]]) + llmSpan.setTag("_ml_obs_tag.tool_definitions", [ + [ + name: "get_weather", + description: "Get weather by city", + schema: [type: "object", properties: [city: [type: "string"]]] + ] + ]) llmSpan.setError(true) llmSpan.setTag(DDTags.ERROR_MSG, "boom") llmSpan.setTag(DDTags.ERROR_TYPE, "java.lang.IllegalStateException")